1 //
2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return 0; // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // lea(rscratch1, RuntimeAddress(addr)
1652 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else if (_entry_point == nullptr) {
1658 // See CallLeafNoFPIndirect
1659 return 1 * NativeInstruction::instruction_size;
1660 } else {
1661 return 6 * NativeInstruction::instruction_size;
1662 }
1663 }
1664
1665 //=============================================================================
1666
1667 #ifndef PRODUCT
1668 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1669 st->print("BREAKPOINT");
1670 }
1671 #endif
1672
1673 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1674 __ brk(0);
1675 }
1676
1677 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1678 return MachNode::size(ra_);
1679 }
1680
1681 //=============================================================================
1682
1683 #ifndef PRODUCT
1684 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1685 st->print("nop \t# %d bytes pad for loops and calls", _count);
1686 }
1687 #endif
1688
1689 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1690 for (int i = 0; i < _count; i++) {
1691 __ nop();
1692 }
1693 }
1694
1695 uint MachNopNode::size(PhaseRegAlloc*) const {
1696 return _count * NativeInstruction::instruction_size;
1697 }
1698
1699 //=============================================================================
1700 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1701
1702 int ConstantTable::calculate_table_base_offset() const {
1703 return 0; // absolute addressing, no offset
1704 }
1705
1706 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1707 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1708 ShouldNotReachHere();
1709 }
1710
1711 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1712 // Empty encoding
1713 }
1714
1715 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1716 return 0;
1717 }
1718
1719 #ifndef PRODUCT
1720 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1721 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1722 }
1723 #endif
1724
1725 #ifndef PRODUCT
1726 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1727 Compile* C = ra_->C;
1728
1729 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1730
1731 if (C->output()->need_stack_bang(framesize))
1732 st->print("# stack bang size=%d\n\t", framesize);
1733
1734 if (VM_Version::use_rop_protection()) {
1735 st->print("ldr zr, [lr]\n\t");
1736 st->print("paciaz\n\t");
1737 }
1738 if (framesize < ((1 << 9) + 2 * wordSize)) {
1739 st->print("sub sp, sp, #%d\n\t", framesize);
1740 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1741 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1742 } else {
1743 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1744 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1745 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1746 st->print("sub sp, sp, rscratch1");
1747 }
1748 if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1749 st->print("\n\t");
1750 st->print("ldr rscratch1, [guard]\n\t");
1751 st->print("dmb ishld\n\t");
1752 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1753 st->print("cmp rscratch1, rscratch2\n\t");
1754 st->print("b.eq skip");
1755 st->print("\n\t");
1756 st->print("blr #nmethod_entry_barrier_stub\n\t");
1757 st->print("b skip\n\t");
1758 st->print("guard: int\n\t");
1759 st->print("\n\t");
1760 st->print("skip:\n\t");
1761 }
1762 }
1763 #endif
1764
1765 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1766 Compile* C = ra_->C;
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 __ verified_entry(C, 0);
1773
1774 if (C->stub_function() == nullptr) {
1775 __ entry_barrier();
1776 }
1777
1778 if (!Compile::current()->output()->in_scratch_emit_size()) {
1779 __ bind(*_verified_entry);
1780 }
1781
1782 if (VerifyStackAtCalls) {
1783 Unimplemented();
1784 }
1785
1786 C->output()->set_frame_complete(__ offset());
1787
1788 if (C->has_mach_constant_base_node()) {
1789 // NOTE: We set the table base offset here because users might be
1790 // emitted before MachConstantBaseNode.
1791 ConstantTable& constant_table = C->output()->constant_table();
1792 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1793 }
1794 }
1795
1796 int MachPrologNode::reloc() const
1797 {
1798 return 0;
1799 }
1800
1801 //=============================================================================
1802
1803 #ifndef PRODUCT
1804 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1805 Compile* C = ra_->C;
1806 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1807
1808 st->print("# pop frame %d\n\t",framesize);
1809
1810 if (framesize == 0) {
1811 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1812 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1813 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1814 st->print("add sp, sp, #%d\n\t", framesize);
1815 } else {
1816 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1817 st->print("add sp, sp, rscratch1\n\t");
1818 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1819 }
1820 if (VM_Version::use_rop_protection()) {
1821 st->print("autiaz\n\t");
1822 st->print("ldr zr, [lr]\n\t");
1823 }
1824
1825 if (do_polling() && C->is_method_compilation()) {
1826 st->print("# test polling word\n\t");
1827 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1828 st->print("cmp sp, rscratch1\n\t");
1829 st->print("bhi #slow_path");
1830 }
1831 }
1832 #endif
1833
1834 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1835 Compile* C = ra_->C;
1836 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1837
1838 __ remove_frame(framesize, C->needs_stack_repair());
1839
1840 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1841 __ reserved_stack_check();
1842 }
1843
1844 if (do_polling() && C->is_method_compilation()) {
1845 Label dummy_label;
1846 Label* code_stub = &dummy_label;
1847 if (!C->output()->in_scratch_emit_size()) {
1848 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1849 C->output()->add_stub(stub);
1850 code_stub = &stub->entry();
1851 }
1852 __ relocate(relocInfo::poll_return_type);
1853 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1854 }
1855 }
1856
1857 int MachEpilogNode::reloc() const {
1858 // Return number of relocatable values contained in this instruction.
1859 return 1; // 1 for polling page.
1860 }
1861
1862 const Pipeline * MachEpilogNode::pipeline() const {
1863 return MachNode::pipeline_class();
1864 }
1865
1866 //=============================================================================
1867
1868 static enum RC rc_class(OptoReg::Name reg) {
1869
1870 if (reg == OptoReg::Bad) {
1871 return rc_bad;
1872 }
1873
1874 // we have 32 int registers * 2 halves
1875 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1876
1877 if (reg < slots_of_int_registers) {
1878 return rc_int;
1879 }
1880
1881 // we have 32 float register * 8 halves
1882 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1883 if (reg < slots_of_int_registers + slots_of_float_registers) {
1884 return rc_float;
1885 }
1886
1887 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1888 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1889 return rc_predicate;
1890 }
1891
1892 // Between predicate regs & stack is the flags.
1893 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1894
1895 return rc_stack;
1896 }
1897
1898 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1899 Compile* C = ra_->C;
1900
1901 // Get registers to move.
1902 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1903 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1904 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1905 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1906
1907 enum RC src_hi_rc = rc_class(src_hi);
1908 enum RC src_lo_rc = rc_class(src_lo);
1909 enum RC dst_hi_rc = rc_class(dst_hi);
1910 enum RC dst_lo_rc = rc_class(dst_lo);
1911
1912 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1913
1914 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1915 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1916 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1917 "expected aligned-adjacent pairs");
1918 }
1919
1920 if (src_lo == dst_lo && src_hi == dst_hi) {
1921 return 0; // Self copy, no move.
1922 }
1923
1924 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1925 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1926 int src_offset = ra_->reg2offset(src_lo);
1927 int dst_offset = ra_->reg2offset(dst_lo);
1928
1929 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1930 uint ireg = ideal_reg();
1931 if (ireg == Op_VecA && masm) {
1932 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1933 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1934 // stack->stack
1935 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1936 sve_vector_reg_size_in_bytes);
1937 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1938 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1939 sve_vector_reg_size_in_bytes);
1940 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1941 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1942 sve_vector_reg_size_in_bytes);
1943 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1944 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1945 as_FloatRegister(Matcher::_regEncode[src_lo]),
1946 as_FloatRegister(Matcher::_regEncode[src_lo]));
1947 } else {
1948 ShouldNotReachHere();
1949 }
1950 } else if (masm) {
1951 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1952 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1953 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1954 // stack->stack
1955 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1956 if (ireg == Op_VecD) {
1957 __ unspill(rscratch1, true, src_offset);
1958 __ spill(rscratch1, true, dst_offset);
1959 } else {
1960 __ spill_copy128(src_offset, dst_offset);
1961 }
1962 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1963 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1964 ireg == Op_VecD ? __ T8B : __ T16B,
1965 as_FloatRegister(Matcher::_regEncode[src_lo]));
1966 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1967 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1968 ireg == Op_VecD ? __ D : __ Q,
1969 ra_->reg2offset(dst_lo));
1970 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1971 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1972 ireg == Op_VecD ? __ D : __ Q,
1973 ra_->reg2offset(src_lo));
1974 } else {
1975 ShouldNotReachHere();
1976 }
1977 }
1978 } else if (masm) {
1979 switch (src_lo_rc) {
1980 case rc_int:
1981 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
1982 if (is64) {
1983 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1984 as_Register(Matcher::_regEncode[src_lo]));
1985 } else {
1986 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1987 as_Register(Matcher::_regEncode[src_lo]));
1988 }
1989 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1990 if (is64) {
1991 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1992 as_Register(Matcher::_regEncode[src_lo]));
1993 } else {
1994 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1995 as_Register(Matcher::_regEncode[src_lo]));
1996 }
1997 } else { // gpr --> stack spill
1998 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1999 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2000 }
2001 break;
2002 case rc_float:
2003 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2004 if (is64) {
2005 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2006 as_FloatRegister(Matcher::_regEncode[src_lo]));
2007 } else {
2008 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2009 as_FloatRegister(Matcher::_regEncode[src_lo]));
2010 }
2011 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2012 if (is64) {
2013 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2014 as_FloatRegister(Matcher::_regEncode[src_lo]));
2015 } else {
2016 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 as_FloatRegister(Matcher::_regEncode[src_lo]));
2018 }
2019 } else { // fpr --> stack spill
2020 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2021 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2022 is64 ? __ D : __ S, dst_offset);
2023 }
2024 break;
2025 case rc_stack:
2026 if (dst_lo_rc == rc_int) { // stack --> gpr load
2027 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2028 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2029 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2030 is64 ? __ D : __ S, src_offset);
2031 } else if (dst_lo_rc == rc_predicate) {
2032 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2033 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2034 } else { // stack --> stack copy
2035 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2036 if (ideal_reg() == Op_RegVectMask) {
2037 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2038 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2039 } else {
2040 __ unspill(rscratch1, is64, src_offset);
2041 __ spill(rscratch1, is64, dst_offset);
2042 }
2043 }
2044 break;
2045 case rc_predicate:
2046 if (dst_lo_rc == rc_predicate) {
2047 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2048 } else if (dst_lo_rc == rc_stack) {
2049 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2050 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2051 } else {
2052 assert(false, "bad src and dst rc_class combination.");
2053 ShouldNotReachHere();
2054 }
2055 break;
2056 default:
2057 assert(false, "bad rc_class for spill");
2058 ShouldNotReachHere();
2059 }
2060 }
2061
2062 if (st) {
2063 st->print("spill ");
2064 if (src_lo_rc == rc_stack) {
2065 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2066 } else {
2067 st->print("%s -> ", Matcher::regName[src_lo]);
2068 }
2069 if (dst_lo_rc == rc_stack) {
2070 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2071 } else {
2072 st->print("%s", Matcher::regName[dst_lo]);
2073 }
2074 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2075 int vsize = 0;
2076 switch (ideal_reg()) {
2077 case Op_VecD:
2078 vsize = 64;
2079 break;
2080 case Op_VecX:
2081 vsize = 128;
2082 break;
2083 case Op_VecA:
2084 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2085 break;
2086 default:
2087 assert(false, "bad register type for spill");
2088 ShouldNotReachHere();
2089 }
2090 st->print("\t# vector spill size = %d", vsize);
2091 } else if (ideal_reg() == Op_RegVectMask) {
2092 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2093 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2094 st->print("\t# predicate spill size = %d", vsize);
2095 } else {
2096 st->print("\t# spill size = %d", is64 ? 64 : 32);
2097 }
2098 }
2099
2100 return 0;
2101
2102 }
2103
2104 #ifndef PRODUCT
2105 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2106 if (!ra_)
2107 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2108 else
2109 implementation(nullptr, ra_, false, st);
2110 }
2111 #endif
2112
2113 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2114 implementation(masm, ra_, false, nullptr);
2115 }
2116
2117 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2118 return MachNode::size(ra_);
2119 }
2120
2121 //=============================================================================
2122
2123 #ifndef PRODUCT
2124 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2125 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2126 int reg = ra_->get_reg_first(this);
2127 st->print("add %s, rsp, #%d]\t# box lock",
2128 Matcher::regName[reg], offset);
2129 }
2130 #endif
2131
2132 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2133 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2134 int reg = ra_->get_encode(this);
2135
2136 // This add will handle any 24-bit signed offset. 24 bits allows an
2137 // 8 megabyte stack frame.
2138 __ add(as_Register(reg), sp, offset);
2139 }
2140
2141 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2142 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2143 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2144
2145 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2146 return NativeInstruction::instruction_size;
2147 } else {
2148 return 2 * NativeInstruction::instruction_size;
2149 }
2150 }
2151
2152 ///=============================================================================
2153 #ifndef PRODUCT
2154 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2155 {
2156 st->print_cr("# MachVEPNode");
2157 if (!_verified) {
2158 st->print_cr("\t load_class");
2159 } else {
2160 st->print_cr("\t unpack_inline_arg");
2161 }
2162 }
2163 #endif
2164
2165 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2166 {
2167 if (!_verified) {
2168 __ ic_check(1);
2169 } else {
2170 // insert a nop at the start of the prolog so we can patch in a
2171 // branch if we need to invalidate the method later
2172 __ nop();
2173
2174 // TODO 8284443 Avoid creation of temporary frame
2175 if (ra_->C->stub_function() == nullptr) {
2176 __ verified_entry(ra_->C, 0);
2177 __ entry_barrier();
2178 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2179 __ remove_frame(framesize, false);
2180 }
2181 // Unpack inline type args passed as oop and then jump to
2182 // the verified entry point (skipping the unverified entry).
2183 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2184 // Emit code for verified entry and save increment for stack repair on return
2185 __ verified_entry(ra_->C, sp_inc);
2186 if (Compile::current()->output()->in_scratch_emit_size()) {
2187 Label dummy_verified_entry;
2188 __ b(dummy_verified_entry);
2189 } else {
2190 __ b(*_verified_entry);
2191 }
2192 }
2193 }
2194
2195 //=============================================================================
2196 #ifndef PRODUCT
2197 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2198 {
2199 st->print_cr("# MachUEPNode");
2200 if (UseCompressedClassPointers) {
2201 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2202 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2203 st->print_cr("\tcmpw rscratch1, r10");
2204 } else {
2205 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2206 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2207 st->print_cr("\tcmp rscratch1, r10");
2208 }
2209 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2210 }
2211 #endif
2212
2213 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2214 {
2215 __ ic_check(InteriorEntryAlignment);
2216 }
2217
2218 // REQUIRED EMIT CODE
2219
2220 //=============================================================================
2221
2222 // Emit exception handler code.
2223 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2224 {
2225 // mov rscratch1 #exception_blob_entry_point
2226 // br rscratch1
2227 // Note that the code buffer's insts_mark is always relative to insts.
2228 // That's why we must use the macroassembler to generate a handler.
2229 address base = __ start_a_stub(size_exception_handler());
2230 if (base == nullptr) {
2231 ciEnv::current()->record_failure("CodeCache is full");
2232 return 0; // CodeBuffer::expand failed
2233 }
2234 int offset = __ offset();
2235 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2236 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2237 __ end_a_stub();
2238 return offset;
2239 }
2240
2241 // Emit deopt handler code.
2242 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2243 {
2244 // Note that the code buffer's insts_mark is always relative to insts.
2245 // That's why we must use the macroassembler to generate a handler.
2246 address base = __ start_a_stub(size_deopt_handler());
2247 if (base == nullptr) {
2248 ciEnv::current()->record_failure("CodeCache is full");
2249 return 0; // CodeBuffer::expand failed
2250 }
2251 int offset = __ offset();
2252
2253 __ adr(lr, __ pc());
2254 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2255
2256 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2257 __ end_a_stub();
2258 return offset;
2259 }
2260
2261 // REQUIRED MATCHER CODE
2262
2263 //=============================================================================
2264
2265 bool Matcher::match_rule_supported(int opcode) {
2266 if (!has_match_rule(opcode))
2267 return false;
2268
2269 switch (opcode) {
2270 case Op_OnSpinWait:
2271 return VM_Version::supports_on_spin_wait();
2272 case Op_CacheWB:
2273 case Op_CacheWBPreSync:
2274 case Op_CacheWBPostSync:
2275 if (!VM_Version::supports_data_cache_line_flush()) {
2276 return false;
2277 }
2278 break;
2279 case Op_ExpandBits:
2280 case Op_CompressBits:
2281 if (!VM_Version::supports_svebitperm()) {
2282 return false;
2283 }
2284 break;
2285 case Op_FmaF:
2286 case Op_FmaD:
2287 case Op_FmaVF:
2288 case Op_FmaVD:
2289 if (!UseFMA) {
2290 return false;
2291 }
2292 break;
2293 }
2294
2295 return true; // Per default match rules are supported.
2296 }
2297
2298 const RegMask* Matcher::predicate_reg_mask(void) {
2299 return &_PR_REG_mask;
2300 }
2301
2302 bool Matcher::supports_vector_calling_convention(void) {
2303 return EnableVectorSupport && UseVectorStubs;
2304 }
2305
2306 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2307 assert(EnableVectorSupport && UseVectorStubs, "sanity");
2308 int lo = V0_num;
2309 int hi = V0_H_num;
2310 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2311 hi = V0_K_num;
2312 }
2313 return OptoRegPair(hi, lo);
2314 }
2315
2316 // Is this branch offset short enough that a short branch can be used?
2317 //
2318 // NOTE: If the platform does not provide any short branch variants, then
2319 // this method should return false for offset 0.
2320 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2321 // The passed offset is relative to address of the branch.
2322
2323 return (-32768 <= offset && offset < 32768);
2324 }
2325
2326 // Vector width in bytes.
2327 int Matcher::vector_width_in_bytes(BasicType bt) {
2328 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2329 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2330 // Minimum 2 values in vector
2331 if (size < 2*type2aelembytes(bt)) size = 0;
2332 // But never < 4
2333 if (size < 4) size = 0;
2334 return size;
2335 }
2336
2337 // Limits on vector size (number of elements) loaded into vector.
2338 int Matcher::max_vector_size(const BasicType bt) {
2339 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2340 }
2341
2342 int Matcher::min_vector_size(const BasicType bt) {
2343 int max_size = max_vector_size(bt);
2344 // Limit the min vector size to 8 bytes.
2345 int size = 8 / type2aelembytes(bt);
2346 if (bt == T_BYTE) {
2347 // To support vector api shuffle/rearrange.
2348 size = 4;
2349 } else if (bt == T_BOOLEAN) {
2350 // To support vector api load/store mask.
2351 size = 2;
2352 }
2353 if (size < 2) size = 2;
2354 return MIN2(size, max_size);
2355 }
2356
2357 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2358 return Matcher::max_vector_size(bt);
2359 }
2360
2361 // Actual max scalable vector register length.
2362 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2363 return Matcher::max_vector_size(bt);
2364 }
2365
2366 // Vector ideal reg.
2367 uint Matcher::vector_ideal_reg(int len) {
2368 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2369 return Op_VecA;
2370 }
2371 switch(len) {
2372 // For 16-bit/32-bit mask vector, reuse VecD.
2373 case 2:
2374 case 4:
2375 case 8: return Op_VecD;
2376 case 16: return Op_VecX;
2377 }
2378 ShouldNotReachHere();
2379 return 0;
2380 }
2381
2382 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2383 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2384 switch (ideal_reg) {
2385 case Op_VecA: return new vecAOper();
2386 case Op_VecD: return new vecDOper();
2387 case Op_VecX: return new vecXOper();
2388 }
2389 ShouldNotReachHere();
2390 return nullptr;
2391 }
2392
2393 bool Matcher::is_reg2reg_move(MachNode* m) {
2394 return false;
2395 }
2396
2397 bool Matcher::is_generic_vector(MachOper* opnd) {
2398 return opnd->opcode() == VREG;
2399 }
2400
2401 // Return whether or not this register is ever used as an argument.
2402 // This function is used on startup to build the trampoline stubs in
2403 // generateOptoStub. Registers not mentioned will be killed by the VM
2404 // call in the trampoline, and arguments in those registers not be
2405 // available to the callee.
2406 bool Matcher::can_be_java_arg(int reg)
2407 {
2408 return
2409 reg == R0_num || reg == R0_H_num ||
2410 reg == R1_num || reg == R1_H_num ||
2411 reg == R2_num || reg == R2_H_num ||
2412 reg == R3_num || reg == R3_H_num ||
2413 reg == R4_num || reg == R4_H_num ||
2414 reg == R5_num || reg == R5_H_num ||
2415 reg == R6_num || reg == R6_H_num ||
2416 reg == R7_num || reg == R7_H_num ||
2417 reg == V0_num || reg == V0_H_num ||
2418 reg == V1_num || reg == V1_H_num ||
2419 reg == V2_num || reg == V2_H_num ||
2420 reg == V3_num || reg == V3_H_num ||
2421 reg == V4_num || reg == V4_H_num ||
2422 reg == V5_num || reg == V5_H_num ||
2423 reg == V6_num || reg == V6_H_num ||
2424 reg == V7_num || reg == V7_H_num;
2425 }
2426
2427 bool Matcher::is_spillable_arg(int reg)
2428 {
2429 return can_be_java_arg(reg);
2430 }
2431
2432 uint Matcher::int_pressure_limit()
2433 {
2434 // JDK-8183543: When taking the number of available registers as int
2435 // register pressure threshold, the jtreg test:
2436 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2437 // failed due to C2 compilation failure with
2438 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2439 //
2440 // A derived pointer is live at CallNode and then is flagged by RA
2441 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2442 // derived pointers and lastly fail to spill after reaching maximum
2443 // number of iterations. Lowering the default pressure threshold to
2444 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2445 // a high register pressure area of the code so that split_DEF can
2446 // generate DefinitionSpillCopy for the derived pointer.
2447 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2448 if (!PreserveFramePointer) {
2449 // When PreserveFramePointer is off, frame pointer is allocatable,
2450 // but different from other SOC registers, it is excluded from
2451 // fatproj's mask because its save type is No-Save. Decrease 1 to
2452 // ensure high pressure at fatproj when PreserveFramePointer is off.
2453 // See check_pressure_at_fatproj().
2454 default_int_pressure_threshold--;
2455 }
2456 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2457 }
2458
2459 uint Matcher::float_pressure_limit()
2460 {
2461 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2462 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2463 }
2464
2465 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2466 return false;
2467 }
2468
2469 RegMask Matcher::divI_proj_mask() {
2470 ShouldNotReachHere();
2471 return RegMask();
2472 }
2473
2474 // Register for MODI projection of divmodI.
2475 RegMask Matcher::modI_proj_mask() {
2476 ShouldNotReachHere();
2477 return RegMask();
2478 }
2479
2480 // Register for DIVL projection of divmodL.
2481 RegMask Matcher::divL_proj_mask() {
2482 ShouldNotReachHere();
2483 return RegMask();
2484 }
2485
2486 // Register for MODL projection of divmodL.
2487 RegMask Matcher::modL_proj_mask() {
2488 ShouldNotReachHere();
2489 return RegMask();
2490 }
2491
2492 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2493 return FP_REG_mask();
2494 }
2495
2496 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2497 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2498 Node* u = addp->fast_out(i);
2499 if (u->is_LoadStore()) {
2500 // On AArch64, LoadStoreNodes (i.e. compare and swap
2501 // instructions) only take register indirect as an operand, so
2502 // any attempt to use an AddPNode as an input to a LoadStoreNode
2503 // must fail.
2504 return false;
2505 }
2506 if (u->is_Mem()) {
2507 int opsize = u->as_Mem()->memory_size();
2508 assert(opsize > 0, "unexpected memory operand size");
2509 if (u->as_Mem()->memory_size() != (1<<shift)) {
2510 return false;
2511 }
2512 }
2513 }
2514 return true;
2515 }
2516
2517 // Convert BootTest condition to Assembler condition.
2518 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2519 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2520 Assembler::Condition result;
2521 switch(cond) {
2522 case BoolTest::eq:
2523 result = Assembler::EQ; break;
2524 case BoolTest::ne:
2525 result = Assembler::NE; break;
2526 case BoolTest::le:
2527 result = Assembler::LE; break;
2528 case BoolTest::ge:
2529 result = Assembler::GE; break;
2530 case BoolTest::lt:
2531 result = Assembler::LT; break;
2532 case BoolTest::gt:
2533 result = Assembler::GT; break;
2534 case BoolTest::ule:
2535 result = Assembler::LS; break;
2536 case BoolTest::uge:
2537 result = Assembler::HS; break;
2538 case BoolTest::ult:
2539 result = Assembler::LO; break;
2540 case BoolTest::ugt:
2541 result = Assembler::HI; break;
2542 case BoolTest::overflow:
2543 result = Assembler::VS; break;
2544 case BoolTest::no_overflow:
2545 result = Assembler::VC; break;
2546 default:
2547 ShouldNotReachHere();
2548 return Assembler::Condition(-1);
2549 }
2550
2551 // Check conversion
2552 if (cond & BoolTest::unsigned_compare) {
2553 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2554 } else {
2555 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2556 }
2557
2558 return result;
2559 }
2560
2561 // Binary src (Replicate con)
2562 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2563 if (n == nullptr || m == nullptr) {
2564 return false;
2565 }
2566
2567 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2568 return false;
2569 }
2570
2571 Node* imm_node = m->in(1);
2572 if (!imm_node->is_Con()) {
2573 return false;
2574 }
2575
2576 const Type* t = imm_node->bottom_type();
2577 if (!(t->isa_int() || t->isa_long())) {
2578 return false;
2579 }
2580
2581 switch (n->Opcode()) {
2582 case Op_AndV:
2583 case Op_OrV:
2584 case Op_XorV: {
2585 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2586 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2587 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2588 }
2589 case Op_AddVB:
2590 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2591 case Op_AddVS:
2592 case Op_AddVI:
2593 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2594 case Op_AddVL:
2595 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2596 default:
2597 return false;
2598 }
2599 }
2600
2601 // (XorV src (Replicate m1))
2602 // (XorVMask src (MaskAll m1))
2603 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2604 if (n != nullptr && m != nullptr) {
2605 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2606 VectorNode::is_all_ones_vector(m);
2607 }
2608 return false;
2609 }
2610
2611 // Should the matcher clone input 'm' of node 'n'?
2612 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2613 if (is_vshift_con_pattern(n, m) ||
2614 is_vector_bitwise_not_pattern(n, m) ||
2615 is_valid_sve_arith_imm_pattern(n, m) ||
2616 is_encode_and_store_pattern(n, m)) {
2617 mstack.push(m, Visit);
2618 return true;
2619 }
2620 return false;
2621 }
2622
2623 // Should the Matcher clone shifts on addressing modes, expecting them
2624 // to be subsumed into complex addressing expressions or compute them
2625 // into registers?
2626 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2627 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2628 return true;
2629 }
2630
2631 Node *off = m->in(AddPNode::Offset);
2632 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2633 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2634 // Are there other uses besides address expressions?
2635 !is_visited(off)) {
2636 address_visited.set(off->_idx); // Flag as address_visited
2637 mstack.push(off->in(2), Visit);
2638 Node *conv = off->in(1);
2639 if (conv->Opcode() == Op_ConvI2L &&
2640 // Are there other uses besides address expressions?
2641 !is_visited(conv)) {
2642 address_visited.set(conv->_idx); // Flag as address_visited
2643 mstack.push(conv->in(1), Pre_Visit);
2644 } else {
2645 mstack.push(conv, Pre_Visit);
2646 }
2647 address_visited.test_set(m->_idx); // Flag as address_visited
2648 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2649 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2650 return true;
2651 } else if (off->Opcode() == Op_ConvI2L &&
2652 // Are there other uses besides address expressions?
2653 !is_visited(off)) {
2654 address_visited.test_set(m->_idx); // Flag as address_visited
2655 address_visited.set(off->_idx); // Flag as address_visited
2656 mstack.push(off->in(1), Pre_Visit);
2657 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2658 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2659 return true;
2660 }
2661 return false;
2662 }
2663
2664 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2665 { \
2666 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2667 guarantee(DISP == 0, "mode not permitted for volatile"); \
2668 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2669 __ INSN(REG, as_Register(BASE)); \
2670 }
2671
2672
2673 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2674 {
2675 Address::extend scale;
2676
2677 // Hooboy, this is fugly. We need a way to communicate to the
2678 // encoder that the index needs to be sign extended, so we have to
2679 // enumerate all the cases.
2680 switch (opcode) {
2681 case INDINDEXSCALEDI2L:
2682 case INDINDEXSCALEDI2LN:
2683 case INDINDEXI2L:
2684 case INDINDEXI2LN:
2685 scale = Address::sxtw(size);
2686 break;
2687 default:
2688 scale = Address::lsl(size);
2689 }
2690
2691 if (index == -1) {
2692 return Address(base, disp);
2693 } else {
2694 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2695 return Address(base, as_Register(index), scale);
2696 }
2697 }
2698
2699
2700 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2701 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2702 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2703 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2704 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2705
2706 // Used for all non-volatile memory accesses. The use of
2707 // $mem->opcode() to discover whether this pattern uses sign-extended
2708 // offsets is something of a kludge.
2709 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2710 Register reg, int opcode,
2711 Register base, int index, int scale, int disp,
2712 int size_in_memory)
2713 {
2714 Address addr = mem2address(opcode, base, index, scale, disp);
2715 if (addr.getMode() == Address::base_plus_offset) {
2716 /* Fix up any out-of-range offsets. */
2717 assert_different_registers(rscratch1, base);
2718 assert_different_registers(rscratch1, reg);
2719 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2720 }
2721 (masm->*insn)(reg, addr);
2722 }
2723
2724 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2725 FloatRegister reg, int opcode,
2726 Register base, int index, int size, int disp,
2727 int size_in_memory)
2728 {
2729 Address::extend scale;
2730
2731 switch (opcode) {
2732 case INDINDEXSCALEDI2L:
2733 case INDINDEXSCALEDI2LN:
2734 scale = Address::sxtw(size);
2735 break;
2736 default:
2737 scale = Address::lsl(size);
2738 }
2739
2740 if (index == -1) {
2741 // Fix up any out-of-range offsets.
2742 assert_different_registers(rscratch1, base);
2743 Address addr = Address(base, disp);
2744 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2745 (masm->*insn)(reg, addr);
2746 } else {
2747 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2748 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2749 }
2750 }
2751
2752 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2753 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2754 int opcode, Register base, int index, int size, int disp)
2755 {
2756 if (index == -1) {
2757 (masm->*insn)(reg, T, Address(base, disp));
2758 } else {
2759 assert(disp == 0, "unsupported address mode");
2760 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2761 }
2762 }
2763
2764 %}
2765
2766
2767
2768 //----------ENCODING BLOCK-----------------------------------------------------
2769 // This block specifies the encoding classes used by the compiler to
2770 // output byte streams. Encoding classes are parameterized macros
2771 // used by Machine Instruction Nodes in order to generate the bit
2772 // encoding of the instruction. Operands specify their base encoding
2773 // interface with the interface keyword. There are currently
2774 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2775 // COND_INTER. REG_INTER causes an operand to generate a function
2776 // which returns its register number when queried. CONST_INTER causes
2777 // an operand to generate a function which returns the value of the
2778 // constant when queried. MEMORY_INTER causes an operand to generate
2779 // four functions which return the Base Register, the Index Register,
2780 // the Scale Value, and the Offset Value of the operand when queried.
2781 // COND_INTER causes an operand to generate six functions which return
2782 // the encoding code (ie - encoding bits for the instruction)
2783 // associated with each basic boolean condition for a conditional
2784 // instruction.
2785 //
2786 // Instructions specify two basic values for encoding. Again, a
2787 // function is available to check if the constant displacement is an
2788 // oop. They use the ins_encode keyword to specify their encoding
2789 // classes (which must be a sequence of enc_class names, and their
2790 // parameters, specified in the encoding block), and they use the
2791 // opcode keyword to specify, in order, their primary, secondary, and
2792 // tertiary opcode. Only the opcode sections which a particular
2793 // instruction needs for encoding need to be specified.
2794 encode %{
2795 // Build emit functions for each basic byte or larger field in the
2796 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2797 // from C++ code in the enc_class source block. Emit functions will
2798 // live in the main source block for now. In future, we can
2799 // generalize this by adding a syntax that specifies the sizes of
2800 // fields in an order, so that the adlc can build the emit functions
2801 // automagically
2802
2803 // catch all for unimplemented encodings
2804 enc_class enc_unimplemented %{
2805 __ unimplemented("C2 catch all");
2806 %}
2807
2808 // BEGIN Non-volatile memory access
2809
2810 // This encoding class is generated automatically from ad_encode.m4.
2811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2812 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2813 Register dst_reg = as_Register($dst$$reg);
2814 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2815 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2816 %}
2817
2818 // This encoding class is generated automatically from ad_encode.m4.
2819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2820 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2821 Register dst_reg = as_Register($dst$$reg);
2822 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2823 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2824 %}
2825
2826 // This encoding class is generated automatically from ad_encode.m4.
2827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2828 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2829 Register dst_reg = as_Register($dst$$reg);
2830 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2831 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2832 %}
2833
2834 // This encoding class is generated automatically from ad_encode.m4.
2835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2836 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2837 Register dst_reg = as_Register($dst$$reg);
2838 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2839 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2840 %}
2841
2842 // This encoding class is generated automatically from ad_encode.m4.
2843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2844 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2845 Register dst_reg = as_Register($dst$$reg);
2846 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2847 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2848 %}
2849
2850 // This encoding class is generated automatically from ad_encode.m4.
2851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2852 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2853 Register dst_reg = as_Register($dst$$reg);
2854 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2855 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2856 %}
2857
2858 // This encoding class is generated automatically from ad_encode.m4.
2859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2860 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2861 Register dst_reg = as_Register($dst$$reg);
2862 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2863 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2864 %}
2865
2866 // This encoding class is generated automatically from ad_encode.m4.
2867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2868 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2869 Register dst_reg = as_Register($dst$$reg);
2870 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2871 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2872 %}
2873
2874 // This encoding class is generated automatically from ad_encode.m4.
2875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2876 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2877 Register dst_reg = as_Register($dst$$reg);
2878 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2879 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2880 %}
2881
2882 // This encoding class is generated automatically from ad_encode.m4.
2883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2884 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2885 Register dst_reg = as_Register($dst$$reg);
2886 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2887 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2888 %}
2889
2890 // This encoding class is generated automatically from ad_encode.m4.
2891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2892 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2893 Register dst_reg = as_Register($dst$$reg);
2894 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2895 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2896 %}
2897
2898 // This encoding class is generated automatically from ad_encode.m4.
2899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2900 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2901 Register dst_reg = as_Register($dst$$reg);
2902 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2903 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2904 %}
2905
2906 // This encoding class is generated automatically from ad_encode.m4.
2907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2908 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2909 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2910 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2911 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2912 %}
2913
2914 // This encoding class is generated automatically from ad_encode.m4.
2915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2916 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2917 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2918 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2919 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2920 %}
2921
2922 // This encoding class is generated automatically from ad_encode.m4.
2923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2924 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2925 Register src_reg = as_Register($src$$reg);
2926 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2927 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2928 %}
2929
2930 // This encoding class is generated automatically from ad_encode.m4.
2931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2932 enc_class aarch64_enc_strb0(memory1 mem) %{
2933 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2934 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2935 %}
2936
2937 // This encoding class is generated automatically from ad_encode.m4.
2938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2939 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2940 Register src_reg = as_Register($src$$reg);
2941 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2942 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2943 %}
2944
2945 // This encoding class is generated automatically from ad_encode.m4.
2946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2947 enc_class aarch64_enc_strh0(memory2 mem) %{
2948 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2949 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2950 %}
2951
2952 // This encoding class is generated automatically from ad_encode.m4.
2953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2954 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2955 Register src_reg = as_Register($src$$reg);
2956 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
2957 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2958 %}
2959
2960 // This encoding class is generated automatically from ad_encode.m4.
2961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2962 enc_class aarch64_enc_strw0(memory4 mem) %{
2963 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
2964 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2965 %}
2966
2967 // This encoding class is generated automatically from ad_encode.m4.
2968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2969 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2970 Register src_reg = as_Register($src$$reg);
2971 // we sometimes get asked to store the stack pointer into the
2972 // current thread -- we cannot do that directly on AArch64
2973 if (src_reg == r31_sp) {
2974 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2975 __ mov(rscratch2, sp);
2976 src_reg = rscratch2;
2977 }
2978 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
2979 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2980 %}
2981
2982 // This encoding class is generated automatically from ad_encode.m4.
2983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2984 enc_class aarch64_enc_str0(memory8 mem) %{
2985 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
2986 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2987 %}
2988
2989 // This encoding class is generated automatically from ad_encode.m4.
2990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2991 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
2992 FloatRegister src_reg = as_FloatRegister($src$$reg);
2993 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
2994 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2995 %}
2996
2997 // This encoding class is generated automatically from ad_encode.m4.
2998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2999 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3000 FloatRegister src_reg = as_FloatRegister($src$$reg);
3001 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3002 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3003 %}
3004
3005 // This encoding class is generated automatically from ad_encode.m4.
3006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3007 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3008 __ membar(Assembler::StoreStore);
3009 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3010 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3011 %}
3012
3013 // END Non-volatile memory access
3014
3015 // Vector loads and stores
3016 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3017 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3018 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3019 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3020 %}
3021
3022 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3023 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3024 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3025 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3026 %}
3027
3028 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3029 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3030 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3031 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3032 %}
3033
3034 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3035 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3036 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3037 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3038 %}
3039
3040 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3041 FloatRegister src_reg = as_FloatRegister($src$$reg);
3042 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3043 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3044 %}
3045
3046 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3047 FloatRegister src_reg = as_FloatRegister($src$$reg);
3048 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3049 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3050 %}
3051
3052 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3053 FloatRegister src_reg = as_FloatRegister($src$$reg);
3054 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3055 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3056 %}
3057
3058 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3059 FloatRegister src_reg = as_FloatRegister($src$$reg);
3060 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3061 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3062 %}
3063
3064 // volatile loads and stores
3065
3066 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3067 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3068 rscratch1, stlrb);
3069 %}
3070
3071 enc_class aarch64_enc_stlrb0(memory mem) %{
3072 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3073 rscratch1, stlrb);
3074 %}
3075
3076 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3077 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3078 rscratch1, stlrh);
3079 %}
3080
3081 enc_class aarch64_enc_stlrh0(memory mem) %{
3082 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3083 rscratch1, stlrh);
3084 %}
3085
3086 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3087 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3088 rscratch1, stlrw);
3089 %}
3090
3091 enc_class aarch64_enc_stlrw0(memory mem) %{
3092 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3093 rscratch1, stlrw);
3094 %}
3095
3096 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3097 Register dst_reg = as_Register($dst$$reg);
3098 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3099 rscratch1, ldarb);
3100 __ sxtbw(dst_reg, dst_reg);
3101 %}
3102
3103 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3104 Register dst_reg = as_Register($dst$$reg);
3105 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3106 rscratch1, ldarb);
3107 __ sxtb(dst_reg, dst_reg);
3108 %}
3109
3110 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3111 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3112 rscratch1, ldarb);
3113 %}
3114
3115 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3116 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3117 rscratch1, ldarb);
3118 %}
3119
3120 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3121 Register dst_reg = as_Register($dst$$reg);
3122 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3123 rscratch1, ldarh);
3124 __ sxthw(dst_reg, dst_reg);
3125 %}
3126
3127 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3128 Register dst_reg = as_Register($dst$$reg);
3129 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3130 rscratch1, ldarh);
3131 __ sxth(dst_reg, dst_reg);
3132 %}
3133
3134 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3135 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3136 rscratch1, ldarh);
3137 %}
3138
3139 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3140 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3141 rscratch1, ldarh);
3142 %}
3143
3144 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3145 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3146 rscratch1, ldarw);
3147 %}
3148
3149 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3150 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3151 rscratch1, ldarw);
3152 %}
3153
3154 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3155 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3156 rscratch1, ldar);
3157 %}
3158
3159 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3160 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3161 rscratch1, ldarw);
3162 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3163 %}
3164
3165 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3166 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3167 rscratch1, ldar);
3168 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3169 %}
3170
3171 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3172 Register src_reg = as_Register($src$$reg);
3173 // we sometimes get asked to store the stack pointer into the
3174 // current thread -- we cannot do that directly on AArch64
3175 if (src_reg == r31_sp) {
3176 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3177 __ mov(rscratch2, sp);
3178 src_reg = rscratch2;
3179 }
3180 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3181 rscratch1, stlr);
3182 %}
3183
3184 enc_class aarch64_enc_stlr0(memory mem) %{
3185 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3186 rscratch1, stlr);
3187 %}
3188
3189 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3190 {
3191 FloatRegister src_reg = as_FloatRegister($src$$reg);
3192 __ fmovs(rscratch2, src_reg);
3193 }
3194 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3195 rscratch1, stlrw);
3196 %}
3197
3198 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3199 {
3200 FloatRegister src_reg = as_FloatRegister($src$$reg);
3201 __ fmovd(rscratch2, src_reg);
3202 }
3203 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3204 rscratch1, stlr);
3205 %}
3206
3207 // synchronized read/update encodings
3208
3209 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3210 Register dst_reg = as_Register($dst$$reg);
3211 Register base = as_Register($mem$$base);
3212 int index = $mem$$index;
3213 int scale = $mem$$scale;
3214 int disp = $mem$$disp;
3215 if (index == -1) {
3216 if (disp != 0) {
3217 __ lea(rscratch1, Address(base, disp));
3218 __ ldaxr(dst_reg, rscratch1);
3219 } else {
3220 // TODO
3221 // should we ever get anything other than this case?
3222 __ ldaxr(dst_reg, base);
3223 }
3224 } else {
3225 Register index_reg = as_Register(index);
3226 if (disp == 0) {
3227 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3228 __ ldaxr(dst_reg, rscratch1);
3229 } else {
3230 __ lea(rscratch1, Address(base, disp));
3231 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3232 __ ldaxr(dst_reg, rscratch1);
3233 }
3234 }
3235 %}
3236
3237 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3238 Register src_reg = as_Register($src$$reg);
3239 Register base = as_Register($mem$$base);
3240 int index = $mem$$index;
3241 int scale = $mem$$scale;
3242 int disp = $mem$$disp;
3243 if (index == -1) {
3244 if (disp != 0) {
3245 __ lea(rscratch2, Address(base, disp));
3246 __ stlxr(rscratch1, src_reg, rscratch2);
3247 } else {
3248 // TODO
3249 // should we ever get anything other than this case?
3250 __ stlxr(rscratch1, src_reg, base);
3251 }
3252 } else {
3253 Register index_reg = as_Register(index);
3254 if (disp == 0) {
3255 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3256 __ stlxr(rscratch1, src_reg, rscratch2);
3257 } else {
3258 __ lea(rscratch2, Address(base, disp));
3259 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3260 __ stlxr(rscratch1, src_reg, rscratch2);
3261 }
3262 }
3263 __ cmpw(rscratch1, zr);
3264 %}
3265
3266 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3267 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3268 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3269 Assembler::xword, /*acquire*/ false, /*release*/ true,
3270 /*weak*/ false, noreg);
3271 %}
3272
3273 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3274 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3275 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3276 Assembler::word, /*acquire*/ false, /*release*/ true,
3277 /*weak*/ false, noreg);
3278 %}
3279
3280 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3281 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3282 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3283 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3284 /*weak*/ false, noreg);
3285 %}
3286
3287 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3288 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3289 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3290 Assembler::byte, /*acquire*/ false, /*release*/ true,
3291 /*weak*/ false, noreg);
3292 %}
3293
3294
3295 // The only difference between aarch64_enc_cmpxchg and
3296 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3297 // CompareAndSwap sequence to serve as a barrier on acquiring a
3298 // lock.
3299 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3300 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3301 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3302 Assembler::xword, /*acquire*/ true, /*release*/ true,
3303 /*weak*/ false, noreg);
3304 %}
3305
3306 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3307 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3308 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3309 Assembler::word, /*acquire*/ true, /*release*/ true,
3310 /*weak*/ false, noreg);
3311 %}
3312
3313 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3314 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3315 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3316 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3317 /*weak*/ false, noreg);
3318 %}
3319
3320 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3321 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3322 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3323 Assembler::byte, /*acquire*/ true, /*release*/ true,
3324 /*weak*/ false, noreg);
3325 %}
3326
3327 // auxiliary used for CompareAndSwapX to set result register
3328 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3329 Register res_reg = as_Register($res$$reg);
3330 __ cset(res_reg, Assembler::EQ);
3331 %}
3332
3333 // prefetch encodings
3334
3335 enc_class aarch64_enc_prefetchw(memory mem) %{
3336 Register base = as_Register($mem$$base);
3337 int index = $mem$$index;
3338 int scale = $mem$$scale;
3339 int disp = $mem$$disp;
3340 if (index == -1) {
3341 // Fix up any out-of-range offsets.
3342 assert_different_registers(rscratch1, base);
3343 Address addr = Address(base, disp);
3344 addr = __ legitimize_address(addr, 8, rscratch1);
3345 __ prfm(addr, PSTL1KEEP);
3346 } else {
3347 Register index_reg = as_Register(index);
3348 if (disp == 0) {
3349 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3350 } else {
3351 __ lea(rscratch1, Address(base, disp));
3352 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3353 }
3354 }
3355 %}
3356
3357 // mov encodings
3358
3359 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3360 uint32_t con = (uint32_t)$src$$constant;
3361 Register dst_reg = as_Register($dst$$reg);
3362 if (con == 0) {
3363 __ movw(dst_reg, zr);
3364 } else {
3365 __ movw(dst_reg, con);
3366 }
3367 %}
3368
3369 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3370 Register dst_reg = as_Register($dst$$reg);
3371 uint64_t con = (uint64_t)$src$$constant;
3372 if (con == 0) {
3373 __ mov(dst_reg, zr);
3374 } else {
3375 __ mov(dst_reg, con);
3376 }
3377 %}
3378
3379 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3380 Register dst_reg = as_Register($dst$$reg);
3381 address con = (address)$src$$constant;
3382 if (con == nullptr || con == (address)1) {
3383 ShouldNotReachHere();
3384 } else {
3385 relocInfo::relocType rtype = $src->constant_reloc();
3386 if (rtype == relocInfo::oop_type) {
3387 __ movoop(dst_reg, (jobject)con);
3388 } else if (rtype == relocInfo::metadata_type) {
3389 __ mov_metadata(dst_reg, (Metadata*)con);
3390 } else {
3391 assert(rtype == relocInfo::none, "unexpected reloc type");
3392 if (! __ is_valid_AArch64_address(con) ||
3393 con < (address)(uintptr_t)os::vm_page_size()) {
3394 __ mov(dst_reg, con);
3395 } else {
3396 uint64_t offset;
3397 __ adrp(dst_reg, con, offset);
3398 __ add(dst_reg, dst_reg, offset);
3399 }
3400 }
3401 }
3402 %}
3403
3404 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3405 Register dst_reg = as_Register($dst$$reg);
3406 __ mov(dst_reg, zr);
3407 %}
3408
3409 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3410 Register dst_reg = as_Register($dst$$reg);
3411 __ mov(dst_reg, (uint64_t)1);
3412 %}
3413
3414 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3415 __ load_byte_map_base($dst$$Register);
3416 %}
3417
3418 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3419 Register dst_reg = as_Register($dst$$reg);
3420 address con = (address)$src$$constant;
3421 if (con == nullptr) {
3422 ShouldNotReachHere();
3423 } else {
3424 relocInfo::relocType rtype = $src->constant_reloc();
3425 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3426 __ set_narrow_oop(dst_reg, (jobject)con);
3427 }
3428 %}
3429
3430 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3431 Register dst_reg = as_Register($dst$$reg);
3432 __ mov(dst_reg, zr);
3433 %}
3434
3435 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3436 Register dst_reg = as_Register($dst$$reg);
3437 address con = (address)$src$$constant;
3438 if (con == nullptr) {
3439 ShouldNotReachHere();
3440 } else {
3441 relocInfo::relocType rtype = $src->constant_reloc();
3442 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3443 __ set_narrow_klass(dst_reg, (Klass *)con);
3444 }
3445 %}
3446
3447 // arithmetic encodings
3448
3449 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3450 Register dst_reg = as_Register($dst$$reg);
3451 Register src_reg = as_Register($src1$$reg);
3452 int32_t con = (int32_t)$src2$$constant;
3453 // add has primary == 0, subtract has primary == 1
3454 if ($primary) { con = -con; }
3455 if (con < 0) {
3456 __ subw(dst_reg, src_reg, -con);
3457 } else {
3458 __ addw(dst_reg, src_reg, con);
3459 }
3460 %}
3461
3462 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3463 Register dst_reg = as_Register($dst$$reg);
3464 Register src_reg = as_Register($src1$$reg);
3465 int32_t con = (int32_t)$src2$$constant;
3466 // add has primary == 0, subtract has primary == 1
3467 if ($primary) { con = -con; }
3468 if (con < 0) {
3469 __ sub(dst_reg, src_reg, -con);
3470 } else {
3471 __ add(dst_reg, src_reg, con);
3472 }
3473 %}
3474
3475 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3476 Register dst_reg = as_Register($dst$$reg);
3477 Register src1_reg = as_Register($src1$$reg);
3478 Register src2_reg = as_Register($src2$$reg);
3479 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3480 %}
3481
3482 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3483 Register dst_reg = as_Register($dst$$reg);
3484 Register src1_reg = as_Register($src1$$reg);
3485 Register src2_reg = as_Register($src2$$reg);
3486 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3487 %}
3488
3489 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3490 Register dst_reg = as_Register($dst$$reg);
3491 Register src1_reg = as_Register($src1$$reg);
3492 Register src2_reg = as_Register($src2$$reg);
3493 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3494 %}
3495
3496 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3497 Register dst_reg = as_Register($dst$$reg);
3498 Register src1_reg = as_Register($src1$$reg);
3499 Register src2_reg = as_Register($src2$$reg);
3500 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3501 %}
3502
3503 // compare instruction encodings
3504
3505 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3506 Register reg1 = as_Register($src1$$reg);
3507 Register reg2 = as_Register($src2$$reg);
3508 __ cmpw(reg1, reg2);
3509 %}
3510
3511 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3512 Register reg = as_Register($src1$$reg);
3513 int32_t val = $src2$$constant;
3514 if (val >= 0) {
3515 __ subsw(zr, reg, val);
3516 } else {
3517 __ addsw(zr, reg, -val);
3518 }
3519 %}
3520
3521 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3522 Register reg1 = as_Register($src1$$reg);
3523 uint32_t val = (uint32_t)$src2$$constant;
3524 __ movw(rscratch1, val);
3525 __ cmpw(reg1, rscratch1);
3526 %}
3527
3528 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3529 Register reg1 = as_Register($src1$$reg);
3530 Register reg2 = as_Register($src2$$reg);
3531 __ cmp(reg1, reg2);
3532 %}
3533
3534 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3535 Register reg = as_Register($src1$$reg);
3536 int64_t val = $src2$$constant;
3537 if (val >= 0) {
3538 __ subs(zr, reg, val);
3539 } else if (val != -val) {
3540 __ adds(zr, reg, -val);
3541 } else {
3542 // aargh, Long.MIN_VALUE is a special case
3543 __ orr(rscratch1, zr, (uint64_t)val);
3544 __ subs(zr, reg, rscratch1);
3545 }
3546 %}
3547
3548 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3549 Register reg1 = as_Register($src1$$reg);
3550 uint64_t val = (uint64_t)$src2$$constant;
3551 __ mov(rscratch1, val);
3552 __ cmp(reg1, rscratch1);
3553 %}
3554
3555 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3556 Register reg1 = as_Register($src1$$reg);
3557 Register reg2 = as_Register($src2$$reg);
3558 __ cmp(reg1, reg2);
3559 %}
3560
3561 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3562 Register reg1 = as_Register($src1$$reg);
3563 Register reg2 = as_Register($src2$$reg);
3564 __ cmpw(reg1, reg2);
3565 %}
3566
3567 enc_class aarch64_enc_testp(iRegP src) %{
3568 Register reg = as_Register($src$$reg);
3569 __ cmp(reg, zr);
3570 %}
3571
3572 enc_class aarch64_enc_testn(iRegN src) %{
3573 Register reg = as_Register($src$$reg);
3574 __ cmpw(reg, zr);
3575 %}
3576
3577 enc_class aarch64_enc_b(label lbl) %{
3578 Label *L = $lbl$$label;
3579 __ b(*L);
3580 %}
3581
3582 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3583 Label *L = $lbl$$label;
3584 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3585 %}
3586
3587 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3588 Label *L = $lbl$$label;
3589 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3590 %}
3591
3592 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3593 %{
3594 Register sub_reg = as_Register($sub$$reg);
3595 Register super_reg = as_Register($super$$reg);
3596 Register temp_reg = as_Register($temp$$reg);
3597 Register result_reg = as_Register($result$$reg);
3598
3599 Label miss;
3600 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3601 nullptr, &miss,
3602 /*set_cond_codes:*/ true);
3603 if ($primary) {
3604 __ mov(result_reg, zr);
3605 }
3606 __ bind(miss);
3607 %}
3608
3609 enc_class aarch64_enc_java_static_call(method meth) %{
3610 address addr = (address)$meth$$method;
3611 address call;
3612 if (!_method) {
3613 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3614 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3615 if (call == nullptr) {
3616 ciEnv::current()->record_failure("CodeCache is full");
3617 return;
3618 }
3619 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3620 // The NOP here is purely to ensure that eliding a call to
3621 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3622 __ nop();
3623 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3624 } else {
3625 int method_index = resolved_method_index(masm);
3626 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3627 : static_call_Relocation::spec(method_index);
3628 call = __ trampoline_call(Address(addr, rspec));
3629 if (call == nullptr) {
3630 ciEnv::current()->record_failure("CodeCache is full");
3631 return;
3632 }
3633 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3634 // Calls of the same statically bound method can share
3635 // a stub to the interpreter.
3636 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3637 } else {
3638 // Emit stub for static call
3639 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3640 if (stub == nullptr) {
3641 ciEnv::current()->record_failure("CodeCache is full");
3642 return;
3643 }
3644 }
3645 }
3646
3647 __ post_call_nop();
3648
3649 // Only non uncommon_trap calls need to reinitialize ptrue.
3650 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3651 __ reinitialize_ptrue();
3652 }
3653 %}
3654
3655 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3656 int method_index = resolved_method_index(masm);
3657 address call = __ ic_call((address)$meth$$method, method_index);
3658 if (call == nullptr) {
3659 ciEnv::current()->record_failure("CodeCache is full");
3660 return;
3661 }
3662 __ post_call_nop();
3663 if (Compile::current()->max_vector_size() > 0) {
3664 __ reinitialize_ptrue();
3665 }
3666 %}
3667
3668 enc_class aarch64_enc_call_epilog() %{
3669 if (VerifyStackAtCalls) {
3670 // Check that stack depth is unchanged: find majik cookie on stack
3671 __ call_Unimplemented();
3672 }
3673 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
3674 // The last return value is not set by the callee but used to pass IsInit information to compiled code.
3675 // Search for the corresponding projection, get the register and emit code that initialized it.
3676 uint con = (tf()->range_cc()->cnt() - 1);
3677 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3678 ProjNode* proj = fast_out(i)->as_Proj();
3679 if (proj->_con == con) {
3680 // Set IsInit if r0 is non-null (a non-null value is returned buffered or scalarized)
3681 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3682 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3683 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3684 __ cmp(r0, zr);
3685 __ cset(toReg, Assembler::NE);
3686 if (reg->is_stack()) {
3687 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3688 __ str(toReg, Address(sp, st_off));
3689 }
3690 break;
3691 }
3692 }
3693 if (return_value_is_used()) {
3694 // An inline type is returned as fields in multiple registers.
3695 // R0 either contains an oop if the inline type is buffered or a pointer
3696 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3697 // if the lowest bit is set to allow C2 to use the oop after null checking.
3698 // r0 &= (r0 & 1) - 1
3699 __ andr(rscratch1, r0, 0x1);
3700 __ sub(rscratch1, rscratch1, 0x1);
3701 __ andr(r0, r0, rscratch1);
3702 }
3703 }
3704 %}
3705
3706 enc_class aarch64_enc_java_to_runtime(method meth) %{
3707 // some calls to generated routines (arraycopy code) are scheduled
3708 // by C2 as runtime calls. if so we can call them using a br (they
3709 // will be in a reachable segment) otherwise we have to use a blr
3710 // which loads the absolute address into a register.
3711 address entry = (address)$meth$$method;
3712 CodeBlob *cb = CodeCache::find_blob(entry);
3713 if (cb) {
3714 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3715 if (call == nullptr) {
3716 ciEnv::current()->record_failure("CodeCache is full");
3717 return;
3718 }
3719 __ post_call_nop();
3720 } else {
3721 Label retaddr;
3722 __ adr(rscratch2, retaddr);
3723 __ lea(rscratch1, RuntimeAddress(entry));
3724 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3725 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3726 __ blr(rscratch1);
3727 __ bind(retaddr);
3728 __ post_call_nop();
3729 __ add(sp, sp, 2 * wordSize);
3730 }
3731 if (Compile::current()->max_vector_size() > 0) {
3732 __ reinitialize_ptrue();
3733 }
3734 %}
3735
3736 enc_class aarch64_enc_rethrow() %{
3737 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3738 %}
3739
3740 enc_class aarch64_enc_ret() %{
3741 #ifdef ASSERT
3742 if (Compile::current()->max_vector_size() > 0) {
3743 __ verify_ptrue();
3744 }
3745 #endif
3746 __ ret(lr);
3747 %}
3748
3749 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3750 Register target_reg = as_Register($jump_target$$reg);
3751 __ br(target_reg);
3752 %}
3753
3754 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3755 Register target_reg = as_Register($jump_target$$reg);
3756 // exception oop should be in r0
3757 // ret addr has been popped into lr
3758 // callee expects it in r3
3759 __ mov(r3, lr);
3760 __ br(target_reg);
3761 %}
3762
3763 %}
3764
3765 //----------FRAME--------------------------------------------------------------
3766 // Definition of frame structure and management information.
3767 //
3768 // S T A C K L A Y O U T Allocators stack-slot number
3769 // | (to get allocators register number
3770 // G Owned by | | v add OptoReg::stack0())
3771 // r CALLER | |
3772 // o | +--------+ pad to even-align allocators stack-slot
3773 // w V | pad0 | numbers; owned by CALLER
3774 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3775 // h ^ | in | 5
3776 // | | args | 4 Holes in incoming args owned by SELF
3777 // | | | | 3
3778 // | | +--------+
3779 // V | | old out| Empty on Intel, window on Sparc
3780 // | old |preserve| Must be even aligned.
3781 // | SP-+--------+----> Matcher::_old_SP, even aligned
3782 // | | in | 3 area for Intel ret address
3783 // Owned by |preserve| Empty on Sparc.
3784 // SELF +--------+
3785 // | | pad2 | 2 pad to align old SP
3786 // | +--------+ 1
3787 // | | locks | 0
3788 // | +--------+----> OptoReg::stack0(), even aligned
3789 // | | pad1 | 11 pad to align new SP
3790 // | +--------+
3791 // | | | 10
3792 // | | spills | 9 spills
3793 // V | | 8 (pad0 slot for callee)
3794 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3795 // ^ | out | 7
3796 // | | args | 6 Holes in outgoing args owned by CALLEE
3797 // Owned by +--------+
3798 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3799 // | new |preserve| Must be even-aligned.
3800 // | SP-+--------+----> Matcher::_new_SP, even aligned
3801 // | | |
3802 //
3803 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3804 // known from SELF's arguments and the Java calling convention.
3805 // Region 6-7 is determined per call site.
3806 // Note 2: If the calling convention leaves holes in the incoming argument
3807 // area, those holes are owned by SELF. Holes in the outgoing area
3808 // are owned by the CALLEE. Holes should not be necessary in the
3809 // incoming area, as the Java calling convention is completely under
3810 // the control of the AD file. Doubles can be sorted and packed to
3811 // avoid holes. Holes in the outgoing arguments may be necessary for
3812 // varargs C calling conventions.
3813 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3814 // even aligned with pad0 as needed.
3815 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3816 // (the latter is true on Intel but is it false on AArch64?)
3817 // region 6-11 is even aligned; it may be padded out more so that
3818 // the region from SP to FP meets the minimum stack alignment.
3819 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3820 // alignment. Region 11, pad1, may be dynamically extended so that
3821 // SP meets the minimum alignment.
3822
3823 frame %{
3824 // These three registers define part of the calling convention
3825 // between compiled code and the interpreter.
3826
3827 // Inline Cache Register or Method for I2C.
3828 inline_cache_reg(R12);
3829
3830 // Number of stack slots consumed by locking an object
3831 sync_stack_slots(2);
3832
3833 // Compiled code's Frame Pointer
3834 frame_pointer(R31);
3835
3836 // Interpreter stores its frame pointer in a register which is
3837 // stored to the stack by I2CAdaptors.
3838 // I2CAdaptors convert from interpreted java to compiled java.
3839 interpreter_frame_pointer(R29);
3840
3841 // Stack alignment requirement
3842 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3843
3844 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3845 // for calls to C. Supports the var-args backing area for register parms.
3846 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3847
3848 // The after-PROLOG location of the return address. Location of
3849 // return address specifies a type (REG or STACK) and a number
3850 // representing the register number (i.e. - use a register name) or
3851 // stack slot.
3852 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3853 // Otherwise, it is above the locks and verification slot and alignment word
3854 // TODO this may well be correct but need to check why that - 2 is there
3855 // ppc port uses 0 but we definitely need to allow for fixed_slots
3856 // which folds in the space used for monitors
3857 return_addr(STACK - 2 +
3858 align_up((Compile::current()->in_preserve_stack_slots() +
3859 Compile::current()->fixed_slots()),
3860 stack_alignment_in_slots()));
3861
3862 // Location of compiled Java return values. Same as C for now.
3863 return_value
3864 %{
3865 // TODO do we allow ideal_reg == Op_RegN???
3866 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3867 "only return normal values");
3868
3869 static const int lo[Op_RegL + 1] = { // enum name
3870 0, // Op_Node
3871 0, // Op_Set
3872 R0_num, // Op_RegN
3873 R0_num, // Op_RegI
3874 R0_num, // Op_RegP
3875 V0_num, // Op_RegF
3876 V0_num, // Op_RegD
3877 R0_num // Op_RegL
3878 };
3879
3880 static const int hi[Op_RegL + 1] = { // enum name
3881 0, // Op_Node
3882 0, // Op_Set
3883 OptoReg::Bad, // Op_RegN
3884 OptoReg::Bad, // Op_RegI
3885 R0_H_num, // Op_RegP
3886 OptoReg::Bad, // Op_RegF
3887 V0_H_num, // Op_RegD
3888 R0_H_num // Op_RegL
3889 };
3890
3891 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3892 %}
3893 %}
3894
3895 //----------ATTRIBUTES---------------------------------------------------------
3896 //----------Operand Attributes-------------------------------------------------
3897 op_attrib op_cost(1); // Required cost attribute
3898
3899 //----------Instruction Attributes---------------------------------------------
3900 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3901 ins_attrib ins_size(32); // Required size attribute (in bits)
3902 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3903 // a non-matching short branch variant
3904 // of some long branch?
3905 ins_attrib ins_alignment(4); // Required alignment attribute (must
3906 // be a power of 2) specifies the
3907 // alignment that some part of the
3908 // instruction (not necessarily the
3909 // start) requires. If > 1, a
3910 // compute_padding() function must be
3911 // provided for the instruction
3912
3913 //----------OPERANDS-----------------------------------------------------------
3914 // Operand definitions must precede instruction definitions for correct parsing
3915 // in the ADLC because operands constitute user defined types which are used in
3916 // instruction definitions.
3917
3918 //----------Simple Operands----------------------------------------------------
3919
3920 // Integer operands 32 bit
3921 // 32 bit immediate
3922 operand immI()
3923 %{
3924 match(ConI);
3925
3926 op_cost(0);
3927 format %{ %}
3928 interface(CONST_INTER);
3929 %}
3930
3931 // 32 bit zero
3932 operand immI0()
3933 %{
3934 predicate(n->get_int() == 0);
3935 match(ConI);
3936
3937 op_cost(0);
3938 format %{ %}
3939 interface(CONST_INTER);
3940 %}
3941
3942 // 32 bit unit increment
3943 operand immI_1()
3944 %{
3945 predicate(n->get_int() == 1);
3946 match(ConI);
3947
3948 op_cost(0);
3949 format %{ %}
3950 interface(CONST_INTER);
3951 %}
3952
3953 // 32 bit unit decrement
3954 operand immI_M1()
3955 %{
3956 predicate(n->get_int() == -1);
3957 match(ConI);
3958
3959 op_cost(0);
3960 format %{ %}
3961 interface(CONST_INTER);
3962 %}
3963
3964 // Shift values for add/sub extension shift
3965 operand immIExt()
3966 %{
3967 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3968 match(ConI);
3969
3970 op_cost(0);
3971 format %{ %}
3972 interface(CONST_INTER);
3973 %}
3974
3975 operand immI_gt_1()
3976 %{
3977 predicate(n->get_int() > 1);
3978 match(ConI);
3979
3980 op_cost(0);
3981 format %{ %}
3982 interface(CONST_INTER);
3983 %}
3984
3985 operand immI_le_4()
3986 %{
3987 predicate(n->get_int() <= 4);
3988 match(ConI);
3989
3990 op_cost(0);
3991 format %{ %}
3992 interface(CONST_INTER);
3993 %}
3994
3995 operand immI_16()
3996 %{
3997 predicate(n->get_int() == 16);
3998 match(ConI);
3999
4000 op_cost(0);
4001 format %{ %}
4002 interface(CONST_INTER);
4003 %}
4004
4005 operand immI_24()
4006 %{
4007 predicate(n->get_int() == 24);
4008 match(ConI);
4009
4010 op_cost(0);
4011 format %{ %}
4012 interface(CONST_INTER);
4013 %}
4014
4015 operand immI_32()
4016 %{
4017 predicate(n->get_int() == 32);
4018 match(ConI);
4019
4020 op_cost(0);
4021 format %{ %}
4022 interface(CONST_INTER);
4023 %}
4024
4025 operand immI_48()
4026 %{
4027 predicate(n->get_int() == 48);
4028 match(ConI);
4029
4030 op_cost(0);
4031 format %{ %}
4032 interface(CONST_INTER);
4033 %}
4034
4035 operand immI_56()
4036 %{
4037 predicate(n->get_int() == 56);
4038 match(ConI);
4039
4040 op_cost(0);
4041 format %{ %}
4042 interface(CONST_INTER);
4043 %}
4044
4045 operand immI_255()
4046 %{
4047 predicate(n->get_int() == 255);
4048 match(ConI);
4049
4050 op_cost(0);
4051 format %{ %}
4052 interface(CONST_INTER);
4053 %}
4054
4055 operand immI_65535()
4056 %{
4057 predicate(n->get_int() == 65535);
4058 match(ConI);
4059
4060 op_cost(0);
4061 format %{ %}
4062 interface(CONST_INTER);
4063 %}
4064
4065 operand immI_positive()
4066 %{
4067 predicate(n->get_int() > 0);
4068 match(ConI);
4069
4070 op_cost(0);
4071 format %{ %}
4072 interface(CONST_INTER);
4073 %}
4074
4075 // BoolTest condition for signed compare
4076 operand immI_cmp_cond()
4077 %{
4078 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4079 match(ConI);
4080
4081 op_cost(0);
4082 format %{ %}
4083 interface(CONST_INTER);
4084 %}
4085
4086 // BoolTest condition for unsigned compare
4087 operand immI_cmpU_cond()
4088 %{
4089 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4090 match(ConI);
4091
4092 op_cost(0);
4093 format %{ %}
4094 interface(CONST_INTER);
4095 %}
4096
4097 operand immL_255()
4098 %{
4099 predicate(n->get_long() == 255L);
4100 match(ConL);
4101
4102 op_cost(0);
4103 format %{ %}
4104 interface(CONST_INTER);
4105 %}
4106
4107 operand immL_65535()
4108 %{
4109 predicate(n->get_long() == 65535L);
4110 match(ConL);
4111
4112 op_cost(0);
4113 format %{ %}
4114 interface(CONST_INTER);
4115 %}
4116
4117 operand immL_4294967295()
4118 %{
4119 predicate(n->get_long() == 4294967295L);
4120 match(ConL);
4121
4122 op_cost(0);
4123 format %{ %}
4124 interface(CONST_INTER);
4125 %}
4126
4127 operand immL_bitmask()
4128 %{
4129 predicate((n->get_long() != 0)
4130 && ((n->get_long() & 0xc000000000000000l) == 0)
4131 && is_power_of_2(n->get_long() + 1));
4132 match(ConL);
4133
4134 op_cost(0);
4135 format %{ %}
4136 interface(CONST_INTER);
4137 %}
4138
4139 operand immI_bitmask()
4140 %{
4141 predicate((n->get_int() != 0)
4142 && ((n->get_int() & 0xc0000000) == 0)
4143 && is_power_of_2(n->get_int() + 1));
4144 match(ConI);
4145
4146 op_cost(0);
4147 format %{ %}
4148 interface(CONST_INTER);
4149 %}
4150
4151 operand immL_positive_bitmaskI()
4152 %{
4153 predicate((n->get_long() != 0)
4154 && ((julong)n->get_long() < 0x80000000ULL)
4155 && is_power_of_2(n->get_long() + 1));
4156 match(ConL);
4157
4158 op_cost(0);
4159 format %{ %}
4160 interface(CONST_INTER);
4161 %}
4162
4163 // Scale values for scaled offset addressing modes (up to long but not quad)
4164 operand immIScale()
4165 %{
4166 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4167 match(ConI);
4168
4169 op_cost(0);
4170 format %{ %}
4171 interface(CONST_INTER);
4172 %}
4173
4174 // 5 bit signed integer
4175 operand immI5()
4176 %{
4177 predicate(Assembler::is_simm(n->get_int(), 5));
4178 match(ConI);
4179
4180 op_cost(0);
4181 format %{ %}
4182 interface(CONST_INTER);
4183 %}
4184
4185 // 7 bit unsigned integer
4186 operand immIU7()
4187 %{
4188 predicate(Assembler::is_uimm(n->get_int(), 7));
4189 match(ConI);
4190
4191 op_cost(0);
4192 format %{ %}
4193 interface(CONST_INTER);
4194 %}
4195
4196 // Offset for scaled or unscaled immediate loads and stores
4197 operand immIOffset()
4198 %{
4199 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4200 match(ConI);
4201
4202 op_cost(0);
4203 format %{ %}
4204 interface(CONST_INTER);
4205 %}
4206
4207 operand immIOffset1()
4208 %{
4209 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4210 match(ConI);
4211
4212 op_cost(0);
4213 format %{ %}
4214 interface(CONST_INTER);
4215 %}
4216
4217 operand immIOffset2()
4218 %{
4219 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4220 match(ConI);
4221
4222 op_cost(0);
4223 format %{ %}
4224 interface(CONST_INTER);
4225 %}
4226
4227 operand immIOffset4()
4228 %{
4229 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4230 match(ConI);
4231
4232 op_cost(0);
4233 format %{ %}
4234 interface(CONST_INTER);
4235 %}
4236
4237 operand immIOffset8()
4238 %{
4239 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4240 match(ConI);
4241
4242 op_cost(0);
4243 format %{ %}
4244 interface(CONST_INTER);
4245 %}
4246
4247 operand immIOffset16()
4248 %{
4249 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4250 match(ConI);
4251
4252 op_cost(0);
4253 format %{ %}
4254 interface(CONST_INTER);
4255 %}
4256
4257 operand immLOffset()
4258 %{
4259 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4260 match(ConL);
4261
4262 op_cost(0);
4263 format %{ %}
4264 interface(CONST_INTER);
4265 %}
4266
4267 operand immLoffset1()
4268 %{
4269 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4270 match(ConL);
4271
4272 op_cost(0);
4273 format %{ %}
4274 interface(CONST_INTER);
4275 %}
4276
4277 operand immLoffset2()
4278 %{
4279 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4280 match(ConL);
4281
4282 op_cost(0);
4283 format %{ %}
4284 interface(CONST_INTER);
4285 %}
4286
4287 operand immLoffset4()
4288 %{
4289 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4290 match(ConL);
4291
4292 op_cost(0);
4293 format %{ %}
4294 interface(CONST_INTER);
4295 %}
4296
4297 operand immLoffset8()
4298 %{
4299 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4300 match(ConL);
4301
4302 op_cost(0);
4303 format %{ %}
4304 interface(CONST_INTER);
4305 %}
4306
4307 operand immLoffset16()
4308 %{
4309 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4310 match(ConL);
4311
4312 op_cost(0);
4313 format %{ %}
4314 interface(CONST_INTER);
4315 %}
4316
4317 // 5 bit signed long integer
4318 operand immL5()
4319 %{
4320 predicate(Assembler::is_simm(n->get_long(), 5));
4321 match(ConL);
4322
4323 op_cost(0);
4324 format %{ %}
4325 interface(CONST_INTER);
4326 %}
4327
4328 // 7 bit unsigned long integer
4329 operand immLU7()
4330 %{
4331 predicate(Assembler::is_uimm(n->get_long(), 7));
4332 match(ConL);
4333
4334 op_cost(0);
4335 format %{ %}
4336 interface(CONST_INTER);
4337 %}
4338
4339 // 8 bit signed value.
4340 operand immI8()
4341 %{
4342 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4343 match(ConI);
4344
4345 op_cost(0);
4346 format %{ %}
4347 interface(CONST_INTER);
4348 %}
4349
4350 // 8 bit signed value (simm8), or #simm8 LSL 8.
4351 operand immI8_shift8()
4352 %{
4353 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4354 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4355 match(ConI);
4356
4357 op_cost(0);
4358 format %{ %}
4359 interface(CONST_INTER);
4360 %}
4361
4362 // 8 bit signed value (simm8), or #simm8 LSL 8.
4363 operand immL8_shift8()
4364 %{
4365 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4366 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4367 match(ConL);
4368
4369 op_cost(0);
4370 format %{ %}
4371 interface(CONST_INTER);
4372 %}
4373
4374 // 8 bit integer valid for vector add sub immediate
4375 operand immBAddSubV()
4376 %{
4377 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4378 match(ConI);
4379
4380 op_cost(0);
4381 format %{ %}
4382 interface(CONST_INTER);
4383 %}
4384
4385 // 32 bit integer valid for add sub immediate
4386 operand immIAddSub()
4387 %{
4388 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4389 match(ConI);
4390 op_cost(0);
4391 format %{ %}
4392 interface(CONST_INTER);
4393 %}
4394
4395 // 32 bit integer valid for vector add sub immediate
4396 operand immIAddSubV()
4397 %{
4398 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4399 match(ConI);
4400
4401 op_cost(0);
4402 format %{ %}
4403 interface(CONST_INTER);
4404 %}
4405
4406 // 32 bit unsigned integer valid for logical immediate
4407
4408 operand immBLog()
4409 %{
4410 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4411 match(ConI);
4412
4413 op_cost(0);
4414 format %{ %}
4415 interface(CONST_INTER);
4416 %}
4417
4418 operand immSLog()
4419 %{
4420 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4421 match(ConI);
4422
4423 op_cost(0);
4424 format %{ %}
4425 interface(CONST_INTER);
4426 %}
4427
4428 operand immILog()
4429 %{
4430 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4431 match(ConI);
4432
4433 op_cost(0);
4434 format %{ %}
4435 interface(CONST_INTER);
4436 %}
4437
4438 // Integer operands 64 bit
4439 // 64 bit immediate
4440 operand immL()
4441 %{
4442 match(ConL);
4443
4444 op_cost(0);
4445 format %{ %}
4446 interface(CONST_INTER);
4447 %}
4448
4449 // 64 bit zero
4450 operand immL0()
4451 %{
4452 predicate(n->get_long() == 0);
4453 match(ConL);
4454
4455 op_cost(0);
4456 format %{ %}
4457 interface(CONST_INTER);
4458 %}
4459
4460 // 64 bit unit decrement
4461 operand immL_M1()
4462 %{
4463 predicate(n->get_long() == -1);
4464 match(ConL);
4465
4466 op_cost(0);
4467 format %{ %}
4468 interface(CONST_INTER);
4469 %}
4470
4471 // 64 bit integer valid for add sub immediate
4472 operand immLAddSub()
4473 %{
4474 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4475 match(ConL);
4476 op_cost(0);
4477 format %{ %}
4478 interface(CONST_INTER);
4479 %}
4480
4481 // 64 bit integer valid for addv subv immediate
4482 operand immLAddSubV()
4483 %{
4484 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4485 match(ConL);
4486
4487 op_cost(0);
4488 format %{ %}
4489 interface(CONST_INTER);
4490 %}
4491
4492 // 64 bit integer valid for logical immediate
4493 operand immLLog()
4494 %{
4495 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4496 match(ConL);
4497 op_cost(0);
4498 format %{ %}
4499 interface(CONST_INTER);
4500 %}
4501
4502 // Long Immediate: low 32-bit mask
4503 operand immL_32bits()
4504 %{
4505 predicate(n->get_long() == 0xFFFFFFFFL);
4506 match(ConL);
4507 op_cost(0);
4508 format %{ %}
4509 interface(CONST_INTER);
4510 %}
4511
4512 // Pointer operands
4513 // Pointer Immediate
4514 operand immP()
4515 %{
4516 match(ConP);
4517
4518 op_cost(0);
4519 format %{ %}
4520 interface(CONST_INTER);
4521 %}
4522
4523 // nullptr Pointer Immediate
4524 operand immP0()
4525 %{
4526 predicate(n->get_ptr() == 0);
4527 match(ConP);
4528
4529 op_cost(0);
4530 format %{ %}
4531 interface(CONST_INTER);
4532 %}
4533
4534 // Pointer Immediate One
4535 // this is used in object initialization (initial object header)
4536 operand immP_1()
4537 %{
4538 predicate(n->get_ptr() == 1);
4539 match(ConP);
4540
4541 op_cost(0);
4542 format %{ %}
4543 interface(CONST_INTER);
4544 %}
4545
4546 // Card Table Byte Map Base
4547 operand immByteMapBase()
4548 %{
4549 // Get base of card map
4550 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4551 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4552 match(ConP);
4553
4554 op_cost(0);
4555 format %{ %}
4556 interface(CONST_INTER);
4557 %}
4558
4559 // Float and Double operands
4560 // Double Immediate
4561 operand immD()
4562 %{
4563 match(ConD);
4564 op_cost(0);
4565 format %{ %}
4566 interface(CONST_INTER);
4567 %}
4568
4569 // Double Immediate: +0.0d
4570 operand immD0()
4571 %{
4572 predicate(jlong_cast(n->getd()) == 0);
4573 match(ConD);
4574
4575 op_cost(0);
4576 format %{ %}
4577 interface(CONST_INTER);
4578 %}
4579
4580 // constant 'double +0.0'.
4581 operand immDPacked()
4582 %{
4583 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4584 match(ConD);
4585 op_cost(0);
4586 format %{ %}
4587 interface(CONST_INTER);
4588 %}
4589
4590 // Float Immediate
4591 operand immF()
4592 %{
4593 match(ConF);
4594 op_cost(0);
4595 format %{ %}
4596 interface(CONST_INTER);
4597 %}
4598
4599 // Float Immediate: +0.0f.
4600 operand immF0()
4601 %{
4602 predicate(jint_cast(n->getf()) == 0);
4603 match(ConF);
4604
4605 op_cost(0);
4606 format %{ %}
4607 interface(CONST_INTER);
4608 %}
4609
4610 //
4611 operand immFPacked()
4612 %{
4613 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4614 match(ConF);
4615 op_cost(0);
4616 format %{ %}
4617 interface(CONST_INTER);
4618 %}
4619
4620 // Narrow pointer operands
4621 // Narrow Pointer Immediate
4622 operand immN()
4623 %{
4624 match(ConN);
4625
4626 op_cost(0);
4627 format %{ %}
4628 interface(CONST_INTER);
4629 %}
4630
4631 // Narrow nullptr Pointer Immediate
4632 operand immN0()
4633 %{
4634 predicate(n->get_narrowcon() == 0);
4635 match(ConN);
4636
4637 op_cost(0);
4638 format %{ %}
4639 interface(CONST_INTER);
4640 %}
4641
4642 operand immNKlass()
4643 %{
4644 match(ConNKlass);
4645
4646 op_cost(0);
4647 format %{ %}
4648 interface(CONST_INTER);
4649 %}
4650
4651 // Integer 32 bit Register Operands
4652 // Integer 32 bitRegister (excludes SP)
4653 operand iRegI()
4654 %{
4655 constraint(ALLOC_IN_RC(any_reg32));
4656 match(RegI);
4657 match(iRegINoSp);
4658 op_cost(0);
4659 format %{ %}
4660 interface(REG_INTER);
4661 %}
4662
4663 // Integer 32 bit Register not Special
4664 operand iRegINoSp()
4665 %{
4666 constraint(ALLOC_IN_RC(no_special_reg32));
4667 match(RegI);
4668 op_cost(0);
4669 format %{ %}
4670 interface(REG_INTER);
4671 %}
4672
4673 // Integer 64 bit Register Operands
4674 // Integer 64 bit Register (includes SP)
4675 operand iRegL()
4676 %{
4677 constraint(ALLOC_IN_RC(any_reg));
4678 match(RegL);
4679 match(iRegLNoSp);
4680 op_cost(0);
4681 format %{ %}
4682 interface(REG_INTER);
4683 %}
4684
4685 // Integer 64 bit Register not Special
4686 operand iRegLNoSp()
4687 %{
4688 constraint(ALLOC_IN_RC(no_special_reg));
4689 match(RegL);
4690 match(iRegL_R0);
4691 format %{ %}
4692 interface(REG_INTER);
4693 %}
4694
4695 // Pointer Register Operands
4696 // Pointer Register
4697 operand iRegP()
4698 %{
4699 constraint(ALLOC_IN_RC(ptr_reg));
4700 match(RegP);
4701 match(iRegPNoSp);
4702 match(iRegP_R0);
4703 //match(iRegP_R2);
4704 //match(iRegP_R4);
4705 match(iRegP_R5);
4706 match(thread_RegP);
4707 op_cost(0);
4708 format %{ %}
4709 interface(REG_INTER);
4710 %}
4711
4712 // Pointer 64 bit Register not Special
4713 operand iRegPNoSp()
4714 %{
4715 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4716 match(RegP);
4717 // match(iRegP);
4718 // match(iRegP_R0);
4719 // match(iRegP_R2);
4720 // match(iRegP_R4);
4721 // match(iRegP_R5);
4722 // match(thread_RegP);
4723 op_cost(0);
4724 format %{ %}
4725 interface(REG_INTER);
4726 %}
4727
4728 // This operand is not allowed to use rfp even if
4729 // rfp is not used to hold the frame pointer.
4730 operand iRegPNoSpNoRfp()
4731 %{
4732 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4733 match(RegP);
4734 match(iRegPNoSp);
4735 op_cost(0);
4736 format %{ %}
4737 interface(REG_INTER);
4738 %}
4739
4740 // Pointer 64 bit Register R0 only
4741 operand iRegP_R0()
4742 %{
4743 constraint(ALLOC_IN_RC(r0_reg));
4744 match(RegP);
4745 // match(iRegP);
4746 match(iRegPNoSp);
4747 op_cost(0);
4748 format %{ %}
4749 interface(REG_INTER);
4750 %}
4751
4752 // Pointer 64 bit Register R1 only
4753 operand iRegP_R1()
4754 %{
4755 constraint(ALLOC_IN_RC(r1_reg));
4756 match(RegP);
4757 // match(iRegP);
4758 match(iRegPNoSp);
4759 op_cost(0);
4760 format %{ %}
4761 interface(REG_INTER);
4762 %}
4763
4764 // Pointer 64 bit Register R2 only
4765 operand iRegP_R2()
4766 %{
4767 constraint(ALLOC_IN_RC(r2_reg));
4768 match(RegP);
4769 // match(iRegP);
4770 match(iRegPNoSp);
4771 op_cost(0);
4772 format %{ %}
4773 interface(REG_INTER);
4774 %}
4775
4776 // Pointer 64 bit Register R3 only
4777 operand iRegP_R3()
4778 %{
4779 constraint(ALLOC_IN_RC(r3_reg));
4780 match(RegP);
4781 // match(iRegP);
4782 match(iRegPNoSp);
4783 op_cost(0);
4784 format %{ %}
4785 interface(REG_INTER);
4786 %}
4787
4788 // Pointer 64 bit Register R4 only
4789 operand iRegP_R4()
4790 %{
4791 constraint(ALLOC_IN_RC(r4_reg));
4792 match(RegP);
4793 // match(iRegP);
4794 match(iRegPNoSp);
4795 op_cost(0);
4796 format %{ %}
4797 interface(REG_INTER);
4798 %}
4799
4800 // Pointer 64 bit Register R5 only
4801 operand iRegP_R5()
4802 %{
4803 constraint(ALLOC_IN_RC(r5_reg));
4804 match(RegP);
4805 // match(iRegP);
4806 match(iRegPNoSp);
4807 op_cost(0);
4808 format %{ %}
4809 interface(REG_INTER);
4810 %}
4811
4812 // Pointer 64 bit Register R10 only
4813 operand iRegP_R10()
4814 %{
4815 constraint(ALLOC_IN_RC(r10_reg));
4816 match(RegP);
4817 // match(iRegP);
4818 match(iRegPNoSp);
4819 op_cost(0);
4820 format %{ %}
4821 interface(REG_INTER);
4822 %}
4823
4824 // Long 64 bit Register R0 only
4825 operand iRegL_R0()
4826 %{
4827 constraint(ALLOC_IN_RC(r0_reg));
4828 match(RegL);
4829 match(iRegLNoSp);
4830 op_cost(0);
4831 format %{ %}
4832 interface(REG_INTER);
4833 %}
4834
4835 // Long 64 bit Register R11 only
4836 operand iRegL_R11()
4837 %{
4838 constraint(ALLOC_IN_RC(r11_reg));
4839 match(RegL);
4840 match(iRegLNoSp);
4841 op_cost(0);
4842 format %{ %}
4843 interface(REG_INTER);
4844 %}
4845
4846 // Register R0 only
4847 operand iRegI_R0()
4848 %{
4849 constraint(ALLOC_IN_RC(int_r0_reg));
4850 match(RegI);
4851 match(iRegINoSp);
4852 op_cost(0);
4853 format %{ %}
4854 interface(REG_INTER);
4855 %}
4856
4857 // Register R2 only
4858 operand iRegI_R2()
4859 %{
4860 constraint(ALLOC_IN_RC(int_r2_reg));
4861 match(RegI);
4862 match(iRegINoSp);
4863 op_cost(0);
4864 format %{ %}
4865 interface(REG_INTER);
4866 %}
4867
4868 // Register R3 only
4869 operand iRegI_R3()
4870 %{
4871 constraint(ALLOC_IN_RC(int_r3_reg));
4872 match(RegI);
4873 match(iRegINoSp);
4874 op_cost(0);
4875 format %{ %}
4876 interface(REG_INTER);
4877 %}
4878
4879
4880 // Register R4 only
4881 operand iRegI_R4()
4882 %{
4883 constraint(ALLOC_IN_RC(int_r4_reg));
4884 match(RegI);
4885 match(iRegINoSp);
4886 op_cost(0);
4887 format %{ %}
4888 interface(REG_INTER);
4889 %}
4890
4891
4892 // Pointer Register Operands
4893 // Narrow Pointer Register
4894 operand iRegN()
4895 %{
4896 constraint(ALLOC_IN_RC(any_reg32));
4897 match(RegN);
4898 match(iRegNNoSp);
4899 op_cost(0);
4900 format %{ %}
4901 interface(REG_INTER);
4902 %}
4903
4904 // Integer 64 bit Register not Special
4905 operand iRegNNoSp()
4906 %{
4907 constraint(ALLOC_IN_RC(no_special_reg32));
4908 match(RegN);
4909 op_cost(0);
4910 format %{ %}
4911 interface(REG_INTER);
4912 %}
4913
4914 // Float Register
4915 // Float register operands
4916 operand vRegF()
4917 %{
4918 constraint(ALLOC_IN_RC(float_reg));
4919 match(RegF);
4920
4921 op_cost(0);
4922 format %{ %}
4923 interface(REG_INTER);
4924 %}
4925
4926 // Double Register
4927 // Double register operands
4928 operand vRegD()
4929 %{
4930 constraint(ALLOC_IN_RC(double_reg));
4931 match(RegD);
4932
4933 op_cost(0);
4934 format %{ %}
4935 interface(REG_INTER);
4936 %}
4937
4938 // Generic vector class. This will be used for
4939 // all vector operands, including NEON and SVE.
4940 operand vReg()
4941 %{
4942 constraint(ALLOC_IN_RC(dynamic));
4943 match(VecA);
4944 match(VecD);
4945 match(VecX);
4946
4947 op_cost(0);
4948 format %{ %}
4949 interface(REG_INTER);
4950 %}
4951
4952 operand vecA()
4953 %{
4954 constraint(ALLOC_IN_RC(vectora_reg));
4955 match(VecA);
4956
4957 op_cost(0);
4958 format %{ %}
4959 interface(REG_INTER);
4960 %}
4961
4962 operand vecD()
4963 %{
4964 constraint(ALLOC_IN_RC(vectord_reg));
4965 match(VecD);
4966
4967 op_cost(0);
4968 format %{ %}
4969 interface(REG_INTER);
4970 %}
4971
4972 operand vecX()
4973 %{
4974 constraint(ALLOC_IN_RC(vectorx_reg));
4975 match(VecX);
4976
4977 op_cost(0);
4978 format %{ %}
4979 interface(REG_INTER);
4980 %}
4981
4982 operand vRegD_V0()
4983 %{
4984 constraint(ALLOC_IN_RC(v0_reg));
4985 match(RegD);
4986 op_cost(0);
4987 format %{ %}
4988 interface(REG_INTER);
4989 %}
4990
4991 operand vRegD_V1()
4992 %{
4993 constraint(ALLOC_IN_RC(v1_reg));
4994 match(RegD);
4995 op_cost(0);
4996 format %{ %}
4997 interface(REG_INTER);
4998 %}
4999
5000 operand vRegD_V2()
5001 %{
5002 constraint(ALLOC_IN_RC(v2_reg));
5003 match(RegD);
5004 op_cost(0);
5005 format %{ %}
5006 interface(REG_INTER);
5007 %}
5008
5009 operand vRegD_V3()
5010 %{
5011 constraint(ALLOC_IN_RC(v3_reg));
5012 match(RegD);
5013 op_cost(0);
5014 format %{ %}
5015 interface(REG_INTER);
5016 %}
5017
5018 operand vRegD_V4()
5019 %{
5020 constraint(ALLOC_IN_RC(v4_reg));
5021 match(RegD);
5022 op_cost(0);
5023 format %{ %}
5024 interface(REG_INTER);
5025 %}
5026
5027 operand vRegD_V5()
5028 %{
5029 constraint(ALLOC_IN_RC(v5_reg));
5030 match(RegD);
5031 op_cost(0);
5032 format %{ %}
5033 interface(REG_INTER);
5034 %}
5035
5036 operand vRegD_V6()
5037 %{
5038 constraint(ALLOC_IN_RC(v6_reg));
5039 match(RegD);
5040 op_cost(0);
5041 format %{ %}
5042 interface(REG_INTER);
5043 %}
5044
5045 operand vRegD_V7()
5046 %{
5047 constraint(ALLOC_IN_RC(v7_reg));
5048 match(RegD);
5049 op_cost(0);
5050 format %{ %}
5051 interface(REG_INTER);
5052 %}
5053
5054 operand vRegD_V12()
5055 %{
5056 constraint(ALLOC_IN_RC(v12_reg));
5057 match(RegD);
5058 op_cost(0);
5059 format %{ %}
5060 interface(REG_INTER);
5061 %}
5062
5063 operand vRegD_V13()
5064 %{
5065 constraint(ALLOC_IN_RC(v13_reg));
5066 match(RegD);
5067 op_cost(0);
5068 format %{ %}
5069 interface(REG_INTER);
5070 %}
5071
5072 operand pReg()
5073 %{
5074 constraint(ALLOC_IN_RC(pr_reg));
5075 match(RegVectMask);
5076 match(pRegGov);
5077 op_cost(0);
5078 format %{ %}
5079 interface(REG_INTER);
5080 %}
5081
5082 operand pRegGov()
5083 %{
5084 constraint(ALLOC_IN_RC(gov_pr));
5085 match(RegVectMask);
5086 match(pReg);
5087 op_cost(0);
5088 format %{ %}
5089 interface(REG_INTER);
5090 %}
5091
5092 operand pRegGov_P0()
5093 %{
5094 constraint(ALLOC_IN_RC(p0_reg));
5095 match(RegVectMask);
5096 op_cost(0);
5097 format %{ %}
5098 interface(REG_INTER);
5099 %}
5100
5101 operand pRegGov_P1()
5102 %{
5103 constraint(ALLOC_IN_RC(p1_reg));
5104 match(RegVectMask);
5105 op_cost(0);
5106 format %{ %}
5107 interface(REG_INTER);
5108 %}
5109
5110 // Flags register, used as output of signed compare instructions
5111
5112 // note that on AArch64 we also use this register as the output for
5113 // for floating point compare instructions (CmpF CmpD). this ensures
5114 // that ordered inequality tests use GT, GE, LT or LE none of which
5115 // pass through cases where the result is unordered i.e. one or both
5116 // inputs to the compare is a NaN. this means that the ideal code can
5117 // replace e.g. a GT with an LE and not end up capturing the NaN case
5118 // (where the comparison should always fail). EQ and NE tests are
5119 // always generated in ideal code so that unordered folds into the NE
5120 // case, matching the behaviour of AArch64 NE.
5121 //
5122 // This differs from x86 where the outputs of FP compares use a
5123 // special FP flags registers and where compares based on this
5124 // register are distinguished into ordered inequalities (cmpOpUCF) and
5125 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5126 // to explicitly handle the unordered case in branches. x86 also has
5127 // to include extra CMoveX rules to accept a cmpOpUCF input.
5128
5129 operand rFlagsReg()
5130 %{
5131 constraint(ALLOC_IN_RC(int_flags));
5132 match(RegFlags);
5133
5134 op_cost(0);
5135 format %{ "RFLAGS" %}
5136 interface(REG_INTER);
5137 %}
5138
5139 // Flags register, used as output of unsigned compare instructions
5140 operand rFlagsRegU()
5141 %{
5142 constraint(ALLOC_IN_RC(int_flags));
5143 match(RegFlags);
5144
5145 op_cost(0);
5146 format %{ "RFLAGSU" %}
5147 interface(REG_INTER);
5148 %}
5149
5150 // Special Registers
5151
5152 // Method Register
5153 operand inline_cache_RegP(iRegP reg)
5154 %{
5155 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5156 match(reg);
5157 match(iRegPNoSp);
5158 op_cost(0);
5159 format %{ %}
5160 interface(REG_INTER);
5161 %}
5162
5163 // Thread Register
5164 operand thread_RegP(iRegP reg)
5165 %{
5166 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5167 match(reg);
5168 op_cost(0);
5169 format %{ %}
5170 interface(REG_INTER);
5171 %}
5172
5173 //----------Memory Operands----------------------------------------------------
5174
5175 operand indirect(iRegP reg)
5176 %{
5177 constraint(ALLOC_IN_RC(ptr_reg));
5178 match(reg);
5179 op_cost(0);
5180 format %{ "[$reg]" %}
5181 interface(MEMORY_INTER) %{
5182 base($reg);
5183 index(0xffffffff);
5184 scale(0x0);
5185 disp(0x0);
5186 %}
5187 %}
5188
5189 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5190 %{
5191 constraint(ALLOC_IN_RC(ptr_reg));
5192 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5193 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5194 op_cost(0);
5195 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5196 interface(MEMORY_INTER) %{
5197 base($reg);
5198 index($ireg);
5199 scale($scale);
5200 disp(0x0);
5201 %}
5202 %}
5203
5204 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5205 %{
5206 constraint(ALLOC_IN_RC(ptr_reg));
5207 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5208 match(AddP reg (LShiftL lreg scale));
5209 op_cost(0);
5210 format %{ "$reg, $lreg lsl($scale)" %}
5211 interface(MEMORY_INTER) %{
5212 base($reg);
5213 index($lreg);
5214 scale($scale);
5215 disp(0x0);
5216 %}
5217 %}
5218
5219 operand indIndexI2L(iRegP reg, iRegI ireg)
5220 %{
5221 constraint(ALLOC_IN_RC(ptr_reg));
5222 match(AddP reg (ConvI2L ireg));
5223 op_cost(0);
5224 format %{ "$reg, $ireg, 0, I2L" %}
5225 interface(MEMORY_INTER) %{
5226 base($reg);
5227 index($ireg);
5228 scale(0x0);
5229 disp(0x0);
5230 %}
5231 %}
5232
5233 operand indIndex(iRegP reg, iRegL lreg)
5234 %{
5235 constraint(ALLOC_IN_RC(ptr_reg));
5236 match(AddP reg lreg);
5237 op_cost(0);
5238 format %{ "$reg, $lreg" %}
5239 interface(MEMORY_INTER) %{
5240 base($reg);
5241 index($lreg);
5242 scale(0x0);
5243 disp(0x0);
5244 %}
5245 %}
5246
5247 operand indOffI1(iRegP reg, immIOffset1 off)
5248 %{
5249 constraint(ALLOC_IN_RC(ptr_reg));
5250 match(AddP reg off);
5251 op_cost(0);
5252 format %{ "[$reg, $off]" %}
5253 interface(MEMORY_INTER) %{
5254 base($reg);
5255 index(0xffffffff);
5256 scale(0x0);
5257 disp($off);
5258 %}
5259 %}
5260
5261 operand indOffI2(iRegP reg, immIOffset2 off)
5262 %{
5263 constraint(ALLOC_IN_RC(ptr_reg));
5264 match(AddP reg off);
5265 op_cost(0);
5266 format %{ "[$reg, $off]" %}
5267 interface(MEMORY_INTER) %{
5268 base($reg);
5269 index(0xffffffff);
5270 scale(0x0);
5271 disp($off);
5272 %}
5273 %}
5274
5275 operand indOffI4(iRegP reg, immIOffset4 off)
5276 %{
5277 constraint(ALLOC_IN_RC(ptr_reg));
5278 match(AddP reg off);
5279 op_cost(0);
5280 format %{ "[$reg, $off]" %}
5281 interface(MEMORY_INTER) %{
5282 base($reg);
5283 index(0xffffffff);
5284 scale(0x0);
5285 disp($off);
5286 %}
5287 %}
5288
5289 operand indOffI8(iRegP reg, immIOffset8 off)
5290 %{
5291 constraint(ALLOC_IN_RC(ptr_reg));
5292 match(AddP reg off);
5293 op_cost(0);
5294 format %{ "[$reg, $off]" %}
5295 interface(MEMORY_INTER) %{
5296 base($reg);
5297 index(0xffffffff);
5298 scale(0x0);
5299 disp($off);
5300 %}
5301 %}
5302
5303 operand indOffI16(iRegP reg, immIOffset16 off)
5304 %{
5305 constraint(ALLOC_IN_RC(ptr_reg));
5306 match(AddP reg off);
5307 op_cost(0);
5308 format %{ "[$reg, $off]" %}
5309 interface(MEMORY_INTER) %{
5310 base($reg);
5311 index(0xffffffff);
5312 scale(0x0);
5313 disp($off);
5314 %}
5315 %}
5316
5317 operand indOffL1(iRegP reg, immLoffset1 off)
5318 %{
5319 constraint(ALLOC_IN_RC(ptr_reg));
5320 match(AddP reg off);
5321 op_cost(0);
5322 format %{ "[$reg, $off]" %}
5323 interface(MEMORY_INTER) %{
5324 base($reg);
5325 index(0xffffffff);
5326 scale(0x0);
5327 disp($off);
5328 %}
5329 %}
5330
5331 operand indOffL2(iRegP reg, immLoffset2 off)
5332 %{
5333 constraint(ALLOC_IN_RC(ptr_reg));
5334 match(AddP reg off);
5335 op_cost(0);
5336 format %{ "[$reg, $off]" %}
5337 interface(MEMORY_INTER) %{
5338 base($reg);
5339 index(0xffffffff);
5340 scale(0x0);
5341 disp($off);
5342 %}
5343 %}
5344
5345 operand indOffL4(iRegP reg, immLoffset4 off)
5346 %{
5347 constraint(ALLOC_IN_RC(ptr_reg));
5348 match(AddP reg off);
5349 op_cost(0);
5350 format %{ "[$reg, $off]" %}
5351 interface(MEMORY_INTER) %{
5352 base($reg);
5353 index(0xffffffff);
5354 scale(0x0);
5355 disp($off);
5356 %}
5357 %}
5358
5359 operand indOffL8(iRegP reg, immLoffset8 off)
5360 %{
5361 constraint(ALLOC_IN_RC(ptr_reg));
5362 match(AddP reg off);
5363 op_cost(0);
5364 format %{ "[$reg, $off]" %}
5365 interface(MEMORY_INTER) %{
5366 base($reg);
5367 index(0xffffffff);
5368 scale(0x0);
5369 disp($off);
5370 %}
5371 %}
5372
5373 operand indOffL16(iRegP reg, immLoffset16 off)
5374 %{
5375 constraint(ALLOC_IN_RC(ptr_reg));
5376 match(AddP reg off);
5377 op_cost(0);
5378 format %{ "[$reg, $off]" %}
5379 interface(MEMORY_INTER) %{
5380 base($reg);
5381 index(0xffffffff);
5382 scale(0x0);
5383 disp($off);
5384 %}
5385 %}
5386
5387 operand indirectX2P(iRegL reg)
5388 %{
5389 constraint(ALLOC_IN_RC(ptr_reg));
5390 match(CastX2P reg);
5391 op_cost(0);
5392 format %{ "[$reg]\t# long -> ptr" %}
5393 interface(MEMORY_INTER) %{
5394 base($reg);
5395 index(0xffffffff);
5396 scale(0x0);
5397 disp(0x0);
5398 %}
5399 %}
5400
5401 operand indOffX2P(iRegL reg, immLOffset off)
5402 %{
5403 constraint(ALLOC_IN_RC(ptr_reg));
5404 match(AddP (CastX2P reg) off);
5405 op_cost(0);
5406 format %{ "[$reg, $off]\t# long -> ptr" %}
5407 interface(MEMORY_INTER) %{
5408 base($reg);
5409 index(0xffffffff);
5410 scale(0x0);
5411 disp($off);
5412 %}
5413 %}
5414
5415 operand indirectN(iRegN reg)
5416 %{
5417 predicate(CompressedOops::shift() == 0);
5418 constraint(ALLOC_IN_RC(ptr_reg));
5419 match(DecodeN reg);
5420 op_cost(0);
5421 format %{ "[$reg]\t# narrow" %}
5422 interface(MEMORY_INTER) %{
5423 base($reg);
5424 index(0xffffffff);
5425 scale(0x0);
5426 disp(0x0);
5427 %}
5428 %}
5429
5430 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5431 %{
5432 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5433 constraint(ALLOC_IN_RC(ptr_reg));
5434 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5435 op_cost(0);
5436 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5437 interface(MEMORY_INTER) %{
5438 base($reg);
5439 index($ireg);
5440 scale($scale);
5441 disp(0x0);
5442 %}
5443 %}
5444
5445 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5446 %{
5447 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5448 constraint(ALLOC_IN_RC(ptr_reg));
5449 match(AddP (DecodeN reg) (LShiftL lreg scale));
5450 op_cost(0);
5451 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5452 interface(MEMORY_INTER) %{
5453 base($reg);
5454 index($lreg);
5455 scale($scale);
5456 disp(0x0);
5457 %}
5458 %}
5459
5460 operand indIndexI2LN(iRegN reg, iRegI ireg)
5461 %{
5462 predicate(CompressedOops::shift() == 0);
5463 constraint(ALLOC_IN_RC(ptr_reg));
5464 match(AddP (DecodeN reg) (ConvI2L ireg));
5465 op_cost(0);
5466 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5467 interface(MEMORY_INTER) %{
5468 base($reg);
5469 index($ireg);
5470 scale(0x0);
5471 disp(0x0);
5472 %}
5473 %}
5474
5475 operand indIndexN(iRegN reg, iRegL lreg)
5476 %{
5477 predicate(CompressedOops::shift() == 0);
5478 constraint(ALLOC_IN_RC(ptr_reg));
5479 match(AddP (DecodeN reg) lreg);
5480 op_cost(0);
5481 format %{ "$reg, $lreg\t# narrow" %}
5482 interface(MEMORY_INTER) %{
5483 base($reg);
5484 index($lreg);
5485 scale(0x0);
5486 disp(0x0);
5487 %}
5488 %}
5489
5490 operand indOffIN(iRegN reg, immIOffset off)
5491 %{
5492 predicate(CompressedOops::shift() == 0);
5493 constraint(ALLOC_IN_RC(ptr_reg));
5494 match(AddP (DecodeN reg) off);
5495 op_cost(0);
5496 format %{ "[$reg, $off]\t# narrow" %}
5497 interface(MEMORY_INTER) %{
5498 base($reg);
5499 index(0xffffffff);
5500 scale(0x0);
5501 disp($off);
5502 %}
5503 %}
5504
5505 operand indOffLN(iRegN reg, immLOffset off)
5506 %{
5507 predicate(CompressedOops::shift() == 0);
5508 constraint(ALLOC_IN_RC(ptr_reg));
5509 match(AddP (DecodeN reg) off);
5510 op_cost(0);
5511 format %{ "[$reg, $off]\t# narrow" %}
5512 interface(MEMORY_INTER) %{
5513 base($reg);
5514 index(0xffffffff);
5515 scale(0x0);
5516 disp($off);
5517 %}
5518 %}
5519
5520
5521 //----------Special Memory Operands--------------------------------------------
5522 // Stack Slot Operand - This operand is used for loading and storing temporary
5523 // values on the stack where a match requires a value to
5524 // flow through memory.
5525 operand stackSlotP(sRegP reg)
5526 %{
5527 constraint(ALLOC_IN_RC(stack_slots));
5528 op_cost(100);
5529 // No match rule because this operand is only generated in matching
5530 // match(RegP);
5531 format %{ "[$reg]" %}
5532 interface(MEMORY_INTER) %{
5533 base(0x1e); // RSP
5534 index(0x0); // No Index
5535 scale(0x0); // No Scale
5536 disp($reg); // Stack Offset
5537 %}
5538 %}
5539
5540 operand stackSlotI(sRegI reg)
5541 %{
5542 constraint(ALLOC_IN_RC(stack_slots));
5543 // No match rule because this operand is only generated in matching
5544 // match(RegI);
5545 format %{ "[$reg]" %}
5546 interface(MEMORY_INTER) %{
5547 base(0x1e); // RSP
5548 index(0x0); // No Index
5549 scale(0x0); // No Scale
5550 disp($reg); // Stack Offset
5551 %}
5552 %}
5553
5554 operand stackSlotF(sRegF reg)
5555 %{
5556 constraint(ALLOC_IN_RC(stack_slots));
5557 // No match rule because this operand is only generated in matching
5558 // match(RegF);
5559 format %{ "[$reg]" %}
5560 interface(MEMORY_INTER) %{
5561 base(0x1e); // RSP
5562 index(0x0); // No Index
5563 scale(0x0); // No Scale
5564 disp($reg); // Stack Offset
5565 %}
5566 %}
5567
5568 operand stackSlotD(sRegD reg)
5569 %{
5570 constraint(ALLOC_IN_RC(stack_slots));
5571 // No match rule because this operand is only generated in matching
5572 // match(RegD);
5573 format %{ "[$reg]" %}
5574 interface(MEMORY_INTER) %{
5575 base(0x1e); // RSP
5576 index(0x0); // No Index
5577 scale(0x0); // No Scale
5578 disp($reg); // Stack Offset
5579 %}
5580 %}
5581
5582 operand stackSlotL(sRegL reg)
5583 %{
5584 constraint(ALLOC_IN_RC(stack_slots));
5585 // No match rule because this operand is only generated in matching
5586 // match(RegL);
5587 format %{ "[$reg]" %}
5588 interface(MEMORY_INTER) %{
5589 base(0x1e); // RSP
5590 index(0x0); // No Index
5591 scale(0x0); // No Scale
5592 disp($reg); // Stack Offset
5593 %}
5594 %}
5595
5596 // Operands for expressing Control Flow
5597 // NOTE: Label is a predefined operand which should not be redefined in
5598 // the AD file. It is generically handled within the ADLC.
5599
5600 //----------Conditional Branch Operands----------------------------------------
5601 // Comparison Op - This is the operation of the comparison, and is limited to
5602 // the following set of codes:
5603 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5604 //
5605 // Other attributes of the comparison, such as unsignedness, are specified
5606 // by the comparison instruction that sets a condition code flags register.
5607 // That result is represented by a flags operand whose subtype is appropriate
5608 // to the unsignedness (etc.) of the comparison.
5609 //
5610 // Later, the instruction which matches both the Comparison Op (a Bool) and
5611 // the flags (produced by the Cmp) specifies the coding of the comparison op
5612 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5613
5614 // used for signed integral comparisons and fp comparisons
5615
5616 operand cmpOp()
5617 %{
5618 match(Bool);
5619
5620 format %{ "" %}
5621 interface(COND_INTER) %{
5622 equal(0x0, "eq");
5623 not_equal(0x1, "ne");
5624 less(0xb, "lt");
5625 greater_equal(0xa, "ge");
5626 less_equal(0xd, "le");
5627 greater(0xc, "gt");
5628 overflow(0x6, "vs");
5629 no_overflow(0x7, "vc");
5630 %}
5631 %}
5632
5633 // used for unsigned integral comparisons
5634
5635 operand cmpOpU()
5636 %{
5637 match(Bool);
5638
5639 format %{ "" %}
5640 interface(COND_INTER) %{
5641 equal(0x0, "eq");
5642 not_equal(0x1, "ne");
5643 less(0x3, "lo");
5644 greater_equal(0x2, "hs");
5645 less_equal(0x9, "ls");
5646 greater(0x8, "hi");
5647 overflow(0x6, "vs");
5648 no_overflow(0x7, "vc");
5649 %}
5650 %}
5651
5652 // used for certain integral comparisons which can be
5653 // converted to cbxx or tbxx instructions
5654
5655 operand cmpOpEqNe()
5656 %{
5657 match(Bool);
5658 op_cost(0);
5659 predicate(n->as_Bool()->_test._test == BoolTest::ne
5660 || n->as_Bool()->_test._test == BoolTest::eq);
5661
5662 format %{ "" %}
5663 interface(COND_INTER) %{
5664 equal(0x0, "eq");
5665 not_equal(0x1, "ne");
5666 less(0xb, "lt");
5667 greater_equal(0xa, "ge");
5668 less_equal(0xd, "le");
5669 greater(0xc, "gt");
5670 overflow(0x6, "vs");
5671 no_overflow(0x7, "vc");
5672 %}
5673 %}
5674
5675 // used for certain integral comparisons which can be
5676 // converted to cbxx or tbxx instructions
5677
5678 operand cmpOpLtGe()
5679 %{
5680 match(Bool);
5681 op_cost(0);
5682
5683 predicate(n->as_Bool()->_test._test == BoolTest::lt
5684 || n->as_Bool()->_test._test == BoolTest::ge);
5685
5686 format %{ "" %}
5687 interface(COND_INTER) %{
5688 equal(0x0, "eq");
5689 not_equal(0x1, "ne");
5690 less(0xb, "lt");
5691 greater_equal(0xa, "ge");
5692 less_equal(0xd, "le");
5693 greater(0xc, "gt");
5694 overflow(0x6, "vs");
5695 no_overflow(0x7, "vc");
5696 %}
5697 %}
5698
5699 // used for certain unsigned integral comparisons which can be
5700 // converted to cbxx or tbxx instructions
5701
5702 operand cmpOpUEqNeLeGt()
5703 %{
5704 match(Bool);
5705 op_cost(0);
5706
5707 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5708 n->as_Bool()->_test._test == BoolTest::ne ||
5709 n->as_Bool()->_test._test == BoolTest::le ||
5710 n->as_Bool()->_test._test == BoolTest::gt);
5711
5712 format %{ "" %}
5713 interface(COND_INTER) %{
5714 equal(0x0, "eq");
5715 not_equal(0x1, "ne");
5716 less(0x3, "lo");
5717 greater_equal(0x2, "hs");
5718 less_equal(0x9, "ls");
5719 greater(0x8, "hi");
5720 overflow(0x6, "vs");
5721 no_overflow(0x7, "vc");
5722 %}
5723 %}
5724
5725 // Special operand allowing long args to int ops to be truncated for free
5726
5727 operand iRegL2I(iRegL reg) %{
5728
5729 op_cost(0);
5730
5731 match(ConvL2I reg);
5732
5733 format %{ "l2i($reg)" %}
5734
5735 interface(REG_INTER)
5736 %}
5737
5738 operand iRegL2P(iRegL reg) %{
5739
5740 op_cost(0);
5741
5742 match(CastX2P reg);
5743
5744 format %{ "l2p($reg)" %}
5745
5746 interface(REG_INTER)
5747 %}
5748
5749 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5750 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5751 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5752 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5753
5754 //----------OPERAND CLASSES----------------------------------------------------
5755 // Operand Classes are groups of operands that are used as to simplify
5756 // instruction definitions by not requiring the AD writer to specify
5757 // separate instructions for every form of operand when the
5758 // instruction accepts multiple operand types with the same basic
5759 // encoding and format. The classic case of this is memory operands.
5760
5761 // memory is used to define read/write location for load/store
5762 // instruction defs. we can turn a memory op into an Address
5763
5764 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5765 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5766
5767 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5768 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5769
5770 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5771 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5772
5773 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5774 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5775
5776 // All of the memory operands. For the pipeline description.
5777 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5778 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5779 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5780
5781
5782 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5783 // operations. it allows the src to be either an iRegI or a (ConvL2I
5784 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5785 // can be elided because the 32-bit instruction will just employ the
5786 // lower 32 bits anyway.
5787 //
5788 // n.b. this does not elide all L2I conversions. if the truncated
5789 // value is consumed by more than one operation then the ConvL2I
5790 // cannot be bundled into the consuming nodes so an l2i gets planted
5791 // (actually a movw $dst $src) and the downstream instructions consume
5792 // the result of the l2i as an iRegI input. That's a shame since the
5793 // movw is actually redundant but its not too costly.
5794
5795 opclass iRegIorL2I(iRegI, iRegL2I);
5796 opclass iRegPorL2P(iRegP, iRegL2P);
5797
5798 //----------PIPELINE-----------------------------------------------------------
5799 // Rules which define the behavior of the target architectures pipeline.
5800
5801 // For specific pipelines, eg A53, define the stages of that pipeline
5802 //pipe_desc(ISS, EX1, EX2, WR);
5803 #define ISS S0
5804 #define EX1 S1
5805 #define EX2 S2
5806 #define WR S3
5807
5808 // Integer ALU reg operation
5809 pipeline %{
5810
5811 attributes %{
5812 // ARM instructions are of fixed length
5813 fixed_size_instructions; // Fixed size instructions TODO does
5814 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5815 // ARM instructions come in 32-bit word units
5816 instruction_unit_size = 4; // An instruction is 4 bytes long
5817 instruction_fetch_unit_size = 64; // The processor fetches one line
5818 instruction_fetch_units = 1; // of 64 bytes
5819
5820 // List of nop instructions
5821 nops( MachNop );
5822 %}
5823
5824 // We don't use an actual pipeline model so don't care about resources
5825 // or description. we do use pipeline classes to introduce fixed
5826 // latencies
5827
5828 //----------RESOURCES----------------------------------------------------------
5829 // Resources are the functional units available to the machine
5830
5831 resources( INS0, INS1, INS01 = INS0 | INS1,
5832 ALU0, ALU1, ALU = ALU0 | ALU1,
5833 MAC,
5834 DIV,
5835 BRANCH,
5836 LDST,
5837 NEON_FP);
5838
5839 //----------PIPELINE DESCRIPTION-----------------------------------------------
5840 // Pipeline Description specifies the stages in the machine's pipeline
5841
5842 // Define the pipeline as a generic 6 stage pipeline
5843 pipe_desc(S0, S1, S2, S3, S4, S5);
5844
5845 //----------PIPELINE CLASSES---------------------------------------------------
5846 // Pipeline Classes describe the stages in which input and output are
5847 // referenced by the hardware pipeline.
5848
5849 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5850 %{
5851 single_instruction;
5852 src1 : S1(read);
5853 src2 : S2(read);
5854 dst : S5(write);
5855 INS01 : ISS;
5856 NEON_FP : S5;
5857 %}
5858
5859 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5860 %{
5861 single_instruction;
5862 src1 : S1(read);
5863 src2 : S2(read);
5864 dst : S5(write);
5865 INS01 : ISS;
5866 NEON_FP : S5;
5867 %}
5868
5869 pipe_class fp_uop_s(vRegF dst, vRegF src)
5870 %{
5871 single_instruction;
5872 src : S1(read);
5873 dst : S5(write);
5874 INS01 : ISS;
5875 NEON_FP : S5;
5876 %}
5877
5878 pipe_class fp_uop_d(vRegD dst, vRegD src)
5879 %{
5880 single_instruction;
5881 src : S1(read);
5882 dst : S5(write);
5883 INS01 : ISS;
5884 NEON_FP : S5;
5885 %}
5886
5887 pipe_class fp_d2f(vRegF dst, vRegD src)
5888 %{
5889 single_instruction;
5890 src : S1(read);
5891 dst : S5(write);
5892 INS01 : ISS;
5893 NEON_FP : S5;
5894 %}
5895
5896 pipe_class fp_f2d(vRegD dst, vRegF src)
5897 %{
5898 single_instruction;
5899 src : S1(read);
5900 dst : S5(write);
5901 INS01 : ISS;
5902 NEON_FP : S5;
5903 %}
5904
5905 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5906 %{
5907 single_instruction;
5908 src : S1(read);
5909 dst : S5(write);
5910 INS01 : ISS;
5911 NEON_FP : S5;
5912 %}
5913
5914 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5915 %{
5916 single_instruction;
5917 src : S1(read);
5918 dst : S5(write);
5919 INS01 : ISS;
5920 NEON_FP : S5;
5921 %}
5922
5923 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5924 %{
5925 single_instruction;
5926 src : S1(read);
5927 dst : S5(write);
5928 INS01 : ISS;
5929 NEON_FP : S5;
5930 %}
5931
5932 pipe_class fp_l2f(vRegF dst, iRegL src)
5933 %{
5934 single_instruction;
5935 src : S1(read);
5936 dst : S5(write);
5937 INS01 : ISS;
5938 NEON_FP : S5;
5939 %}
5940
5941 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5942 %{
5943 single_instruction;
5944 src : S1(read);
5945 dst : S5(write);
5946 INS01 : ISS;
5947 NEON_FP : S5;
5948 %}
5949
5950 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5951 %{
5952 single_instruction;
5953 src : S1(read);
5954 dst : S5(write);
5955 INS01 : ISS;
5956 NEON_FP : S5;
5957 %}
5958
5959 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5960 %{
5961 single_instruction;
5962 src : S1(read);
5963 dst : S5(write);
5964 INS01 : ISS;
5965 NEON_FP : S5;
5966 %}
5967
5968 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5969 %{
5970 single_instruction;
5971 src : S1(read);
5972 dst : S5(write);
5973 INS01 : ISS;
5974 NEON_FP : S5;
5975 %}
5976
5977 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5978 %{
5979 single_instruction;
5980 src1 : S1(read);
5981 src2 : S2(read);
5982 dst : S5(write);
5983 INS0 : ISS;
5984 NEON_FP : S5;
5985 %}
5986
5987 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
5988 %{
5989 single_instruction;
5990 src1 : S1(read);
5991 src2 : S2(read);
5992 dst : S5(write);
5993 INS0 : ISS;
5994 NEON_FP : S5;
5995 %}
5996
5997 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
5998 %{
5999 single_instruction;
6000 cr : S1(read);
6001 src1 : S1(read);
6002 src2 : S1(read);
6003 dst : S3(write);
6004 INS01 : ISS;
6005 NEON_FP : S3;
6006 %}
6007
6008 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6009 %{
6010 single_instruction;
6011 cr : S1(read);
6012 src1 : S1(read);
6013 src2 : S1(read);
6014 dst : S3(write);
6015 INS01 : ISS;
6016 NEON_FP : S3;
6017 %}
6018
6019 pipe_class fp_imm_s(vRegF dst)
6020 %{
6021 single_instruction;
6022 dst : S3(write);
6023 INS01 : ISS;
6024 NEON_FP : S3;
6025 %}
6026
6027 pipe_class fp_imm_d(vRegD dst)
6028 %{
6029 single_instruction;
6030 dst : S3(write);
6031 INS01 : ISS;
6032 NEON_FP : S3;
6033 %}
6034
6035 pipe_class fp_load_constant_s(vRegF dst)
6036 %{
6037 single_instruction;
6038 dst : S4(write);
6039 INS01 : ISS;
6040 NEON_FP : S4;
6041 %}
6042
6043 pipe_class fp_load_constant_d(vRegD dst)
6044 %{
6045 single_instruction;
6046 dst : S4(write);
6047 INS01 : ISS;
6048 NEON_FP : S4;
6049 %}
6050
6051 //------- Integer ALU operations --------------------------
6052
6053 // Integer ALU reg-reg operation
6054 // Operands needed in EX1, result generated in EX2
6055 // Eg. ADD x0, x1, x2
6056 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6057 %{
6058 single_instruction;
6059 dst : EX2(write);
6060 src1 : EX1(read);
6061 src2 : EX1(read);
6062 INS01 : ISS; // Dual issue as instruction 0 or 1
6063 ALU : EX2;
6064 %}
6065
6066 // Integer ALU reg-reg operation with constant shift
6067 // Shifted register must be available in LATE_ISS instead of EX1
6068 // Eg. ADD x0, x1, x2, LSL #2
6069 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6070 %{
6071 single_instruction;
6072 dst : EX2(write);
6073 src1 : EX1(read);
6074 src2 : ISS(read);
6075 INS01 : ISS;
6076 ALU : EX2;
6077 %}
6078
6079 // Integer ALU reg operation with constant shift
6080 // Eg. LSL x0, x1, #shift
6081 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6082 %{
6083 single_instruction;
6084 dst : EX2(write);
6085 src1 : ISS(read);
6086 INS01 : ISS;
6087 ALU : EX2;
6088 %}
6089
6090 // Integer ALU reg-reg operation with variable shift
6091 // Both operands must be available in LATE_ISS instead of EX1
6092 // Result is available in EX1 instead of EX2
6093 // Eg. LSLV x0, x1, x2
6094 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6095 %{
6096 single_instruction;
6097 dst : EX1(write);
6098 src1 : ISS(read);
6099 src2 : ISS(read);
6100 INS01 : ISS;
6101 ALU : EX1;
6102 %}
6103
6104 // Integer ALU reg-reg operation with extract
6105 // As for _vshift above, but result generated in EX2
6106 // Eg. EXTR x0, x1, x2, #N
6107 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6108 %{
6109 single_instruction;
6110 dst : EX2(write);
6111 src1 : ISS(read);
6112 src2 : ISS(read);
6113 INS1 : ISS; // Can only dual issue as Instruction 1
6114 ALU : EX1;
6115 %}
6116
6117 // Integer ALU reg operation
6118 // Eg. NEG x0, x1
6119 pipe_class ialu_reg(iRegI dst, iRegI src)
6120 %{
6121 single_instruction;
6122 dst : EX2(write);
6123 src : EX1(read);
6124 INS01 : ISS;
6125 ALU : EX2;
6126 %}
6127
6128 // Integer ALU reg mmediate operation
6129 // Eg. ADD x0, x1, #N
6130 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6131 %{
6132 single_instruction;
6133 dst : EX2(write);
6134 src1 : EX1(read);
6135 INS01 : ISS;
6136 ALU : EX2;
6137 %}
6138
6139 // Integer ALU immediate operation (no source operands)
6140 // Eg. MOV x0, #N
6141 pipe_class ialu_imm(iRegI dst)
6142 %{
6143 single_instruction;
6144 dst : EX1(write);
6145 INS01 : ISS;
6146 ALU : EX1;
6147 %}
6148
6149 //------- Compare operation -------------------------------
6150
6151 // Compare reg-reg
6152 // Eg. CMP x0, x1
6153 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6154 %{
6155 single_instruction;
6156 // fixed_latency(16);
6157 cr : EX2(write);
6158 op1 : EX1(read);
6159 op2 : EX1(read);
6160 INS01 : ISS;
6161 ALU : EX2;
6162 %}
6163
6164 // Compare reg-reg
6165 // Eg. CMP x0, #N
6166 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6167 %{
6168 single_instruction;
6169 // fixed_latency(16);
6170 cr : EX2(write);
6171 op1 : EX1(read);
6172 INS01 : ISS;
6173 ALU : EX2;
6174 %}
6175
6176 //------- Conditional instructions ------------------------
6177
6178 // Conditional no operands
6179 // Eg. CSINC x0, zr, zr, <cond>
6180 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6181 %{
6182 single_instruction;
6183 cr : EX1(read);
6184 dst : EX2(write);
6185 INS01 : ISS;
6186 ALU : EX2;
6187 %}
6188
6189 // Conditional 2 operand
6190 // EG. CSEL X0, X1, X2, <cond>
6191 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6192 %{
6193 single_instruction;
6194 cr : EX1(read);
6195 src1 : EX1(read);
6196 src2 : EX1(read);
6197 dst : EX2(write);
6198 INS01 : ISS;
6199 ALU : EX2;
6200 %}
6201
6202 // Conditional 2 operand
6203 // EG. CSEL X0, X1, X2, <cond>
6204 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6205 %{
6206 single_instruction;
6207 cr : EX1(read);
6208 src : EX1(read);
6209 dst : EX2(write);
6210 INS01 : ISS;
6211 ALU : EX2;
6212 %}
6213
6214 //------- Multiply pipeline operations --------------------
6215
6216 // Multiply reg-reg
6217 // Eg. MUL w0, w1, w2
6218 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6219 %{
6220 single_instruction;
6221 dst : WR(write);
6222 src1 : ISS(read);
6223 src2 : ISS(read);
6224 INS01 : ISS;
6225 MAC : WR;
6226 %}
6227
6228 // Multiply accumulate
6229 // Eg. MADD w0, w1, w2, w3
6230 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6231 %{
6232 single_instruction;
6233 dst : WR(write);
6234 src1 : ISS(read);
6235 src2 : ISS(read);
6236 src3 : ISS(read);
6237 INS01 : ISS;
6238 MAC : WR;
6239 %}
6240
6241 // Eg. MUL w0, w1, w2
6242 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6243 %{
6244 single_instruction;
6245 fixed_latency(3); // Maximum latency for 64 bit mul
6246 dst : WR(write);
6247 src1 : ISS(read);
6248 src2 : ISS(read);
6249 INS01 : ISS;
6250 MAC : WR;
6251 %}
6252
6253 // Multiply accumulate
6254 // Eg. MADD w0, w1, w2, w3
6255 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6256 %{
6257 single_instruction;
6258 fixed_latency(3); // Maximum latency for 64 bit mul
6259 dst : WR(write);
6260 src1 : ISS(read);
6261 src2 : ISS(read);
6262 src3 : ISS(read);
6263 INS01 : ISS;
6264 MAC : WR;
6265 %}
6266
6267 //------- Divide pipeline operations --------------------
6268
6269 // Eg. SDIV w0, w1, w2
6270 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6271 %{
6272 single_instruction;
6273 fixed_latency(8); // Maximum latency for 32 bit divide
6274 dst : WR(write);
6275 src1 : ISS(read);
6276 src2 : ISS(read);
6277 INS0 : ISS; // Can only dual issue as instruction 0
6278 DIV : WR;
6279 %}
6280
6281 // Eg. SDIV x0, x1, x2
6282 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6283 %{
6284 single_instruction;
6285 fixed_latency(16); // Maximum latency for 64 bit divide
6286 dst : WR(write);
6287 src1 : ISS(read);
6288 src2 : ISS(read);
6289 INS0 : ISS; // Can only dual issue as instruction 0
6290 DIV : WR;
6291 %}
6292
6293 //------- Load pipeline operations ------------------------
6294
6295 // Load - prefetch
6296 // Eg. PFRM <mem>
6297 pipe_class iload_prefetch(memory mem)
6298 %{
6299 single_instruction;
6300 mem : ISS(read);
6301 INS01 : ISS;
6302 LDST : WR;
6303 %}
6304
6305 // Load - reg, mem
6306 // Eg. LDR x0, <mem>
6307 pipe_class iload_reg_mem(iRegI dst, memory mem)
6308 %{
6309 single_instruction;
6310 dst : WR(write);
6311 mem : ISS(read);
6312 INS01 : ISS;
6313 LDST : WR;
6314 %}
6315
6316 // Load - reg, reg
6317 // Eg. LDR x0, [sp, x1]
6318 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6319 %{
6320 single_instruction;
6321 dst : WR(write);
6322 src : ISS(read);
6323 INS01 : ISS;
6324 LDST : WR;
6325 %}
6326
6327 //------- Store pipeline operations -----------------------
6328
6329 // Store - zr, mem
6330 // Eg. STR zr, <mem>
6331 pipe_class istore_mem(memory mem)
6332 %{
6333 single_instruction;
6334 mem : ISS(read);
6335 INS01 : ISS;
6336 LDST : WR;
6337 %}
6338
6339 // Store - reg, mem
6340 // Eg. STR x0, <mem>
6341 pipe_class istore_reg_mem(iRegI src, memory mem)
6342 %{
6343 single_instruction;
6344 mem : ISS(read);
6345 src : EX2(read);
6346 INS01 : ISS;
6347 LDST : WR;
6348 %}
6349
6350 // Store - reg, reg
6351 // Eg. STR x0, [sp, x1]
6352 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6353 %{
6354 single_instruction;
6355 dst : ISS(read);
6356 src : EX2(read);
6357 INS01 : ISS;
6358 LDST : WR;
6359 %}
6360
6361 //------- Store pipeline operations -----------------------
6362
6363 // Branch
6364 pipe_class pipe_branch()
6365 %{
6366 single_instruction;
6367 INS01 : ISS;
6368 BRANCH : EX1;
6369 %}
6370
6371 // Conditional branch
6372 pipe_class pipe_branch_cond(rFlagsReg cr)
6373 %{
6374 single_instruction;
6375 cr : EX1(read);
6376 INS01 : ISS;
6377 BRANCH : EX1;
6378 %}
6379
6380 // Compare & Branch
6381 // EG. CBZ/CBNZ
6382 pipe_class pipe_cmp_branch(iRegI op1)
6383 %{
6384 single_instruction;
6385 op1 : EX1(read);
6386 INS01 : ISS;
6387 BRANCH : EX1;
6388 %}
6389
6390 //------- Synchronisation operations ----------------------
6391
6392 // Any operation requiring serialization.
6393 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6394 pipe_class pipe_serial()
6395 %{
6396 single_instruction;
6397 force_serialization;
6398 fixed_latency(16);
6399 INS01 : ISS(2); // Cannot dual issue with any other instruction
6400 LDST : WR;
6401 %}
6402
6403 // Generic big/slow expanded idiom - also serialized
6404 pipe_class pipe_slow()
6405 %{
6406 instruction_count(10);
6407 multiple_bundles;
6408 force_serialization;
6409 fixed_latency(16);
6410 INS01 : ISS(2); // Cannot dual issue with any other instruction
6411 LDST : WR;
6412 %}
6413
6414 // Empty pipeline class
6415 pipe_class pipe_class_empty()
6416 %{
6417 single_instruction;
6418 fixed_latency(0);
6419 %}
6420
6421 // Default pipeline class.
6422 pipe_class pipe_class_default()
6423 %{
6424 single_instruction;
6425 fixed_latency(2);
6426 %}
6427
6428 // Pipeline class for compares.
6429 pipe_class pipe_class_compare()
6430 %{
6431 single_instruction;
6432 fixed_latency(16);
6433 %}
6434
6435 // Pipeline class for memory operations.
6436 pipe_class pipe_class_memory()
6437 %{
6438 single_instruction;
6439 fixed_latency(16);
6440 %}
6441
6442 // Pipeline class for call.
6443 pipe_class pipe_class_call()
6444 %{
6445 single_instruction;
6446 fixed_latency(100);
6447 %}
6448
6449 // Define the class for the Nop node.
6450 define %{
6451 MachNop = pipe_class_empty;
6452 %}
6453
6454 %}
6455 //----------INSTRUCTIONS-------------------------------------------------------
6456 //
6457 // match -- States which machine-independent subtree may be replaced
6458 // by this instruction.
6459 // ins_cost -- The estimated cost of this instruction is used by instruction
6460 // selection to identify a minimum cost tree of machine
6461 // instructions that matches a tree of machine-independent
6462 // instructions.
6463 // format -- A string providing the disassembly for this instruction.
6464 // The value of an instruction's operand may be inserted
6465 // by referring to it with a '$' prefix.
6466 // opcode -- Three instruction opcodes may be provided. These are referred
6467 // to within an encode class as $primary, $secondary, and $tertiary
6468 // rrspectively. The primary opcode is commonly used to
6469 // indicate the type of machine instruction, while secondary
6470 // and tertiary are often used for prefix options or addressing
6471 // modes.
6472 // ins_encode -- A list of encode classes with parameters. The encode class
6473 // name must have been defined in an 'enc_class' specification
6474 // in the encode section of the architecture description.
6475
6476 // ============================================================================
6477 // Memory (Load/Store) Instructions
6478
6479 // Load Instructions
6480
6481 // Load Byte (8 bit signed)
6482 instruct loadB(iRegINoSp dst, memory1 mem)
6483 %{
6484 match(Set dst (LoadB mem));
6485 predicate(!needs_acquiring_load(n));
6486
6487 ins_cost(4 * INSN_COST);
6488 format %{ "ldrsbw $dst, $mem\t# byte" %}
6489
6490 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6491
6492 ins_pipe(iload_reg_mem);
6493 %}
6494
6495 // Load Byte (8 bit signed) into long
6496 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6497 %{
6498 match(Set dst (ConvI2L (LoadB mem)));
6499 predicate(!needs_acquiring_load(n->in(1)));
6500
6501 ins_cost(4 * INSN_COST);
6502 format %{ "ldrsb $dst, $mem\t# byte" %}
6503
6504 ins_encode(aarch64_enc_ldrsb(dst, mem));
6505
6506 ins_pipe(iload_reg_mem);
6507 %}
6508
6509 // Load Byte (8 bit unsigned)
6510 instruct loadUB(iRegINoSp dst, memory1 mem)
6511 %{
6512 match(Set dst (LoadUB mem));
6513 predicate(!needs_acquiring_load(n));
6514
6515 ins_cost(4 * INSN_COST);
6516 format %{ "ldrbw $dst, $mem\t# byte" %}
6517
6518 ins_encode(aarch64_enc_ldrb(dst, mem));
6519
6520 ins_pipe(iload_reg_mem);
6521 %}
6522
6523 // Load Byte (8 bit unsigned) into long
6524 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6525 %{
6526 match(Set dst (ConvI2L (LoadUB mem)));
6527 predicate(!needs_acquiring_load(n->in(1)));
6528
6529 ins_cost(4 * INSN_COST);
6530 format %{ "ldrb $dst, $mem\t# byte" %}
6531
6532 ins_encode(aarch64_enc_ldrb(dst, mem));
6533
6534 ins_pipe(iload_reg_mem);
6535 %}
6536
6537 // Load Short (16 bit signed)
6538 instruct loadS(iRegINoSp dst, memory2 mem)
6539 %{
6540 match(Set dst (LoadS mem));
6541 predicate(!needs_acquiring_load(n));
6542
6543 ins_cost(4 * INSN_COST);
6544 format %{ "ldrshw $dst, $mem\t# short" %}
6545
6546 ins_encode(aarch64_enc_ldrshw(dst, mem));
6547
6548 ins_pipe(iload_reg_mem);
6549 %}
6550
6551 // Load Short (16 bit signed) into long
6552 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6553 %{
6554 match(Set dst (ConvI2L (LoadS mem)));
6555 predicate(!needs_acquiring_load(n->in(1)));
6556
6557 ins_cost(4 * INSN_COST);
6558 format %{ "ldrsh $dst, $mem\t# short" %}
6559
6560 ins_encode(aarch64_enc_ldrsh(dst, mem));
6561
6562 ins_pipe(iload_reg_mem);
6563 %}
6564
6565 // Load Char (16 bit unsigned)
6566 instruct loadUS(iRegINoSp dst, memory2 mem)
6567 %{
6568 match(Set dst (LoadUS mem));
6569 predicate(!needs_acquiring_load(n));
6570
6571 ins_cost(4 * INSN_COST);
6572 format %{ "ldrh $dst, $mem\t# short" %}
6573
6574 ins_encode(aarch64_enc_ldrh(dst, mem));
6575
6576 ins_pipe(iload_reg_mem);
6577 %}
6578
6579 // Load Short/Char (16 bit unsigned) into long
6580 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6581 %{
6582 match(Set dst (ConvI2L (LoadUS mem)));
6583 predicate(!needs_acquiring_load(n->in(1)));
6584
6585 ins_cost(4 * INSN_COST);
6586 format %{ "ldrh $dst, $mem\t# short" %}
6587
6588 ins_encode(aarch64_enc_ldrh(dst, mem));
6589
6590 ins_pipe(iload_reg_mem);
6591 %}
6592
6593 // Load Integer (32 bit signed)
6594 instruct loadI(iRegINoSp dst, memory4 mem)
6595 %{
6596 match(Set dst (LoadI mem));
6597 predicate(!needs_acquiring_load(n));
6598
6599 ins_cost(4 * INSN_COST);
6600 format %{ "ldrw $dst, $mem\t# int" %}
6601
6602 ins_encode(aarch64_enc_ldrw(dst, mem));
6603
6604 ins_pipe(iload_reg_mem);
6605 %}
6606
6607 // Load Integer (32 bit signed) into long
6608 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6609 %{
6610 match(Set dst (ConvI2L (LoadI mem)));
6611 predicate(!needs_acquiring_load(n->in(1)));
6612
6613 ins_cost(4 * INSN_COST);
6614 format %{ "ldrsw $dst, $mem\t# int" %}
6615
6616 ins_encode(aarch64_enc_ldrsw(dst, mem));
6617
6618 ins_pipe(iload_reg_mem);
6619 %}
6620
6621 // Load Integer (32 bit unsigned) into long
6622 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6623 %{
6624 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6625 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6626
6627 ins_cost(4 * INSN_COST);
6628 format %{ "ldrw $dst, $mem\t# int" %}
6629
6630 ins_encode(aarch64_enc_ldrw(dst, mem));
6631
6632 ins_pipe(iload_reg_mem);
6633 %}
6634
6635 // Load Long (64 bit signed)
6636 instruct loadL(iRegLNoSp dst, memory8 mem)
6637 %{
6638 match(Set dst (LoadL mem));
6639 predicate(!needs_acquiring_load(n));
6640
6641 ins_cost(4 * INSN_COST);
6642 format %{ "ldr $dst, $mem\t# int" %}
6643
6644 ins_encode(aarch64_enc_ldr(dst, mem));
6645
6646 ins_pipe(iload_reg_mem);
6647 %}
6648
6649 // Load Range
6650 instruct loadRange(iRegINoSp dst, memory4 mem)
6651 %{
6652 match(Set dst (LoadRange mem));
6653
6654 ins_cost(4 * INSN_COST);
6655 format %{ "ldrw $dst, $mem\t# range" %}
6656
6657 ins_encode(aarch64_enc_ldrw(dst, mem));
6658
6659 ins_pipe(iload_reg_mem);
6660 %}
6661
6662 // Load Pointer
6663 instruct loadP(iRegPNoSp dst, memory8 mem)
6664 %{
6665 match(Set dst (LoadP mem));
6666 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6667
6668 ins_cost(4 * INSN_COST);
6669 format %{ "ldr $dst, $mem\t# ptr" %}
6670
6671 ins_encode(aarch64_enc_ldr(dst, mem));
6672
6673 ins_pipe(iload_reg_mem);
6674 %}
6675
6676 // Load Compressed Pointer
6677 instruct loadN(iRegNNoSp dst, memory4 mem)
6678 %{
6679 match(Set dst (LoadN mem));
6680 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6681
6682 ins_cost(4 * INSN_COST);
6683 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6684
6685 ins_encode(aarch64_enc_ldrw(dst, mem));
6686
6687 ins_pipe(iload_reg_mem);
6688 %}
6689
6690 // Load Klass Pointer
6691 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6692 %{
6693 match(Set dst (LoadKlass mem));
6694 predicate(!needs_acquiring_load(n));
6695
6696 ins_cost(4 * INSN_COST);
6697 format %{ "ldr $dst, $mem\t# class" %}
6698
6699 ins_encode(aarch64_enc_ldr(dst, mem));
6700
6701 ins_pipe(iload_reg_mem);
6702 %}
6703
6704 // Load Narrow Klass Pointer
6705 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6706 %{
6707 match(Set dst (LoadNKlass mem));
6708 predicate(!needs_acquiring_load(n));
6709
6710 ins_cost(4 * INSN_COST);
6711 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6712
6713 ins_encode(aarch64_enc_ldrw(dst, mem));
6714
6715 ins_pipe(iload_reg_mem);
6716 %}
6717
6718 // Load Float
6719 instruct loadF(vRegF dst, memory4 mem)
6720 %{
6721 match(Set dst (LoadF mem));
6722 predicate(!needs_acquiring_load(n));
6723
6724 ins_cost(4 * INSN_COST);
6725 format %{ "ldrs $dst, $mem\t# float" %}
6726
6727 ins_encode( aarch64_enc_ldrs(dst, mem) );
6728
6729 ins_pipe(pipe_class_memory);
6730 %}
6731
6732 // Load Double
6733 instruct loadD(vRegD dst, memory8 mem)
6734 %{
6735 match(Set dst (LoadD mem));
6736 predicate(!needs_acquiring_load(n));
6737
6738 ins_cost(4 * INSN_COST);
6739 format %{ "ldrd $dst, $mem\t# double" %}
6740
6741 ins_encode( aarch64_enc_ldrd(dst, mem) );
6742
6743 ins_pipe(pipe_class_memory);
6744 %}
6745
6746
6747 // Load Int Constant
6748 instruct loadConI(iRegINoSp dst, immI src)
6749 %{
6750 match(Set dst src);
6751
6752 ins_cost(INSN_COST);
6753 format %{ "mov $dst, $src\t# int" %}
6754
6755 ins_encode( aarch64_enc_movw_imm(dst, src) );
6756
6757 ins_pipe(ialu_imm);
6758 %}
6759
6760 // Load Long Constant
6761 instruct loadConL(iRegLNoSp dst, immL src)
6762 %{
6763 match(Set dst src);
6764
6765 ins_cost(INSN_COST);
6766 format %{ "mov $dst, $src\t# long" %}
6767
6768 ins_encode( aarch64_enc_mov_imm(dst, src) );
6769
6770 ins_pipe(ialu_imm);
6771 %}
6772
6773 // Load Pointer Constant
6774
6775 instruct loadConP(iRegPNoSp dst, immP con)
6776 %{
6777 match(Set dst con);
6778
6779 ins_cost(INSN_COST * 4);
6780 format %{
6781 "mov $dst, $con\t# ptr"
6782 %}
6783
6784 ins_encode(aarch64_enc_mov_p(dst, con));
6785
6786 ins_pipe(ialu_imm);
6787 %}
6788
6789 // Load Null Pointer Constant
6790
6791 instruct loadConP0(iRegPNoSp dst, immP0 con)
6792 %{
6793 match(Set dst con);
6794
6795 ins_cost(INSN_COST);
6796 format %{ "mov $dst, $con\t# nullptr ptr" %}
6797
6798 ins_encode(aarch64_enc_mov_p0(dst, con));
6799
6800 ins_pipe(ialu_imm);
6801 %}
6802
6803 // Load Pointer Constant One
6804
6805 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6806 %{
6807 match(Set dst con);
6808
6809 ins_cost(INSN_COST);
6810 format %{ "mov $dst, $con\t# nullptr ptr" %}
6811
6812 ins_encode(aarch64_enc_mov_p1(dst, con));
6813
6814 ins_pipe(ialu_imm);
6815 %}
6816
6817 // Load Byte Map Base Constant
6818
6819 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6820 %{
6821 match(Set dst con);
6822
6823 ins_cost(INSN_COST);
6824 format %{ "adr $dst, $con\t# Byte Map Base" %}
6825
6826 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6827
6828 ins_pipe(ialu_imm);
6829 %}
6830
6831 // Load Narrow Pointer Constant
6832
6833 instruct loadConN(iRegNNoSp dst, immN con)
6834 %{
6835 match(Set dst con);
6836
6837 ins_cost(INSN_COST * 4);
6838 format %{ "mov $dst, $con\t# compressed ptr" %}
6839
6840 ins_encode(aarch64_enc_mov_n(dst, con));
6841
6842 ins_pipe(ialu_imm);
6843 %}
6844
6845 // Load Narrow Null Pointer Constant
6846
6847 instruct loadConN0(iRegNNoSp dst, immN0 con)
6848 %{
6849 match(Set dst con);
6850
6851 ins_cost(INSN_COST);
6852 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6853
6854 ins_encode(aarch64_enc_mov_n0(dst, con));
6855
6856 ins_pipe(ialu_imm);
6857 %}
6858
6859 // Load Narrow Klass Constant
6860
6861 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6862 %{
6863 match(Set dst con);
6864
6865 ins_cost(INSN_COST);
6866 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6867
6868 ins_encode(aarch64_enc_mov_nk(dst, con));
6869
6870 ins_pipe(ialu_imm);
6871 %}
6872
6873 // Load Packed Float Constant
6874
6875 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6876 match(Set dst con);
6877 ins_cost(INSN_COST * 4);
6878 format %{ "fmovs $dst, $con"%}
6879 ins_encode %{
6880 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6881 %}
6882
6883 ins_pipe(fp_imm_s);
6884 %}
6885
6886 // Load Float Constant
6887
6888 instruct loadConF(vRegF dst, immF con) %{
6889 match(Set dst con);
6890
6891 ins_cost(INSN_COST * 4);
6892
6893 format %{
6894 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6895 %}
6896
6897 ins_encode %{
6898 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6899 %}
6900
6901 ins_pipe(fp_load_constant_s);
6902 %}
6903
6904 // Load Packed Double Constant
6905
6906 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6907 match(Set dst con);
6908 ins_cost(INSN_COST);
6909 format %{ "fmovd $dst, $con"%}
6910 ins_encode %{
6911 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6912 %}
6913
6914 ins_pipe(fp_imm_d);
6915 %}
6916
6917 // Load Double Constant
6918
6919 instruct loadConD(vRegD dst, immD con) %{
6920 match(Set dst con);
6921
6922 ins_cost(INSN_COST * 5);
6923 format %{
6924 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6925 %}
6926
6927 ins_encode %{
6928 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6929 %}
6930
6931 ins_pipe(fp_load_constant_d);
6932 %}
6933
6934 // Store Instructions
6935
6936 // Store Byte
6937 instruct storeB(iRegIorL2I src, memory1 mem)
6938 %{
6939 match(Set mem (StoreB mem src));
6940 predicate(!needs_releasing_store(n));
6941
6942 ins_cost(INSN_COST);
6943 format %{ "strb $src, $mem\t# byte" %}
6944
6945 ins_encode(aarch64_enc_strb(src, mem));
6946
6947 ins_pipe(istore_reg_mem);
6948 %}
6949
6950
6951 instruct storeimmB0(immI0 zero, memory1 mem)
6952 %{
6953 match(Set mem (StoreB mem zero));
6954 predicate(!needs_releasing_store(n));
6955
6956 ins_cost(INSN_COST);
6957 format %{ "strb rscractch2, $mem\t# byte" %}
6958
6959 ins_encode(aarch64_enc_strb0(mem));
6960
6961 ins_pipe(istore_mem);
6962 %}
6963
6964 // Store Char/Short
6965 instruct storeC(iRegIorL2I src, memory2 mem)
6966 %{
6967 match(Set mem (StoreC mem src));
6968 predicate(!needs_releasing_store(n));
6969
6970 ins_cost(INSN_COST);
6971 format %{ "strh $src, $mem\t# short" %}
6972
6973 ins_encode(aarch64_enc_strh(src, mem));
6974
6975 ins_pipe(istore_reg_mem);
6976 %}
6977
6978 instruct storeimmC0(immI0 zero, memory2 mem)
6979 %{
6980 match(Set mem (StoreC mem zero));
6981 predicate(!needs_releasing_store(n));
6982
6983 ins_cost(INSN_COST);
6984 format %{ "strh zr, $mem\t# short" %}
6985
6986 ins_encode(aarch64_enc_strh0(mem));
6987
6988 ins_pipe(istore_mem);
6989 %}
6990
6991 // Store Integer
6992
6993 instruct storeI(iRegIorL2I src, memory4 mem)
6994 %{
6995 match(Set mem(StoreI mem src));
6996 predicate(!needs_releasing_store(n));
6997
6998 ins_cost(INSN_COST);
6999 format %{ "strw $src, $mem\t# int" %}
7000
7001 ins_encode(aarch64_enc_strw(src, mem));
7002
7003 ins_pipe(istore_reg_mem);
7004 %}
7005
7006 instruct storeimmI0(immI0 zero, memory4 mem)
7007 %{
7008 match(Set mem(StoreI mem zero));
7009 predicate(!needs_releasing_store(n));
7010
7011 ins_cost(INSN_COST);
7012 format %{ "strw zr, $mem\t# int" %}
7013
7014 ins_encode(aarch64_enc_strw0(mem));
7015
7016 ins_pipe(istore_mem);
7017 %}
7018
7019 // Store Long (64 bit signed)
7020 instruct storeL(iRegL src, memory8 mem)
7021 %{
7022 match(Set mem (StoreL mem src));
7023 predicate(!needs_releasing_store(n));
7024
7025 ins_cost(INSN_COST);
7026 format %{ "str $src, $mem\t# int" %}
7027
7028 ins_encode(aarch64_enc_str(src, mem));
7029
7030 ins_pipe(istore_reg_mem);
7031 %}
7032
7033 // Store Long (64 bit signed)
7034 instruct storeimmL0(immL0 zero, memory8 mem)
7035 %{
7036 match(Set mem (StoreL mem zero));
7037 predicate(!needs_releasing_store(n));
7038
7039 ins_cost(INSN_COST);
7040 format %{ "str zr, $mem\t# int" %}
7041
7042 ins_encode(aarch64_enc_str0(mem));
7043
7044 ins_pipe(istore_mem);
7045 %}
7046
7047 // Store Pointer
7048 instruct storeP(iRegP src, memory8 mem)
7049 %{
7050 match(Set mem (StoreP mem src));
7051 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7052
7053 ins_cost(INSN_COST);
7054 format %{ "str $src, $mem\t# ptr" %}
7055
7056 ins_encode(aarch64_enc_str(src, mem));
7057
7058 ins_pipe(istore_reg_mem);
7059 %}
7060
7061 // Store Pointer
7062 instruct storeimmP0(immP0 zero, memory8 mem)
7063 %{
7064 match(Set mem (StoreP mem zero));
7065 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7066
7067 ins_cost(INSN_COST);
7068 format %{ "str zr, $mem\t# ptr" %}
7069
7070 ins_encode(aarch64_enc_str0(mem));
7071
7072 ins_pipe(istore_mem);
7073 %}
7074
7075 // Store Compressed Pointer
7076 instruct storeN(iRegN src, memory4 mem)
7077 %{
7078 match(Set mem (StoreN mem src));
7079 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7080
7081 ins_cost(INSN_COST);
7082 format %{ "strw $src, $mem\t# compressed ptr" %}
7083
7084 ins_encode(aarch64_enc_strw(src, mem));
7085
7086 ins_pipe(istore_reg_mem);
7087 %}
7088
7089 instruct storeImmN0(immN0 zero, memory4 mem)
7090 %{
7091 match(Set mem (StoreN mem zero));
7092 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7093
7094 ins_cost(INSN_COST);
7095 format %{ "strw zr, $mem\t# compressed ptr" %}
7096
7097 ins_encode(aarch64_enc_strw0(mem));
7098
7099 ins_pipe(istore_mem);
7100 %}
7101
7102 // Store Float
7103 instruct storeF(vRegF src, memory4 mem)
7104 %{
7105 match(Set mem (StoreF mem src));
7106 predicate(!needs_releasing_store(n));
7107
7108 ins_cost(INSN_COST);
7109 format %{ "strs $src, $mem\t# float" %}
7110
7111 ins_encode( aarch64_enc_strs(src, mem) );
7112
7113 ins_pipe(pipe_class_memory);
7114 %}
7115
7116 // TODO
7117 // implement storeImmF0 and storeFImmPacked
7118
7119 // Store Double
7120 instruct storeD(vRegD src, memory8 mem)
7121 %{
7122 match(Set mem (StoreD mem src));
7123 predicate(!needs_releasing_store(n));
7124
7125 ins_cost(INSN_COST);
7126 format %{ "strd $src, $mem\t# double" %}
7127
7128 ins_encode( aarch64_enc_strd(src, mem) );
7129
7130 ins_pipe(pipe_class_memory);
7131 %}
7132
7133 // Store Compressed Klass Pointer
7134 instruct storeNKlass(iRegN src, memory4 mem)
7135 %{
7136 predicate(!needs_releasing_store(n));
7137 match(Set mem (StoreNKlass mem src));
7138
7139 ins_cost(INSN_COST);
7140 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7141
7142 ins_encode(aarch64_enc_strw(src, mem));
7143
7144 ins_pipe(istore_reg_mem);
7145 %}
7146
7147 // TODO
7148 // implement storeImmD0 and storeDImmPacked
7149
7150 // prefetch instructions
7151 // Must be safe to execute with invalid address (cannot fault).
7152
7153 instruct prefetchalloc( memory8 mem ) %{
7154 match(PrefetchAllocation mem);
7155
7156 ins_cost(INSN_COST);
7157 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7158
7159 ins_encode( aarch64_enc_prefetchw(mem) );
7160
7161 ins_pipe(iload_prefetch);
7162 %}
7163
7164 // ---------------- volatile loads and stores ----------------
7165
7166 // Load Byte (8 bit signed)
7167 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7168 %{
7169 match(Set dst (LoadB mem));
7170
7171 ins_cost(VOLATILE_REF_COST);
7172 format %{ "ldarsb $dst, $mem\t# byte" %}
7173
7174 ins_encode(aarch64_enc_ldarsb(dst, mem));
7175
7176 ins_pipe(pipe_serial);
7177 %}
7178
7179 // Load Byte (8 bit signed) into long
7180 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7181 %{
7182 match(Set dst (ConvI2L (LoadB mem)));
7183
7184 ins_cost(VOLATILE_REF_COST);
7185 format %{ "ldarsb $dst, $mem\t# byte" %}
7186
7187 ins_encode(aarch64_enc_ldarsb(dst, mem));
7188
7189 ins_pipe(pipe_serial);
7190 %}
7191
7192 // Load Byte (8 bit unsigned)
7193 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7194 %{
7195 match(Set dst (LoadUB mem));
7196
7197 ins_cost(VOLATILE_REF_COST);
7198 format %{ "ldarb $dst, $mem\t# byte" %}
7199
7200 ins_encode(aarch64_enc_ldarb(dst, mem));
7201
7202 ins_pipe(pipe_serial);
7203 %}
7204
7205 // Load Byte (8 bit unsigned) into long
7206 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7207 %{
7208 match(Set dst (ConvI2L (LoadUB mem)));
7209
7210 ins_cost(VOLATILE_REF_COST);
7211 format %{ "ldarb $dst, $mem\t# byte" %}
7212
7213 ins_encode(aarch64_enc_ldarb(dst, mem));
7214
7215 ins_pipe(pipe_serial);
7216 %}
7217
7218 // Load Short (16 bit signed)
7219 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7220 %{
7221 match(Set dst (LoadS mem));
7222
7223 ins_cost(VOLATILE_REF_COST);
7224 format %{ "ldarshw $dst, $mem\t# short" %}
7225
7226 ins_encode(aarch64_enc_ldarshw(dst, mem));
7227
7228 ins_pipe(pipe_serial);
7229 %}
7230
7231 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7232 %{
7233 match(Set dst (LoadUS mem));
7234
7235 ins_cost(VOLATILE_REF_COST);
7236 format %{ "ldarhw $dst, $mem\t# short" %}
7237
7238 ins_encode(aarch64_enc_ldarhw(dst, mem));
7239
7240 ins_pipe(pipe_serial);
7241 %}
7242
7243 // Load Short/Char (16 bit unsigned) into long
7244 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7245 %{
7246 match(Set dst (ConvI2L (LoadUS mem)));
7247
7248 ins_cost(VOLATILE_REF_COST);
7249 format %{ "ldarh $dst, $mem\t# short" %}
7250
7251 ins_encode(aarch64_enc_ldarh(dst, mem));
7252
7253 ins_pipe(pipe_serial);
7254 %}
7255
7256 // Load Short/Char (16 bit signed) into long
7257 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7258 %{
7259 match(Set dst (ConvI2L (LoadS mem)));
7260
7261 ins_cost(VOLATILE_REF_COST);
7262 format %{ "ldarh $dst, $mem\t# short" %}
7263
7264 ins_encode(aarch64_enc_ldarsh(dst, mem));
7265
7266 ins_pipe(pipe_serial);
7267 %}
7268
7269 // Load Integer (32 bit signed)
7270 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7271 %{
7272 match(Set dst (LoadI mem));
7273
7274 ins_cost(VOLATILE_REF_COST);
7275 format %{ "ldarw $dst, $mem\t# int" %}
7276
7277 ins_encode(aarch64_enc_ldarw(dst, mem));
7278
7279 ins_pipe(pipe_serial);
7280 %}
7281
7282 // Load Integer (32 bit unsigned) into long
7283 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7284 %{
7285 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7286
7287 ins_cost(VOLATILE_REF_COST);
7288 format %{ "ldarw $dst, $mem\t# int" %}
7289
7290 ins_encode(aarch64_enc_ldarw(dst, mem));
7291
7292 ins_pipe(pipe_serial);
7293 %}
7294
7295 // Load Long (64 bit signed)
7296 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7297 %{
7298 match(Set dst (LoadL mem));
7299
7300 ins_cost(VOLATILE_REF_COST);
7301 format %{ "ldar $dst, $mem\t# int" %}
7302
7303 ins_encode(aarch64_enc_ldar(dst, mem));
7304
7305 ins_pipe(pipe_serial);
7306 %}
7307
7308 // Load Pointer
7309 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7310 %{
7311 match(Set dst (LoadP mem));
7312 predicate(n->as_Load()->barrier_data() == 0);
7313
7314 ins_cost(VOLATILE_REF_COST);
7315 format %{ "ldar $dst, $mem\t# ptr" %}
7316
7317 ins_encode(aarch64_enc_ldar(dst, mem));
7318
7319 ins_pipe(pipe_serial);
7320 %}
7321
7322 // Load Compressed Pointer
7323 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7324 %{
7325 match(Set dst (LoadN mem));
7326 predicate(n->as_Load()->barrier_data() == 0);
7327
7328 ins_cost(VOLATILE_REF_COST);
7329 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7330
7331 ins_encode(aarch64_enc_ldarw(dst, mem));
7332
7333 ins_pipe(pipe_serial);
7334 %}
7335
7336 // Load Float
7337 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7338 %{
7339 match(Set dst (LoadF mem));
7340
7341 ins_cost(VOLATILE_REF_COST);
7342 format %{ "ldars $dst, $mem\t# float" %}
7343
7344 ins_encode( aarch64_enc_fldars(dst, mem) );
7345
7346 ins_pipe(pipe_serial);
7347 %}
7348
7349 // Load Double
7350 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7351 %{
7352 match(Set dst (LoadD mem));
7353
7354 ins_cost(VOLATILE_REF_COST);
7355 format %{ "ldard $dst, $mem\t# double" %}
7356
7357 ins_encode( aarch64_enc_fldard(dst, mem) );
7358
7359 ins_pipe(pipe_serial);
7360 %}
7361
7362 // Store Byte
7363 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7364 %{
7365 match(Set mem (StoreB mem src));
7366
7367 ins_cost(VOLATILE_REF_COST);
7368 format %{ "stlrb $src, $mem\t# byte" %}
7369
7370 ins_encode(aarch64_enc_stlrb(src, mem));
7371
7372 ins_pipe(pipe_class_memory);
7373 %}
7374
7375 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7376 %{
7377 match(Set mem (StoreB mem zero));
7378
7379 ins_cost(VOLATILE_REF_COST);
7380 format %{ "stlrb zr, $mem\t# byte" %}
7381
7382 ins_encode(aarch64_enc_stlrb0(mem));
7383
7384 ins_pipe(pipe_class_memory);
7385 %}
7386
7387 // Store Char/Short
7388 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7389 %{
7390 match(Set mem (StoreC mem src));
7391
7392 ins_cost(VOLATILE_REF_COST);
7393 format %{ "stlrh $src, $mem\t# short" %}
7394
7395 ins_encode(aarch64_enc_stlrh(src, mem));
7396
7397 ins_pipe(pipe_class_memory);
7398 %}
7399
7400 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7401 %{
7402 match(Set mem (StoreC mem zero));
7403
7404 ins_cost(VOLATILE_REF_COST);
7405 format %{ "stlrh zr, $mem\t# short" %}
7406
7407 ins_encode(aarch64_enc_stlrh0(mem));
7408
7409 ins_pipe(pipe_class_memory);
7410 %}
7411
7412 // Store Integer
7413
7414 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7415 %{
7416 match(Set mem(StoreI mem src));
7417
7418 ins_cost(VOLATILE_REF_COST);
7419 format %{ "stlrw $src, $mem\t# int" %}
7420
7421 ins_encode(aarch64_enc_stlrw(src, mem));
7422
7423 ins_pipe(pipe_class_memory);
7424 %}
7425
7426 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7427 %{
7428 match(Set mem(StoreI mem zero));
7429
7430 ins_cost(VOLATILE_REF_COST);
7431 format %{ "stlrw zr, $mem\t# int" %}
7432
7433 ins_encode(aarch64_enc_stlrw0(mem));
7434
7435 ins_pipe(pipe_class_memory);
7436 %}
7437
7438 // Store Long (64 bit signed)
7439 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7440 %{
7441 match(Set mem (StoreL mem src));
7442
7443 ins_cost(VOLATILE_REF_COST);
7444 format %{ "stlr $src, $mem\t# int" %}
7445
7446 ins_encode(aarch64_enc_stlr(src, mem));
7447
7448 ins_pipe(pipe_class_memory);
7449 %}
7450
7451 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7452 %{
7453 match(Set mem (StoreL mem zero));
7454
7455 ins_cost(VOLATILE_REF_COST);
7456 format %{ "stlr zr, $mem\t# int" %}
7457
7458 ins_encode(aarch64_enc_stlr0(mem));
7459
7460 ins_pipe(pipe_class_memory);
7461 %}
7462
7463 // Store Pointer
7464 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7465 %{
7466 match(Set mem (StoreP mem src));
7467 predicate(n->as_Store()->barrier_data() == 0);
7468
7469 ins_cost(VOLATILE_REF_COST);
7470 format %{ "stlr $src, $mem\t# ptr" %}
7471
7472 ins_encode(aarch64_enc_stlr(src, mem));
7473
7474 ins_pipe(pipe_class_memory);
7475 %}
7476
7477 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7478 %{
7479 match(Set mem (StoreP mem zero));
7480 predicate(n->as_Store()->barrier_data() == 0);
7481
7482 ins_cost(VOLATILE_REF_COST);
7483 format %{ "stlr zr, $mem\t# ptr" %}
7484
7485 ins_encode(aarch64_enc_stlr0(mem));
7486
7487 ins_pipe(pipe_class_memory);
7488 %}
7489
7490 // Store Compressed Pointer
7491 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7492 %{
7493 match(Set mem (StoreN mem src));
7494 predicate(n->as_Store()->barrier_data() == 0);
7495
7496 ins_cost(VOLATILE_REF_COST);
7497 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7498
7499 ins_encode(aarch64_enc_stlrw(src, mem));
7500
7501 ins_pipe(pipe_class_memory);
7502 %}
7503
7504 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7505 %{
7506 match(Set mem (StoreN mem zero));
7507 predicate(n->as_Store()->barrier_data() == 0);
7508
7509 ins_cost(VOLATILE_REF_COST);
7510 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7511
7512 ins_encode(aarch64_enc_stlrw0(mem));
7513
7514 ins_pipe(pipe_class_memory);
7515 %}
7516
7517 // Store Float
7518 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7519 %{
7520 match(Set mem (StoreF mem src));
7521
7522 ins_cost(VOLATILE_REF_COST);
7523 format %{ "stlrs $src, $mem\t# float" %}
7524
7525 ins_encode( aarch64_enc_fstlrs(src, mem) );
7526
7527 ins_pipe(pipe_class_memory);
7528 %}
7529
7530 // TODO
7531 // implement storeImmF0 and storeFImmPacked
7532
7533 // Store Double
7534 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7535 %{
7536 match(Set mem (StoreD mem src));
7537
7538 ins_cost(VOLATILE_REF_COST);
7539 format %{ "stlrd $src, $mem\t# double" %}
7540
7541 ins_encode( aarch64_enc_fstlrd(src, mem) );
7542
7543 ins_pipe(pipe_class_memory);
7544 %}
7545
7546 // ---------------- end of volatile loads and stores ----------------
7547
7548 instruct cacheWB(indirect addr)
7549 %{
7550 predicate(VM_Version::supports_data_cache_line_flush());
7551 match(CacheWB addr);
7552
7553 ins_cost(100);
7554 format %{"cache wb $addr" %}
7555 ins_encode %{
7556 assert($addr->index_position() < 0, "should be");
7557 assert($addr$$disp == 0, "should be");
7558 __ cache_wb(Address($addr$$base$$Register, 0));
7559 %}
7560 ins_pipe(pipe_slow); // XXX
7561 %}
7562
7563 instruct cacheWBPreSync()
7564 %{
7565 predicate(VM_Version::supports_data_cache_line_flush());
7566 match(CacheWBPreSync);
7567
7568 ins_cost(100);
7569 format %{"cache wb presync" %}
7570 ins_encode %{
7571 __ cache_wbsync(true);
7572 %}
7573 ins_pipe(pipe_slow); // XXX
7574 %}
7575
7576 instruct cacheWBPostSync()
7577 %{
7578 predicate(VM_Version::supports_data_cache_line_flush());
7579 match(CacheWBPostSync);
7580
7581 ins_cost(100);
7582 format %{"cache wb postsync" %}
7583 ins_encode %{
7584 __ cache_wbsync(false);
7585 %}
7586 ins_pipe(pipe_slow); // XXX
7587 %}
7588
7589 // ============================================================================
7590 // BSWAP Instructions
7591
7592 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7593 match(Set dst (ReverseBytesI src));
7594
7595 ins_cost(INSN_COST);
7596 format %{ "revw $dst, $src" %}
7597
7598 ins_encode %{
7599 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7600 %}
7601
7602 ins_pipe(ialu_reg);
7603 %}
7604
7605 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7606 match(Set dst (ReverseBytesL src));
7607
7608 ins_cost(INSN_COST);
7609 format %{ "rev $dst, $src" %}
7610
7611 ins_encode %{
7612 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7613 %}
7614
7615 ins_pipe(ialu_reg);
7616 %}
7617
7618 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7619 match(Set dst (ReverseBytesUS src));
7620
7621 ins_cost(INSN_COST);
7622 format %{ "rev16w $dst, $src" %}
7623
7624 ins_encode %{
7625 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7626 %}
7627
7628 ins_pipe(ialu_reg);
7629 %}
7630
7631 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7632 match(Set dst (ReverseBytesS src));
7633
7634 ins_cost(INSN_COST);
7635 format %{ "rev16w $dst, $src\n\t"
7636 "sbfmw $dst, $dst, #0, #15" %}
7637
7638 ins_encode %{
7639 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7640 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7641 %}
7642
7643 ins_pipe(ialu_reg);
7644 %}
7645
7646 // ============================================================================
7647 // Zero Count Instructions
7648
7649 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7650 match(Set dst (CountLeadingZerosI src));
7651
7652 ins_cost(INSN_COST);
7653 format %{ "clzw $dst, $src" %}
7654 ins_encode %{
7655 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7656 %}
7657
7658 ins_pipe(ialu_reg);
7659 %}
7660
7661 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7662 match(Set dst (CountLeadingZerosL src));
7663
7664 ins_cost(INSN_COST);
7665 format %{ "clz $dst, $src" %}
7666 ins_encode %{
7667 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7668 %}
7669
7670 ins_pipe(ialu_reg);
7671 %}
7672
7673 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7674 match(Set dst (CountTrailingZerosI src));
7675
7676 ins_cost(INSN_COST * 2);
7677 format %{ "rbitw $dst, $src\n\t"
7678 "clzw $dst, $dst" %}
7679 ins_encode %{
7680 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7681 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7682 %}
7683
7684 ins_pipe(ialu_reg);
7685 %}
7686
7687 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7688 match(Set dst (CountTrailingZerosL src));
7689
7690 ins_cost(INSN_COST * 2);
7691 format %{ "rbit $dst, $src\n\t"
7692 "clz $dst, $dst" %}
7693 ins_encode %{
7694 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7695 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7696 %}
7697
7698 ins_pipe(ialu_reg);
7699 %}
7700
7701 //---------- Population Count Instructions -------------------------------------
7702 //
7703
7704 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7705 match(Set dst (PopCountI src));
7706 effect(TEMP tmp);
7707 ins_cost(INSN_COST * 13);
7708
7709 format %{ "movw $src, $src\n\t"
7710 "mov $tmp, $src\t# vector (1D)\n\t"
7711 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7712 "addv $tmp, $tmp\t# vector (8B)\n\t"
7713 "mov $dst, $tmp\t# vector (1D)" %}
7714 ins_encode %{
7715 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7716 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7717 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7718 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7719 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7720 %}
7721
7722 ins_pipe(pipe_class_default);
7723 %}
7724
7725 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7726 match(Set dst (PopCountI (LoadI mem)));
7727 effect(TEMP tmp);
7728 ins_cost(INSN_COST * 13);
7729
7730 format %{ "ldrs $tmp, $mem\n\t"
7731 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7732 "addv $tmp, $tmp\t# vector (8B)\n\t"
7733 "mov $dst, $tmp\t# vector (1D)" %}
7734 ins_encode %{
7735 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7736 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7737 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7738 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7739 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7740 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7741 %}
7742
7743 ins_pipe(pipe_class_default);
7744 %}
7745
7746 // Note: Long.bitCount(long) returns an int.
7747 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7748 match(Set dst (PopCountL src));
7749 effect(TEMP tmp);
7750 ins_cost(INSN_COST * 13);
7751
7752 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7753 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7754 "addv $tmp, $tmp\t# vector (8B)\n\t"
7755 "mov $dst, $tmp\t# vector (1D)" %}
7756 ins_encode %{
7757 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7758 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7759 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7760 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7761 %}
7762
7763 ins_pipe(pipe_class_default);
7764 %}
7765
7766 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7767 match(Set dst (PopCountL (LoadL mem)));
7768 effect(TEMP tmp);
7769 ins_cost(INSN_COST * 13);
7770
7771 format %{ "ldrd $tmp, $mem\n\t"
7772 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7773 "addv $tmp, $tmp\t# vector (8B)\n\t"
7774 "mov $dst, $tmp\t# vector (1D)" %}
7775 ins_encode %{
7776 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7777 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7778 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7779 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7780 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7781 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7782 %}
7783
7784 ins_pipe(pipe_class_default);
7785 %}
7786
7787 // ============================================================================
7788 // VerifyVectorAlignment Instruction
7789
7790 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7791 match(Set addr (VerifyVectorAlignment addr mask));
7792 effect(KILL cr);
7793 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7794 ins_encode %{
7795 Label Lskip;
7796 // check if masked bits of addr are zero
7797 __ tst($addr$$Register, $mask$$constant);
7798 __ br(Assembler::EQ, Lskip);
7799 __ stop("verify_vector_alignment found a misaligned vector memory access");
7800 __ bind(Lskip);
7801 %}
7802 ins_pipe(pipe_slow);
7803 %}
7804
7805 // ============================================================================
7806 // MemBar Instruction
7807
7808 instruct load_fence() %{
7809 match(LoadFence);
7810 ins_cost(VOLATILE_REF_COST);
7811
7812 format %{ "load_fence" %}
7813
7814 ins_encode %{
7815 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7816 %}
7817 ins_pipe(pipe_serial);
7818 %}
7819
7820 instruct unnecessary_membar_acquire() %{
7821 predicate(unnecessary_acquire(n));
7822 match(MemBarAcquire);
7823 ins_cost(0);
7824
7825 format %{ "membar_acquire (elided)" %}
7826
7827 ins_encode %{
7828 __ block_comment("membar_acquire (elided)");
7829 %}
7830
7831 ins_pipe(pipe_class_empty);
7832 %}
7833
7834 instruct membar_acquire() %{
7835 match(MemBarAcquire);
7836 ins_cost(VOLATILE_REF_COST);
7837
7838 format %{ "membar_acquire\n\t"
7839 "dmb ishld" %}
7840
7841 ins_encode %{
7842 __ block_comment("membar_acquire");
7843 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7844 %}
7845
7846 ins_pipe(pipe_serial);
7847 %}
7848
7849
7850 instruct membar_acquire_lock() %{
7851 match(MemBarAcquireLock);
7852 ins_cost(VOLATILE_REF_COST);
7853
7854 format %{ "membar_acquire_lock (elided)" %}
7855
7856 ins_encode %{
7857 __ block_comment("membar_acquire_lock (elided)");
7858 %}
7859
7860 ins_pipe(pipe_serial);
7861 %}
7862
7863 instruct store_fence() %{
7864 match(StoreFence);
7865 ins_cost(VOLATILE_REF_COST);
7866
7867 format %{ "store_fence" %}
7868
7869 ins_encode %{
7870 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7871 %}
7872 ins_pipe(pipe_serial);
7873 %}
7874
7875 instruct unnecessary_membar_release() %{
7876 predicate(unnecessary_release(n));
7877 match(MemBarRelease);
7878 ins_cost(0);
7879
7880 format %{ "membar_release (elided)" %}
7881
7882 ins_encode %{
7883 __ block_comment("membar_release (elided)");
7884 %}
7885 ins_pipe(pipe_serial);
7886 %}
7887
7888 instruct membar_release() %{
7889 match(MemBarRelease);
7890 ins_cost(VOLATILE_REF_COST);
7891
7892 format %{ "membar_release\n\t"
7893 "dmb ishst\n\tdmb ishld" %}
7894
7895 ins_encode %{
7896 __ block_comment("membar_release");
7897 // These will be merged if AlwaysMergeDMB is enabled.
7898 __ membar(Assembler::StoreStore);
7899 __ membar(Assembler::LoadStore);
7900 %}
7901 ins_pipe(pipe_serial);
7902 %}
7903
7904 instruct membar_storestore() %{
7905 match(MemBarStoreStore);
7906 match(StoreStoreFence);
7907 ins_cost(VOLATILE_REF_COST);
7908
7909 format %{ "MEMBAR-store-store" %}
7910
7911 ins_encode %{
7912 __ membar(Assembler::StoreStore);
7913 %}
7914 ins_pipe(pipe_serial);
7915 %}
7916
7917 instruct membar_release_lock() %{
7918 match(MemBarReleaseLock);
7919 ins_cost(VOLATILE_REF_COST);
7920
7921 format %{ "membar_release_lock (elided)" %}
7922
7923 ins_encode %{
7924 __ block_comment("membar_release_lock (elided)");
7925 %}
7926
7927 ins_pipe(pipe_serial);
7928 %}
7929
7930 instruct unnecessary_membar_volatile() %{
7931 predicate(unnecessary_volatile(n));
7932 match(MemBarVolatile);
7933 ins_cost(0);
7934
7935 format %{ "membar_volatile (elided)" %}
7936
7937 ins_encode %{
7938 __ block_comment("membar_volatile (elided)");
7939 %}
7940
7941 ins_pipe(pipe_serial);
7942 %}
7943
7944 instruct membar_volatile() %{
7945 match(MemBarVolatile);
7946 ins_cost(VOLATILE_REF_COST*100);
7947
7948 format %{ "membar_volatile\n\t"
7949 "dmb ish"%}
7950
7951 ins_encode %{
7952 __ block_comment("membar_volatile");
7953 __ membar(Assembler::StoreLoad);
7954 %}
7955
7956 ins_pipe(pipe_serial);
7957 %}
7958
7959 // ============================================================================
7960 // Cast/Convert Instructions
7961
7962 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7963 match(Set dst (CastX2P src));
7964
7965 ins_cost(INSN_COST);
7966 format %{ "mov $dst, $src\t# long -> ptr" %}
7967
7968 ins_encode %{
7969 if ($dst$$reg != $src$$reg) {
7970 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7971 }
7972 %}
7973
7974 ins_pipe(ialu_reg);
7975 %}
7976
7977 instruct castN2X(iRegLNoSp dst, iRegN src) %{
7978 match(Set dst (CastP2X src));
7979
7980 ins_cost(INSN_COST);
7981 format %{ "mov $dst, $src\t# ptr -> long" %}
7982
7983 ins_encode %{
7984 if ($dst$$reg != $src$$reg) {
7985 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7986 }
7987 %}
7988
7989 ins_pipe(ialu_reg);
7990 %}
7991
7992 instruct castP2X(iRegLNoSp dst, iRegP src) %{
7993 match(Set dst (CastP2X src));
7994
7995 ins_cost(INSN_COST);
7996 format %{ "mov $dst, $src\t# ptr -> long" %}
7997
7998 ins_encode %{
7999 if ($dst$$reg != $src$$reg) {
8000 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8001 }
8002 %}
8003
8004 ins_pipe(ialu_reg);
8005 %}
8006
8007 // Convert oop into int for vectors alignment masking
8008 instruct convP2I(iRegINoSp dst, iRegP src) %{
8009 match(Set dst (ConvL2I (CastP2X src)));
8010
8011 ins_cost(INSN_COST);
8012 format %{ "movw $dst, $src\t# ptr -> int" %}
8013 ins_encode %{
8014 __ movw($dst$$Register, $src$$Register);
8015 %}
8016
8017 ins_pipe(ialu_reg);
8018 %}
8019
8020 // Convert compressed oop into int for vectors alignment masking
8021 // in case of 32bit oops (heap < 4Gb).
8022 instruct convN2I(iRegINoSp dst, iRegN src)
8023 %{
8024 predicate(CompressedOops::shift() == 0);
8025 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8026
8027 ins_cost(INSN_COST);
8028 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8029 ins_encode %{
8030 __ movw($dst$$Register, $src$$Register);
8031 %}
8032
8033 ins_pipe(ialu_reg);
8034 %}
8035
8036
8037 // Convert oop pointer into compressed form
8038 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8039 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8040 match(Set dst (EncodeP src));
8041 effect(KILL cr);
8042 ins_cost(INSN_COST * 3);
8043 format %{ "encode_heap_oop $dst, $src" %}
8044 ins_encode %{
8045 Register s = $src$$Register;
8046 Register d = $dst$$Register;
8047 __ encode_heap_oop(d, s);
8048 %}
8049 ins_pipe(ialu_reg);
8050 %}
8051
8052 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8053 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8054 match(Set dst (EncodeP src));
8055 ins_cost(INSN_COST * 3);
8056 format %{ "encode_heap_oop_not_null $dst, $src" %}
8057 ins_encode %{
8058 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8059 %}
8060 ins_pipe(ialu_reg);
8061 %}
8062
8063 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8064 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8065 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8066 match(Set dst (DecodeN src));
8067 ins_cost(INSN_COST * 3);
8068 format %{ "decode_heap_oop $dst, $src" %}
8069 ins_encode %{
8070 Register s = $src$$Register;
8071 Register d = $dst$$Register;
8072 __ decode_heap_oop(d, s);
8073 %}
8074 ins_pipe(ialu_reg);
8075 %}
8076
8077 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8078 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8079 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8080 match(Set dst (DecodeN src));
8081 ins_cost(INSN_COST * 3);
8082 format %{ "decode_heap_oop_not_null $dst, $src" %}
8083 ins_encode %{
8084 Register s = $src$$Register;
8085 Register d = $dst$$Register;
8086 __ decode_heap_oop_not_null(d, s);
8087 %}
8088 ins_pipe(ialu_reg);
8089 %}
8090
8091 // n.b. AArch64 implementations of encode_klass_not_null and
8092 // decode_klass_not_null do not modify the flags register so, unlike
8093 // Intel, we don't kill CR as a side effect here
8094
8095 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8096 match(Set dst (EncodePKlass src));
8097
8098 ins_cost(INSN_COST * 3);
8099 format %{ "encode_klass_not_null $dst,$src" %}
8100
8101 ins_encode %{
8102 Register src_reg = as_Register($src$$reg);
8103 Register dst_reg = as_Register($dst$$reg);
8104 __ encode_klass_not_null(dst_reg, src_reg);
8105 %}
8106
8107 ins_pipe(ialu_reg);
8108 %}
8109
8110 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8111 match(Set dst (DecodeNKlass src));
8112
8113 ins_cost(INSN_COST * 3);
8114 format %{ "decode_klass_not_null $dst,$src" %}
8115
8116 ins_encode %{
8117 Register src_reg = as_Register($src$$reg);
8118 Register dst_reg = as_Register($dst$$reg);
8119 if (dst_reg != src_reg) {
8120 __ decode_klass_not_null(dst_reg, src_reg);
8121 } else {
8122 __ decode_klass_not_null(dst_reg);
8123 }
8124 %}
8125
8126 ins_pipe(ialu_reg);
8127 %}
8128
8129 instruct checkCastPP(iRegPNoSp dst)
8130 %{
8131 match(Set dst (CheckCastPP dst));
8132
8133 size(0);
8134 format %{ "# checkcastPP of $dst" %}
8135 ins_encode(/* empty encoding */);
8136 ins_pipe(pipe_class_empty);
8137 %}
8138
8139 instruct castPP(iRegPNoSp dst)
8140 %{
8141 match(Set dst (CastPP dst));
8142
8143 size(0);
8144 format %{ "# castPP of $dst" %}
8145 ins_encode(/* empty encoding */);
8146 ins_pipe(pipe_class_empty);
8147 %}
8148
8149 instruct castII(iRegI dst)
8150 %{
8151 match(Set dst (CastII dst));
8152
8153 size(0);
8154 format %{ "# castII of $dst" %}
8155 ins_encode(/* empty encoding */);
8156 ins_cost(0);
8157 ins_pipe(pipe_class_empty);
8158 %}
8159
8160 instruct castLL(iRegL dst)
8161 %{
8162 match(Set dst (CastLL dst));
8163
8164 size(0);
8165 format %{ "# castLL of $dst" %}
8166 ins_encode(/* empty encoding */);
8167 ins_cost(0);
8168 ins_pipe(pipe_class_empty);
8169 %}
8170
8171 instruct castFF(vRegF dst)
8172 %{
8173 match(Set dst (CastFF dst));
8174
8175 size(0);
8176 format %{ "# castFF of $dst" %}
8177 ins_encode(/* empty encoding */);
8178 ins_cost(0);
8179 ins_pipe(pipe_class_empty);
8180 %}
8181
8182 instruct castDD(vRegD dst)
8183 %{
8184 match(Set dst (CastDD dst));
8185
8186 size(0);
8187 format %{ "# castDD of $dst" %}
8188 ins_encode(/* empty encoding */);
8189 ins_cost(0);
8190 ins_pipe(pipe_class_empty);
8191 %}
8192
8193 instruct castVV(vReg dst)
8194 %{
8195 match(Set dst (CastVV dst));
8196
8197 size(0);
8198 format %{ "# castVV of $dst" %}
8199 ins_encode(/* empty encoding */);
8200 ins_cost(0);
8201 ins_pipe(pipe_class_empty);
8202 %}
8203
8204 instruct castVVMask(pRegGov dst)
8205 %{
8206 match(Set dst (CastVV dst));
8207
8208 size(0);
8209 format %{ "# castVV of $dst" %}
8210 ins_encode(/* empty encoding */);
8211 ins_cost(0);
8212 ins_pipe(pipe_class_empty);
8213 %}
8214
8215 // ============================================================================
8216 // Atomic operation instructions
8217 //
8218
8219 // standard CompareAndSwapX when we are using barriers
8220 // these have higher priority than the rules selected by a predicate
8221
8222 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8223 // can't match them
8224
8225 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8226
8227 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8228 ins_cost(2 * VOLATILE_REF_COST);
8229
8230 effect(KILL cr);
8231
8232 format %{
8233 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8234 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8235 %}
8236
8237 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8238 aarch64_enc_cset_eq(res));
8239
8240 ins_pipe(pipe_slow);
8241 %}
8242
8243 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8244
8245 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8246 ins_cost(2 * VOLATILE_REF_COST);
8247
8248 effect(KILL cr);
8249
8250 format %{
8251 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8252 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8253 %}
8254
8255 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8256 aarch64_enc_cset_eq(res));
8257
8258 ins_pipe(pipe_slow);
8259 %}
8260
8261 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8262
8263 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8264 ins_cost(2 * VOLATILE_REF_COST);
8265
8266 effect(KILL cr);
8267
8268 format %{
8269 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8270 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8271 %}
8272
8273 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8274 aarch64_enc_cset_eq(res));
8275
8276 ins_pipe(pipe_slow);
8277 %}
8278
8279 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8280
8281 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8282 ins_cost(2 * VOLATILE_REF_COST);
8283
8284 effect(KILL cr);
8285
8286 format %{
8287 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8288 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8289 %}
8290
8291 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8292 aarch64_enc_cset_eq(res));
8293
8294 ins_pipe(pipe_slow);
8295 %}
8296
8297 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8298
8299 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8300 predicate(n->as_LoadStore()->barrier_data() == 0);
8301 ins_cost(2 * VOLATILE_REF_COST);
8302
8303 effect(KILL cr);
8304
8305 format %{
8306 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8307 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8308 %}
8309
8310 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8311 aarch64_enc_cset_eq(res));
8312
8313 ins_pipe(pipe_slow);
8314 %}
8315
8316 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8317
8318 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8319 predicate(n->as_LoadStore()->barrier_data() == 0);
8320 ins_cost(2 * VOLATILE_REF_COST);
8321
8322 effect(KILL cr);
8323
8324 format %{
8325 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8326 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8327 %}
8328
8329 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8330 aarch64_enc_cset_eq(res));
8331
8332 ins_pipe(pipe_slow);
8333 %}
8334
8335 // alternative CompareAndSwapX when we are eliding barriers
8336
8337 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8338
8339 predicate(needs_acquiring_load_exclusive(n));
8340 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8341 ins_cost(VOLATILE_REF_COST);
8342
8343 effect(KILL cr);
8344
8345 format %{
8346 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8347 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8348 %}
8349
8350 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8351 aarch64_enc_cset_eq(res));
8352
8353 ins_pipe(pipe_slow);
8354 %}
8355
8356 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8357
8358 predicate(needs_acquiring_load_exclusive(n));
8359 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8360 ins_cost(VOLATILE_REF_COST);
8361
8362 effect(KILL cr);
8363
8364 format %{
8365 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8366 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8367 %}
8368
8369 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8370 aarch64_enc_cset_eq(res));
8371
8372 ins_pipe(pipe_slow);
8373 %}
8374
8375 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8376
8377 predicate(needs_acquiring_load_exclusive(n));
8378 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8379 ins_cost(VOLATILE_REF_COST);
8380
8381 effect(KILL cr);
8382
8383 format %{
8384 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8385 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8386 %}
8387
8388 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8389 aarch64_enc_cset_eq(res));
8390
8391 ins_pipe(pipe_slow);
8392 %}
8393
8394 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8395
8396 predicate(needs_acquiring_load_exclusive(n));
8397 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8398 ins_cost(VOLATILE_REF_COST);
8399
8400 effect(KILL cr);
8401
8402 format %{
8403 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8404 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8405 %}
8406
8407 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8408 aarch64_enc_cset_eq(res));
8409
8410 ins_pipe(pipe_slow);
8411 %}
8412
8413 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8414
8415 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8416 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8417 ins_cost(VOLATILE_REF_COST);
8418
8419 effect(KILL cr);
8420
8421 format %{
8422 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8423 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8424 %}
8425
8426 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8427 aarch64_enc_cset_eq(res));
8428
8429 ins_pipe(pipe_slow);
8430 %}
8431
8432 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8433
8434 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8435 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8436 ins_cost(VOLATILE_REF_COST);
8437
8438 effect(KILL cr);
8439
8440 format %{
8441 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8442 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8443 %}
8444
8445 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8446 aarch64_enc_cset_eq(res));
8447
8448 ins_pipe(pipe_slow);
8449 %}
8450
8451
8452 // ---------------------------------------------------------------------
8453
8454 // BEGIN This section of the file is automatically generated. Do not edit --------------
8455
8456 // Sundry CAS operations. Note that release is always true,
8457 // regardless of the memory ordering of the CAS. This is because we
8458 // need the volatile case to be sequentially consistent but there is
8459 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8460 // can't check the type of memory ordering here, so we always emit a
8461 // STLXR.
8462
8463 // This section is generated from cas.m4
8464
8465
8466 // This pattern is generated automatically from cas.m4.
8467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8468 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8469 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8470 ins_cost(2 * VOLATILE_REF_COST);
8471 effect(TEMP_DEF res, KILL cr);
8472 format %{
8473 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8474 %}
8475 ins_encode %{
8476 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8477 Assembler::byte, /*acquire*/ false, /*release*/ true,
8478 /*weak*/ false, $res$$Register);
8479 __ sxtbw($res$$Register, $res$$Register);
8480 %}
8481 ins_pipe(pipe_slow);
8482 %}
8483
8484 // This pattern is generated automatically from cas.m4.
8485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8486 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8487 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8488 ins_cost(2 * VOLATILE_REF_COST);
8489 effect(TEMP_DEF res, KILL cr);
8490 format %{
8491 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8492 %}
8493 ins_encode %{
8494 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8495 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8496 /*weak*/ false, $res$$Register);
8497 __ sxthw($res$$Register, $res$$Register);
8498 %}
8499 ins_pipe(pipe_slow);
8500 %}
8501
8502 // This pattern is generated automatically from cas.m4.
8503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8504 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8505 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8506 ins_cost(2 * VOLATILE_REF_COST);
8507 effect(TEMP_DEF res, KILL cr);
8508 format %{
8509 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8510 %}
8511 ins_encode %{
8512 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8513 Assembler::word, /*acquire*/ false, /*release*/ true,
8514 /*weak*/ false, $res$$Register);
8515 %}
8516 ins_pipe(pipe_slow);
8517 %}
8518
8519 // This pattern is generated automatically from cas.m4.
8520 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8521 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8522 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8523 ins_cost(2 * VOLATILE_REF_COST);
8524 effect(TEMP_DEF res, KILL cr);
8525 format %{
8526 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8527 %}
8528 ins_encode %{
8529 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8530 Assembler::xword, /*acquire*/ false, /*release*/ true,
8531 /*weak*/ false, $res$$Register);
8532 %}
8533 ins_pipe(pipe_slow);
8534 %}
8535
8536 // This pattern is generated automatically from cas.m4.
8537 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8538 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8539 predicate(n->as_LoadStore()->barrier_data() == 0);
8540 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8541 ins_cost(2 * VOLATILE_REF_COST);
8542 effect(TEMP_DEF res, KILL cr);
8543 format %{
8544 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8545 %}
8546 ins_encode %{
8547 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8548 Assembler::word, /*acquire*/ false, /*release*/ true,
8549 /*weak*/ false, $res$$Register);
8550 %}
8551 ins_pipe(pipe_slow);
8552 %}
8553
8554 // This pattern is generated automatically from cas.m4.
8555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8556 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8557 predicate(n->as_LoadStore()->barrier_data() == 0);
8558 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8559 ins_cost(2 * VOLATILE_REF_COST);
8560 effect(TEMP_DEF res, KILL cr);
8561 format %{
8562 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8563 %}
8564 ins_encode %{
8565 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8566 Assembler::xword, /*acquire*/ false, /*release*/ true,
8567 /*weak*/ false, $res$$Register);
8568 %}
8569 ins_pipe(pipe_slow);
8570 %}
8571
8572 // This pattern is generated automatically from cas.m4.
8573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8574 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8575 predicate(needs_acquiring_load_exclusive(n));
8576 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8577 ins_cost(VOLATILE_REF_COST);
8578 effect(TEMP_DEF res, KILL cr);
8579 format %{
8580 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8581 %}
8582 ins_encode %{
8583 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8584 Assembler::byte, /*acquire*/ true, /*release*/ true,
8585 /*weak*/ false, $res$$Register);
8586 __ sxtbw($res$$Register, $res$$Register);
8587 %}
8588 ins_pipe(pipe_slow);
8589 %}
8590
8591 // This pattern is generated automatically from cas.m4.
8592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8593 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8594 predicate(needs_acquiring_load_exclusive(n));
8595 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8596 ins_cost(VOLATILE_REF_COST);
8597 effect(TEMP_DEF res, KILL cr);
8598 format %{
8599 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8600 %}
8601 ins_encode %{
8602 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8603 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8604 /*weak*/ false, $res$$Register);
8605 __ sxthw($res$$Register, $res$$Register);
8606 %}
8607 ins_pipe(pipe_slow);
8608 %}
8609
8610 // This pattern is generated automatically from cas.m4.
8611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8612 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8613 predicate(needs_acquiring_load_exclusive(n));
8614 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8615 ins_cost(VOLATILE_REF_COST);
8616 effect(TEMP_DEF res, KILL cr);
8617 format %{
8618 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8619 %}
8620 ins_encode %{
8621 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8622 Assembler::word, /*acquire*/ true, /*release*/ true,
8623 /*weak*/ false, $res$$Register);
8624 %}
8625 ins_pipe(pipe_slow);
8626 %}
8627
8628 // This pattern is generated automatically from cas.m4.
8629 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8630 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8631 predicate(needs_acquiring_load_exclusive(n));
8632 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8633 ins_cost(VOLATILE_REF_COST);
8634 effect(TEMP_DEF res, KILL cr);
8635 format %{
8636 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8637 %}
8638 ins_encode %{
8639 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8640 Assembler::xword, /*acquire*/ true, /*release*/ true,
8641 /*weak*/ false, $res$$Register);
8642 %}
8643 ins_pipe(pipe_slow);
8644 %}
8645
8646 // This pattern is generated automatically from cas.m4.
8647 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8648 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8649 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8650 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8651 ins_cost(VOLATILE_REF_COST);
8652 effect(TEMP_DEF res, KILL cr);
8653 format %{
8654 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8655 %}
8656 ins_encode %{
8657 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8658 Assembler::word, /*acquire*/ true, /*release*/ true,
8659 /*weak*/ false, $res$$Register);
8660 %}
8661 ins_pipe(pipe_slow);
8662 %}
8663
8664 // This pattern is generated automatically from cas.m4.
8665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8666 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8667 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8668 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8669 ins_cost(VOLATILE_REF_COST);
8670 effect(TEMP_DEF res, KILL cr);
8671 format %{
8672 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8673 %}
8674 ins_encode %{
8675 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8676 Assembler::xword, /*acquire*/ true, /*release*/ true,
8677 /*weak*/ false, $res$$Register);
8678 %}
8679 ins_pipe(pipe_slow);
8680 %}
8681
8682 // This pattern is generated automatically from cas.m4.
8683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8684 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8685 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8686 ins_cost(2 * VOLATILE_REF_COST);
8687 effect(KILL cr);
8688 format %{
8689 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8690 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8691 %}
8692 ins_encode %{
8693 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8694 Assembler::byte, /*acquire*/ false, /*release*/ true,
8695 /*weak*/ true, noreg);
8696 __ csetw($res$$Register, Assembler::EQ);
8697 %}
8698 ins_pipe(pipe_slow);
8699 %}
8700
8701 // This pattern is generated automatically from cas.m4.
8702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8703 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8704 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8705 ins_cost(2 * VOLATILE_REF_COST);
8706 effect(KILL cr);
8707 format %{
8708 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8709 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8710 %}
8711 ins_encode %{
8712 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8713 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8714 /*weak*/ true, noreg);
8715 __ csetw($res$$Register, Assembler::EQ);
8716 %}
8717 ins_pipe(pipe_slow);
8718 %}
8719
8720 // This pattern is generated automatically from cas.m4.
8721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8722 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8723 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8724 ins_cost(2 * VOLATILE_REF_COST);
8725 effect(KILL cr);
8726 format %{
8727 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8728 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8729 %}
8730 ins_encode %{
8731 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8732 Assembler::word, /*acquire*/ false, /*release*/ true,
8733 /*weak*/ true, noreg);
8734 __ csetw($res$$Register, Assembler::EQ);
8735 %}
8736 ins_pipe(pipe_slow);
8737 %}
8738
8739 // This pattern is generated automatically from cas.m4.
8740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8741 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8742 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8743 ins_cost(2 * VOLATILE_REF_COST);
8744 effect(KILL cr);
8745 format %{
8746 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8747 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8748 %}
8749 ins_encode %{
8750 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8751 Assembler::xword, /*acquire*/ false, /*release*/ true,
8752 /*weak*/ true, noreg);
8753 __ csetw($res$$Register, Assembler::EQ);
8754 %}
8755 ins_pipe(pipe_slow);
8756 %}
8757
8758 // This pattern is generated automatically from cas.m4.
8759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8760 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8761 predicate(n->as_LoadStore()->barrier_data() == 0);
8762 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8763 ins_cost(2 * VOLATILE_REF_COST);
8764 effect(KILL cr);
8765 format %{
8766 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8767 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8768 %}
8769 ins_encode %{
8770 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8771 Assembler::word, /*acquire*/ false, /*release*/ true,
8772 /*weak*/ true, noreg);
8773 __ csetw($res$$Register, Assembler::EQ);
8774 %}
8775 ins_pipe(pipe_slow);
8776 %}
8777
8778 // This pattern is generated automatically from cas.m4.
8779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8780 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8781 predicate(n->as_LoadStore()->barrier_data() == 0);
8782 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8783 ins_cost(2 * VOLATILE_REF_COST);
8784 effect(KILL cr);
8785 format %{
8786 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8787 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8788 %}
8789 ins_encode %{
8790 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8791 Assembler::xword, /*acquire*/ false, /*release*/ true,
8792 /*weak*/ true, noreg);
8793 __ csetw($res$$Register, Assembler::EQ);
8794 %}
8795 ins_pipe(pipe_slow);
8796 %}
8797
8798 // This pattern is generated automatically from cas.m4.
8799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8800 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8801 predicate(needs_acquiring_load_exclusive(n));
8802 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8803 ins_cost(VOLATILE_REF_COST);
8804 effect(KILL cr);
8805 format %{
8806 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8807 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8808 %}
8809 ins_encode %{
8810 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8811 Assembler::byte, /*acquire*/ true, /*release*/ true,
8812 /*weak*/ true, noreg);
8813 __ csetw($res$$Register, Assembler::EQ);
8814 %}
8815 ins_pipe(pipe_slow);
8816 %}
8817
8818 // This pattern is generated automatically from cas.m4.
8819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8820 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8821 predicate(needs_acquiring_load_exclusive(n));
8822 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8823 ins_cost(VOLATILE_REF_COST);
8824 effect(KILL cr);
8825 format %{
8826 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8827 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8828 %}
8829 ins_encode %{
8830 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8831 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8832 /*weak*/ true, noreg);
8833 __ csetw($res$$Register, Assembler::EQ);
8834 %}
8835 ins_pipe(pipe_slow);
8836 %}
8837
8838 // This pattern is generated automatically from cas.m4.
8839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8840 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8841 predicate(needs_acquiring_load_exclusive(n));
8842 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8843 ins_cost(VOLATILE_REF_COST);
8844 effect(KILL cr);
8845 format %{
8846 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8847 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8848 %}
8849 ins_encode %{
8850 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8851 Assembler::word, /*acquire*/ true, /*release*/ true,
8852 /*weak*/ true, noreg);
8853 __ csetw($res$$Register, Assembler::EQ);
8854 %}
8855 ins_pipe(pipe_slow);
8856 %}
8857
8858 // This pattern is generated automatically from cas.m4.
8859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8860 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8861 predicate(needs_acquiring_load_exclusive(n));
8862 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8863 ins_cost(VOLATILE_REF_COST);
8864 effect(KILL cr);
8865 format %{
8866 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8867 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8868 %}
8869 ins_encode %{
8870 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8871 Assembler::xword, /*acquire*/ true, /*release*/ true,
8872 /*weak*/ true, noreg);
8873 __ csetw($res$$Register, Assembler::EQ);
8874 %}
8875 ins_pipe(pipe_slow);
8876 %}
8877
8878 // This pattern is generated automatically from cas.m4.
8879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8880 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8881 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8882 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8883 ins_cost(VOLATILE_REF_COST);
8884 effect(KILL cr);
8885 format %{
8886 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8887 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8888 %}
8889 ins_encode %{
8890 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8891 Assembler::word, /*acquire*/ true, /*release*/ true,
8892 /*weak*/ true, noreg);
8893 __ csetw($res$$Register, Assembler::EQ);
8894 %}
8895 ins_pipe(pipe_slow);
8896 %}
8897
8898 // This pattern is generated automatically from cas.m4.
8899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8900 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8901 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8902 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8903 ins_cost(VOLATILE_REF_COST);
8904 effect(KILL cr);
8905 format %{
8906 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8907 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8908 %}
8909 ins_encode %{
8910 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8911 Assembler::xword, /*acquire*/ true, /*release*/ true,
8912 /*weak*/ true, noreg);
8913 __ csetw($res$$Register, Assembler::EQ);
8914 %}
8915 ins_pipe(pipe_slow);
8916 %}
8917
8918 // END This section of the file is automatically generated. Do not edit --------------
8919 // ---------------------------------------------------------------------
8920
8921 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8922 match(Set prev (GetAndSetI mem newv));
8923 ins_cost(2 * VOLATILE_REF_COST);
8924 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8925 ins_encode %{
8926 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8927 %}
8928 ins_pipe(pipe_serial);
8929 %}
8930
8931 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8932 match(Set prev (GetAndSetL mem newv));
8933 ins_cost(2 * VOLATILE_REF_COST);
8934 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8935 ins_encode %{
8936 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8937 %}
8938 ins_pipe(pipe_serial);
8939 %}
8940
8941 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8942 predicate(n->as_LoadStore()->barrier_data() == 0);
8943 match(Set prev (GetAndSetN mem newv));
8944 ins_cost(2 * VOLATILE_REF_COST);
8945 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8946 ins_encode %{
8947 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8948 %}
8949 ins_pipe(pipe_serial);
8950 %}
8951
8952 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
8953 predicate(n->as_LoadStore()->barrier_data() == 0);
8954 match(Set prev (GetAndSetP mem newv));
8955 ins_cost(2 * VOLATILE_REF_COST);
8956 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8957 ins_encode %{
8958 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8959 %}
8960 ins_pipe(pipe_serial);
8961 %}
8962
8963 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
8964 predicate(needs_acquiring_load_exclusive(n));
8965 match(Set prev (GetAndSetI mem newv));
8966 ins_cost(VOLATILE_REF_COST);
8967 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8968 ins_encode %{
8969 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8970 %}
8971 ins_pipe(pipe_serial);
8972 %}
8973
8974 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
8975 predicate(needs_acquiring_load_exclusive(n));
8976 match(Set prev (GetAndSetL mem newv));
8977 ins_cost(VOLATILE_REF_COST);
8978 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8979 ins_encode %{
8980 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8981 %}
8982 ins_pipe(pipe_serial);
8983 %}
8984
8985 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
8986 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8987 match(Set prev (GetAndSetN mem newv));
8988 ins_cost(VOLATILE_REF_COST);
8989 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8990 ins_encode %{
8991 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8992 %}
8993 ins_pipe(pipe_serial);
8994 %}
8995
8996 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
8997 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8998 match(Set prev (GetAndSetP mem newv));
8999 ins_cost(VOLATILE_REF_COST);
9000 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9001 ins_encode %{
9002 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9003 %}
9004 ins_pipe(pipe_serial);
9005 %}
9006
9007
9008 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9009 match(Set newval (GetAndAddL mem incr));
9010 ins_cost(2 * VOLATILE_REF_COST + 1);
9011 format %{ "get_and_addL $newval, [$mem], $incr" %}
9012 ins_encode %{
9013 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9014 %}
9015 ins_pipe(pipe_serial);
9016 %}
9017
9018 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9019 predicate(n->as_LoadStore()->result_not_used());
9020 match(Set dummy (GetAndAddL mem incr));
9021 ins_cost(2 * VOLATILE_REF_COST);
9022 format %{ "get_and_addL [$mem], $incr" %}
9023 ins_encode %{
9024 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9025 %}
9026 ins_pipe(pipe_serial);
9027 %}
9028
9029 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9030 match(Set newval (GetAndAddL mem incr));
9031 ins_cost(2 * VOLATILE_REF_COST + 1);
9032 format %{ "get_and_addL $newval, [$mem], $incr" %}
9033 ins_encode %{
9034 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9035 %}
9036 ins_pipe(pipe_serial);
9037 %}
9038
9039 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9040 predicate(n->as_LoadStore()->result_not_used());
9041 match(Set dummy (GetAndAddL mem incr));
9042 ins_cost(2 * VOLATILE_REF_COST);
9043 format %{ "get_and_addL [$mem], $incr" %}
9044 ins_encode %{
9045 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9046 %}
9047 ins_pipe(pipe_serial);
9048 %}
9049
9050 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9051 match(Set newval (GetAndAddI mem incr));
9052 ins_cost(2 * VOLATILE_REF_COST + 1);
9053 format %{ "get_and_addI $newval, [$mem], $incr" %}
9054 ins_encode %{
9055 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9056 %}
9057 ins_pipe(pipe_serial);
9058 %}
9059
9060 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9061 predicate(n->as_LoadStore()->result_not_used());
9062 match(Set dummy (GetAndAddI mem incr));
9063 ins_cost(2 * VOLATILE_REF_COST);
9064 format %{ "get_and_addI [$mem], $incr" %}
9065 ins_encode %{
9066 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9067 %}
9068 ins_pipe(pipe_serial);
9069 %}
9070
9071 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9072 match(Set newval (GetAndAddI mem incr));
9073 ins_cost(2 * VOLATILE_REF_COST + 1);
9074 format %{ "get_and_addI $newval, [$mem], $incr" %}
9075 ins_encode %{
9076 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9077 %}
9078 ins_pipe(pipe_serial);
9079 %}
9080
9081 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9082 predicate(n->as_LoadStore()->result_not_used());
9083 match(Set dummy (GetAndAddI mem incr));
9084 ins_cost(2 * VOLATILE_REF_COST);
9085 format %{ "get_and_addI [$mem], $incr" %}
9086 ins_encode %{
9087 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9088 %}
9089 ins_pipe(pipe_serial);
9090 %}
9091
9092 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9093 predicate(needs_acquiring_load_exclusive(n));
9094 match(Set newval (GetAndAddL mem incr));
9095 ins_cost(VOLATILE_REF_COST + 1);
9096 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9097 ins_encode %{
9098 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9099 %}
9100 ins_pipe(pipe_serial);
9101 %}
9102
9103 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9104 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9105 match(Set dummy (GetAndAddL mem incr));
9106 ins_cost(VOLATILE_REF_COST);
9107 format %{ "get_and_addL_acq [$mem], $incr" %}
9108 ins_encode %{
9109 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9110 %}
9111 ins_pipe(pipe_serial);
9112 %}
9113
9114 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9115 predicate(needs_acquiring_load_exclusive(n));
9116 match(Set newval (GetAndAddL mem incr));
9117 ins_cost(VOLATILE_REF_COST + 1);
9118 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9119 ins_encode %{
9120 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9121 %}
9122 ins_pipe(pipe_serial);
9123 %}
9124
9125 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9126 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9127 match(Set dummy (GetAndAddL mem incr));
9128 ins_cost(VOLATILE_REF_COST);
9129 format %{ "get_and_addL_acq [$mem], $incr" %}
9130 ins_encode %{
9131 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9132 %}
9133 ins_pipe(pipe_serial);
9134 %}
9135
9136 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9137 predicate(needs_acquiring_load_exclusive(n));
9138 match(Set newval (GetAndAddI mem incr));
9139 ins_cost(VOLATILE_REF_COST + 1);
9140 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9141 ins_encode %{
9142 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9143 %}
9144 ins_pipe(pipe_serial);
9145 %}
9146
9147 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9148 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9149 match(Set dummy (GetAndAddI mem incr));
9150 ins_cost(VOLATILE_REF_COST);
9151 format %{ "get_and_addI_acq [$mem], $incr" %}
9152 ins_encode %{
9153 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9154 %}
9155 ins_pipe(pipe_serial);
9156 %}
9157
9158 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9159 predicate(needs_acquiring_load_exclusive(n));
9160 match(Set newval (GetAndAddI mem incr));
9161 ins_cost(VOLATILE_REF_COST + 1);
9162 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9163 ins_encode %{
9164 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9165 %}
9166 ins_pipe(pipe_serial);
9167 %}
9168
9169 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9170 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9171 match(Set dummy (GetAndAddI mem incr));
9172 ins_cost(VOLATILE_REF_COST);
9173 format %{ "get_and_addI_acq [$mem], $incr" %}
9174 ins_encode %{
9175 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9176 %}
9177 ins_pipe(pipe_serial);
9178 %}
9179
9180 // Manifest a CmpU result in an integer register.
9181 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9182 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9183 %{
9184 match(Set dst (CmpU3 src1 src2));
9185 effect(KILL flags);
9186
9187 ins_cost(INSN_COST * 3);
9188 format %{
9189 "cmpw $src1, $src2\n\t"
9190 "csetw $dst, ne\n\t"
9191 "cnegw $dst, lo\t# CmpU3(reg)"
9192 %}
9193 ins_encode %{
9194 __ cmpw($src1$$Register, $src2$$Register);
9195 __ csetw($dst$$Register, Assembler::NE);
9196 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9197 %}
9198
9199 ins_pipe(pipe_class_default);
9200 %}
9201
9202 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9203 %{
9204 match(Set dst (CmpU3 src1 src2));
9205 effect(KILL flags);
9206
9207 ins_cost(INSN_COST * 3);
9208 format %{
9209 "subsw zr, $src1, $src2\n\t"
9210 "csetw $dst, ne\n\t"
9211 "cnegw $dst, lo\t# CmpU3(imm)"
9212 %}
9213 ins_encode %{
9214 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9215 __ csetw($dst$$Register, Assembler::NE);
9216 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9217 %}
9218
9219 ins_pipe(pipe_class_default);
9220 %}
9221
9222 // Manifest a CmpUL result in an integer register.
9223 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9224 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9225 %{
9226 match(Set dst (CmpUL3 src1 src2));
9227 effect(KILL flags);
9228
9229 ins_cost(INSN_COST * 3);
9230 format %{
9231 "cmp $src1, $src2\n\t"
9232 "csetw $dst, ne\n\t"
9233 "cnegw $dst, lo\t# CmpUL3(reg)"
9234 %}
9235 ins_encode %{
9236 __ cmp($src1$$Register, $src2$$Register);
9237 __ csetw($dst$$Register, Assembler::NE);
9238 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9239 %}
9240
9241 ins_pipe(pipe_class_default);
9242 %}
9243
9244 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9245 %{
9246 match(Set dst (CmpUL3 src1 src2));
9247 effect(KILL flags);
9248
9249 ins_cost(INSN_COST * 3);
9250 format %{
9251 "subs zr, $src1, $src2\n\t"
9252 "csetw $dst, ne\n\t"
9253 "cnegw $dst, lo\t# CmpUL3(imm)"
9254 %}
9255 ins_encode %{
9256 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9257 __ csetw($dst$$Register, Assembler::NE);
9258 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9259 %}
9260
9261 ins_pipe(pipe_class_default);
9262 %}
9263
9264 // Manifest a CmpL result in an integer register.
9265 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9266 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9267 %{
9268 match(Set dst (CmpL3 src1 src2));
9269 effect(KILL flags);
9270
9271 ins_cost(INSN_COST * 3);
9272 format %{
9273 "cmp $src1, $src2\n\t"
9274 "csetw $dst, ne\n\t"
9275 "cnegw $dst, lt\t# CmpL3(reg)"
9276 %}
9277 ins_encode %{
9278 __ cmp($src1$$Register, $src2$$Register);
9279 __ csetw($dst$$Register, Assembler::NE);
9280 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9281 %}
9282
9283 ins_pipe(pipe_class_default);
9284 %}
9285
9286 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9287 %{
9288 match(Set dst (CmpL3 src1 src2));
9289 effect(KILL flags);
9290
9291 ins_cost(INSN_COST * 3);
9292 format %{
9293 "subs zr, $src1, $src2\n\t"
9294 "csetw $dst, ne\n\t"
9295 "cnegw $dst, lt\t# CmpL3(imm)"
9296 %}
9297 ins_encode %{
9298 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9299 __ csetw($dst$$Register, Assembler::NE);
9300 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9301 %}
9302
9303 ins_pipe(pipe_class_default);
9304 %}
9305
9306 // ============================================================================
9307 // Conditional Move Instructions
9308
9309 // n.b. we have identical rules for both a signed compare op (cmpOp)
9310 // and an unsigned compare op (cmpOpU). it would be nice if we could
9311 // define an op class which merged both inputs and use it to type the
9312 // argument to a single rule. unfortunatelyt his fails because the
9313 // opclass does not live up to the COND_INTER interface of its
9314 // component operands. When the generic code tries to negate the
9315 // operand it ends up running the generci Machoper::negate method
9316 // which throws a ShouldNotHappen. So, we have to provide two flavours
9317 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9318
9319 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9320 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9321
9322 ins_cost(INSN_COST * 2);
9323 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9324
9325 ins_encode %{
9326 __ cselw(as_Register($dst$$reg),
9327 as_Register($src2$$reg),
9328 as_Register($src1$$reg),
9329 (Assembler::Condition)$cmp$$cmpcode);
9330 %}
9331
9332 ins_pipe(icond_reg_reg);
9333 %}
9334
9335 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9336 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9337
9338 ins_cost(INSN_COST * 2);
9339 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9340
9341 ins_encode %{
9342 __ cselw(as_Register($dst$$reg),
9343 as_Register($src2$$reg),
9344 as_Register($src1$$reg),
9345 (Assembler::Condition)$cmp$$cmpcode);
9346 %}
9347
9348 ins_pipe(icond_reg_reg);
9349 %}
9350
9351 // special cases where one arg is zero
9352
9353 // n.b. this is selected in preference to the rule above because it
9354 // avoids loading constant 0 into a source register
9355
9356 // TODO
9357 // we ought only to be able to cull one of these variants as the ideal
9358 // transforms ought always to order the zero consistently (to left/right?)
9359
9360 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9361 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9362
9363 ins_cost(INSN_COST * 2);
9364 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9365
9366 ins_encode %{
9367 __ cselw(as_Register($dst$$reg),
9368 as_Register($src$$reg),
9369 zr,
9370 (Assembler::Condition)$cmp$$cmpcode);
9371 %}
9372
9373 ins_pipe(icond_reg);
9374 %}
9375
9376 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9377 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9378
9379 ins_cost(INSN_COST * 2);
9380 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9381
9382 ins_encode %{
9383 __ cselw(as_Register($dst$$reg),
9384 as_Register($src$$reg),
9385 zr,
9386 (Assembler::Condition)$cmp$$cmpcode);
9387 %}
9388
9389 ins_pipe(icond_reg);
9390 %}
9391
9392 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9393 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9394
9395 ins_cost(INSN_COST * 2);
9396 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9397
9398 ins_encode %{
9399 __ cselw(as_Register($dst$$reg),
9400 zr,
9401 as_Register($src$$reg),
9402 (Assembler::Condition)$cmp$$cmpcode);
9403 %}
9404
9405 ins_pipe(icond_reg);
9406 %}
9407
9408 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9409 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9410
9411 ins_cost(INSN_COST * 2);
9412 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9413
9414 ins_encode %{
9415 __ cselw(as_Register($dst$$reg),
9416 zr,
9417 as_Register($src$$reg),
9418 (Assembler::Condition)$cmp$$cmpcode);
9419 %}
9420
9421 ins_pipe(icond_reg);
9422 %}
9423
9424 // special case for creating a boolean 0 or 1
9425
9426 // n.b. this is selected in preference to the rule above because it
9427 // avoids loading constants 0 and 1 into a source register
9428
9429 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9430 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9431
9432 ins_cost(INSN_COST * 2);
9433 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9434
9435 ins_encode %{
9436 // equivalently
9437 // cset(as_Register($dst$$reg),
9438 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9439 __ csincw(as_Register($dst$$reg),
9440 zr,
9441 zr,
9442 (Assembler::Condition)$cmp$$cmpcode);
9443 %}
9444
9445 ins_pipe(icond_none);
9446 %}
9447
9448 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9449 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9450
9451 ins_cost(INSN_COST * 2);
9452 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9453
9454 ins_encode %{
9455 // equivalently
9456 // cset(as_Register($dst$$reg),
9457 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9458 __ csincw(as_Register($dst$$reg),
9459 zr,
9460 zr,
9461 (Assembler::Condition)$cmp$$cmpcode);
9462 %}
9463
9464 ins_pipe(icond_none);
9465 %}
9466
9467 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9468 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9469
9470 ins_cost(INSN_COST * 2);
9471 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9472
9473 ins_encode %{
9474 __ csel(as_Register($dst$$reg),
9475 as_Register($src2$$reg),
9476 as_Register($src1$$reg),
9477 (Assembler::Condition)$cmp$$cmpcode);
9478 %}
9479
9480 ins_pipe(icond_reg_reg);
9481 %}
9482
9483 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9484 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9485
9486 ins_cost(INSN_COST * 2);
9487 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9488
9489 ins_encode %{
9490 __ csel(as_Register($dst$$reg),
9491 as_Register($src2$$reg),
9492 as_Register($src1$$reg),
9493 (Assembler::Condition)$cmp$$cmpcode);
9494 %}
9495
9496 ins_pipe(icond_reg_reg);
9497 %}
9498
9499 // special cases where one arg is zero
9500
9501 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9502 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9503
9504 ins_cost(INSN_COST * 2);
9505 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9506
9507 ins_encode %{
9508 __ csel(as_Register($dst$$reg),
9509 zr,
9510 as_Register($src$$reg),
9511 (Assembler::Condition)$cmp$$cmpcode);
9512 %}
9513
9514 ins_pipe(icond_reg);
9515 %}
9516
9517 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9518 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9519
9520 ins_cost(INSN_COST * 2);
9521 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9522
9523 ins_encode %{
9524 __ csel(as_Register($dst$$reg),
9525 zr,
9526 as_Register($src$$reg),
9527 (Assembler::Condition)$cmp$$cmpcode);
9528 %}
9529
9530 ins_pipe(icond_reg);
9531 %}
9532
9533 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9534 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9535
9536 ins_cost(INSN_COST * 2);
9537 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9538
9539 ins_encode %{
9540 __ csel(as_Register($dst$$reg),
9541 as_Register($src$$reg),
9542 zr,
9543 (Assembler::Condition)$cmp$$cmpcode);
9544 %}
9545
9546 ins_pipe(icond_reg);
9547 %}
9548
9549 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9550 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9551
9552 ins_cost(INSN_COST * 2);
9553 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9554
9555 ins_encode %{
9556 __ csel(as_Register($dst$$reg),
9557 as_Register($src$$reg),
9558 zr,
9559 (Assembler::Condition)$cmp$$cmpcode);
9560 %}
9561
9562 ins_pipe(icond_reg);
9563 %}
9564
9565 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9566 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9567
9568 ins_cost(INSN_COST * 2);
9569 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9570
9571 ins_encode %{
9572 __ csel(as_Register($dst$$reg),
9573 as_Register($src2$$reg),
9574 as_Register($src1$$reg),
9575 (Assembler::Condition)$cmp$$cmpcode);
9576 %}
9577
9578 ins_pipe(icond_reg_reg);
9579 %}
9580
9581 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9582 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9583
9584 ins_cost(INSN_COST * 2);
9585 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9586
9587 ins_encode %{
9588 __ csel(as_Register($dst$$reg),
9589 as_Register($src2$$reg),
9590 as_Register($src1$$reg),
9591 (Assembler::Condition)$cmp$$cmpcode);
9592 %}
9593
9594 ins_pipe(icond_reg_reg);
9595 %}
9596
9597 // special cases where one arg is zero
9598
9599 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9600 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9601
9602 ins_cost(INSN_COST * 2);
9603 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9604
9605 ins_encode %{
9606 __ csel(as_Register($dst$$reg),
9607 zr,
9608 as_Register($src$$reg),
9609 (Assembler::Condition)$cmp$$cmpcode);
9610 %}
9611
9612 ins_pipe(icond_reg);
9613 %}
9614
9615 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9616 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9617
9618 ins_cost(INSN_COST * 2);
9619 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9620
9621 ins_encode %{
9622 __ csel(as_Register($dst$$reg),
9623 zr,
9624 as_Register($src$$reg),
9625 (Assembler::Condition)$cmp$$cmpcode);
9626 %}
9627
9628 ins_pipe(icond_reg);
9629 %}
9630
9631 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9632 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9633
9634 ins_cost(INSN_COST * 2);
9635 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9636
9637 ins_encode %{
9638 __ csel(as_Register($dst$$reg),
9639 as_Register($src$$reg),
9640 zr,
9641 (Assembler::Condition)$cmp$$cmpcode);
9642 %}
9643
9644 ins_pipe(icond_reg);
9645 %}
9646
9647 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9648 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9649
9650 ins_cost(INSN_COST * 2);
9651 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9652
9653 ins_encode %{
9654 __ csel(as_Register($dst$$reg),
9655 as_Register($src$$reg),
9656 zr,
9657 (Assembler::Condition)$cmp$$cmpcode);
9658 %}
9659
9660 ins_pipe(icond_reg);
9661 %}
9662
9663 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9664 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9665
9666 ins_cost(INSN_COST * 2);
9667 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9668
9669 ins_encode %{
9670 __ cselw(as_Register($dst$$reg),
9671 as_Register($src2$$reg),
9672 as_Register($src1$$reg),
9673 (Assembler::Condition)$cmp$$cmpcode);
9674 %}
9675
9676 ins_pipe(icond_reg_reg);
9677 %}
9678
9679 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9680 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9681
9682 ins_cost(INSN_COST * 2);
9683 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9684
9685 ins_encode %{
9686 __ cselw(as_Register($dst$$reg),
9687 as_Register($src2$$reg),
9688 as_Register($src1$$reg),
9689 (Assembler::Condition)$cmp$$cmpcode);
9690 %}
9691
9692 ins_pipe(icond_reg_reg);
9693 %}
9694
9695 // special cases where one arg is zero
9696
9697 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9698 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9699
9700 ins_cost(INSN_COST * 2);
9701 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9702
9703 ins_encode %{
9704 __ cselw(as_Register($dst$$reg),
9705 zr,
9706 as_Register($src$$reg),
9707 (Assembler::Condition)$cmp$$cmpcode);
9708 %}
9709
9710 ins_pipe(icond_reg);
9711 %}
9712
9713 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9714 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9715
9716 ins_cost(INSN_COST * 2);
9717 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9718
9719 ins_encode %{
9720 __ cselw(as_Register($dst$$reg),
9721 zr,
9722 as_Register($src$$reg),
9723 (Assembler::Condition)$cmp$$cmpcode);
9724 %}
9725
9726 ins_pipe(icond_reg);
9727 %}
9728
9729 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9730 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9731
9732 ins_cost(INSN_COST * 2);
9733 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9734
9735 ins_encode %{
9736 __ cselw(as_Register($dst$$reg),
9737 as_Register($src$$reg),
9738 zr,
9739 (Assembler::Condition)$cmp$$cmpcode);
9740 %}
9741
9742 ins_pipe(icond_reg);
9743 %}
9744
9745 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9746 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9747
9748 ins_cost(INSN_COST * 2);
9749 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9750
9751 ins_encode %{
9752 __ cselw(as_Register($dst$$reg),
9753 as_Register($src$$reg),
9754 zr,
9755 (Assembler::Condition)$cmp$$cmpcode);
9756 %}
9757
9758 ins_pipe(icond_reg);
9759 %}
9760
9761 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9762 %{
9763 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9764
9765 ins_cost(INSN_COST * 3);
9766
9767 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9768 ins_encode %{
9769 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9770 __ fcsels(as_FloatRegister($dst$$reg),
9771 as_FloatRegister($src2$$reg),
9772 as_FloatRegister($src1$$reg),
9773 cond);
9774 %}
9775
9776 ins_pipe(fp_cond_reg_reg_s);
9777 %}
9778
9779 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9780 %{
9781 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9782
9783 ins_cost(INSN_COST * 3);
9784
9785 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9786 ins_encode %{
9787 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9788 __ fcsels(as_FloatRegister($dst$$reg),
9789 as_FloatRegister($src2$$reg),
9790 as_FloatRegister($src1$$reg),
9791 cond);
9792 %}
9793
9794 ins_pipe(fp_cond_reg_reg_s);
9795 %}
9796
9797 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9798 %{
9799 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9800
9801 ins_cost(INSN_COST * 3);
9802
9803 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9804 ins_encode %{
9805 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9806 __ fcseld(as_FloatRegister($dst$$reg),
9807 as_FloatRegister($src2$$reg),
9808 as_FloatRegister($src1$$reg),
9809 cond);
9810 %}
9811
9812 ins_pipe(fp_cond_reg_reg_d);
9813 %}
9814
9815 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9816 %{
9817 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9818
9819 ins_cost(INSN_COST * 3);
9820
9821 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9822 ins_encode %{
9823 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9824 __ fcseld(as_FloatRegister($dst$$reg),
9825 as_FloatRegister($src2$$reg),
9826 as_FloatRegister($src1$$reg),
9827 cond);
9828 %}
9829
9830 ins_pipe(fp_cond_reg_reg_d);
9831 %}
9832
9833 // ============================================================================
9834 // Arithmetic Instructions
9835 //
9836
9837 // Integer Addition
9838
9839 // TODO
9840 // these currently employ operations which do not set CR and hence are
9841 // not flagged as killing CR but we would like to isolate the cases
9842 // where we want to set flags from those where we don't. need to work
9843 // out how to do that.
9844
9845 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9846 match(Set dst (AddI src1 src2));
9847
9848 ins_cost(INSN_COST);
9849 format %{ "addw $dst, $src1, $src2" %}
9850
9851 ins_encode %{
9852 __ addw(as_Register($dst$$reg),
9853 as_Register($src1$$reg),
9854 as_Register($src2$$reg));
9855 %}
9856
9857 ins_pipe(ialu_reg_reg);
9858 %}
9859
9860 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9861 match(Set dst (AddI src1 src2));
9862
9863 ins_cost(INSN_COST);
9864 format %{ "addw $dst, $src1, $src2" %}
9865
9866 // use opcode to indicate that this is an add not a sub
9867 opcode(0x0);
9868
9869 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9870
9871 ins_pipe(ialu_reg_imm);
9872 %}
9873
9874 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9875 match(Set dst (AddI (ConvL2I src1) src2));
9876
9877 ins_cost(INSN_COST);
9878 format %{ "addw $dst, $src1, $src2" %}
9879
9880 // use opcode to indicate that this is an add not a sub
9881 opcode(0x0);
9882
9883 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9884
9885 ins_pipe(ialu_reg_imm);
9886 %}
9887
9888 // Pointer Addition
9889 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9890 match(Set dst (AddP src1 src2));
9891
9892 ins_cost(INSN_COST);
9893 format %{ "add $dst, $src1, $src2\t# ptr" %}
9894
9895 ins_encode %{
9896 __ add(as_Register($dst$$reg),
9897 as_Register($src1$$reg),
9898 as_Register($src2$$reg));
9899 %}
9900
9901 ins_pipe(ialu_reg_reg);
9902 %}
9903
9904 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9905 match(Set dst (AddP src1 (ConvI2L src2)));
9906
9907 ins_cost(1.9 * INSN_COST);
9908 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9909
9910 ins_encode %{
9911 __ add(as_Register($dst$$reg),
9912 as_Register($src1$$reg),
9913 as_Register($src2$$reg), ext::sxtw);
9914 %}
9915
9916 ins_pipe(ialu_reg_reg);
9917 %}
9918
9919 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9920 match(Set dst (AddP src1 (LShiftL src2 scale)));
9921
9922 ins_cost(1.9 * INSN_COST);
9923 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9924
9925 ins_encode %{
9926 __ lea(as_Register($dst$$reg),
9927 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9928 Address::lsl($scale$$constant)));
9929 %}
9930
9931 ins_pipe(ialu_reg_reg_shift);
9932 %}
9933
9934 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9935 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9936
9937 ins_cost(1.9 * INSN_COST);
9938 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9939
9940 ins_encode %{
9941 __ lea(as_Register($dst$$reg),
9942 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9943 Address::sxtw($scale$$constant)));
9944 %}
9945
9946 ins_pipe(ialu_reg_reg_shift);
9947 %}
9948
9949 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9950 match(Set dst (LShiftL (ConvI2L src) scale));
9951
9952 ins_cost(INSN_COST);
9953 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9954
9955 ins_encode %{
9956 __ sbfiz(as_Register($dst$$reg),
9957 as_Register($src$$reg),
9958 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9959 %}
9960
9961 ins_pipe(ialu_reg_shift);
9962 %}
9963
9964 // Pointer Immediate Addition
9965 // n.b. this needs to be more expensive than using an indirect memory
9966 // operand
9967 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9968 match(Set dst (AddP src1 src2));
9969
9970 ins_cost(INSN_COST);
9971 format %{ "add $dst, $src1, $src2\t# ptr" %}
9972
9973 // use opcode to indicate that this is an add not a sub
9974 opcode(0x0);
9975
9976 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9977
9978 ins_pipe(ialu_reg_imm);
9979 %}
9980
9981 // Long Addition
9982 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9983
9984 match(Set dst (AddL src1 src2));
9985
9986 ins_cost(INSN_COST);
9987 format %{ "add $dst, $src1, $src2" %}
9988
9989 ins_encode %{
9990 __ add(as_Register($dst$$reg),
9991 as_Register($src1$$reg),
9992 as_Register($src2$$reg));
9993 %}
9994
9995 ins_pipe(ialu_reg_reg);
9996 %}
9997
9998 // No constant pool entries requiredLong Immediate Addition.
9999 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10000 match(Set dst (AddL src1 src2));
10001
10002 ins_cost(INSN_COST);
10003 format %{ "add $dst, $src1, $src2" %}
10004
10005 // use opcode to indicate that this is an add not a sub
10006 opcode(0x0);
10007
10008 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10009
10010 ins_pipe(ialu_reg_imm);
10011 %}
10012
10013 // Integer Subtraction
10014 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10015 match(Set dst (SubI src1 src2));
10016
10017 ins_cost(INSN_COST);
10018 format %{ "subw $dst, $src1, $src2" %}
10019
10020 ins_encode %{
10021 __ subw(as_Register($dst$$reg),
10022 as_Register($src1$$reg),
10023 as_Register($src2$$reg));
10024 %}
10025
10026 ins_pipe(ialu_reg_reg);
10027 %}
10028
10029 // Immediate Subtraction
10030 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10031 match(Set dst (SubI src1 src2));
10032
10033 ins_cost(INSN_COST);
10034 format %{ "subw $dst, $src1, $src2" %}
10035
10036 // use opcode to indicate that this is a sub not an add
10037 opcode(0x1);
10038
10039 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10040
10041 ins_pipe(ialu_reg_imm);
10042 %}
10043
10044 // Long Subtraction
10045 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10046
10047 match(Set dst (SubL src1 src2));
10048
10049 ins_cost(INSN_COST);
10050 format %{ "sub $dst, $src1, $src2" %}
10051
10052 ins_encode %{
10053 __ sub(as_Register($dst$$reg),
10054 as_Register($src1$$reg),
10055 as_Register($src2$$reg));
10056 %}
10057
10058 ins_pipe(ialu_reg_reg);
10059 %}
10060
10061 // No constant pool entries requiredLong Immediate Subtraction.
10062 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10063 match(Set dst (SubL src1 src2));
10064
10065 ins_cost(INSN_COST);
10066 format %{ "sub$dst, $src1, $src2" %}
10067
10068 // use opcode to indicate that this is a sub not an add
10069 opcode(0x1);
10070
10071 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10072
10073 ins_pipe(ialu_reg_imm);
10074 %}
10075
10076 // Integer Negation (special case for sub)
10077
10078 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10079 match(Set dst (SubI zero src));
10080
10081 ins_cost(INSN_COST);
10082 format %{ "negw $dst, $src\t# int" %}
10083
10084 ins_encode %{
10085 __ negw(as_Register($dst$$reg),
10086 as_Register($src$$reg));
10087 %}
10088
10089 ins_pipe(ialu_reg);
10090 %}
10091
10092 // Long Negation
10093
10094 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10095 match(Set dst (SubL zero src));
10096
10097 ins_cost(INSN_COST);
10098 format %{ "neg $dst, $src\t# long" %}
10099
10100 ins_encode %{
10101 __ neg(as_Register($dst$$reg),
10102 as_Register($src$$reg));
10103 %}
10104
10105 ins_pipe(ialu_reg);
10106 %}
10107
10108 // Integer Multiply
10109
10110 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10111 match(Set dst (MulI src1 src2));
10112
10113 ins_cost(INSN_COST * 3);
10114 format %{ "mulw $dst, $src1, $src2" %}
10115
10116 ins_encode %{
10117 __ mulw(as_Register($dst$$reg),
10118 as_Register($src1$$reg),
10119 as_Register($src2$$reg));
10120 %}
10121
10122 ins_pipe(imul_reg_reg);
10123 %}
10124
10125 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10126 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10127
10128 ins_cost(INSN_COST * 3);
10129 format %{ "smull $dst, $src1, $src2" %}
10130
10131 ins_encode %{
10132 __ smull(as_Register($dst$$reg),
10133 as_Register($src1$$reg),
10134 as_Register($src2$$reg));
10135 %}
10136
10137 ins_pipe(imul_reg_reg);
10138 %}
10139
10140 // Long Multiply
10141
10142 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10143 match(Set dst (MulL src1 src2));
10144
10145 ins_cost(INSN_COST * 5);
10146 format %{ "mul $dst, $src1, $src2" %}
10147
10148 ins_encode %{
10149 __ mul(as_Register($dst$$reg),
10150 as_Register($src1$$reg),
10151 as_Register($src2$$reg));
10152 %}
10153
10154 ins_pipe(lmul_reg_reg);
10155 %}
10156
10157 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10158 %{
10159 match(Set dst (MulHiL src1 src2));
10160
10161 ins_cost(INSN_COST * 7);
10162 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10163
10164 ins_encode %{
10165 __ smulh(as_Register($dst$$reg),
10166 as_Register($src1$$reg),
10167 as_Register($src2$$reg));
10168 %}
10169
10170 ins_pipe(lmul_reg_reg);
10171 %}
10172
10173 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10174 %{
10175 match(Set dst (UMulHiL src1 src2));
10176
10177 ins_cost(INSN_COST * 7);
10178 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10179
10180 ins_encode %{
10181 __ umulh(as_Register($dst$$reg),
10182 as_Register($src1$$reg),
10183 as_Register($src2$$reg));
10184 %}
10185
10186 ins_pipe(lmul_reg_reg);
10187 %}
10188
10189 // Combined Integer Multiply & Add/Sub
10190
10191 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10192 match(Set dst (AddI src3 (MulI src1 src2)));
10193
10194 ins_cost(INSN_COST * 3);
10195 format %{ "madd $dst, $src1, $src2, $src3" %}
10196
10197 ins_encode %{
10198 __ maddw(as_Register($dst$$reg),
10199 as_Register($src1$$reg),
10200 as_Register($src2$$reg),
10201 as_Register($src3$$reg));
10202 %}
10203
10204 ins_pipe(imac_reg_reg);
10205 %}
10206
10207 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10208 match(Set dst (SubI src3 (MulI src1 src2)));
10209
10210 ins_cost(INSN_COST * 3);
10211 format %{ "msub $dst, $src1, $src2, $src3" %}
10212
10213 ins_encode %{
10214 __ msubw(as_Register($dst$$reg),
10215 as_Register($src1$$reg),
10216 as_Register($src2$$reg),
10217 as_Register($src3$$reg));
10218 %}
10219
10220 ins_pipe(imac_reg_reg);
10221 %}
10222
10223 // Combined Integer Multiply & Neg
10224
10225 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10226 match(Set dst (MulI (SubI zero src1) src2));
10227
10228 ins_cost(INSN_COST * 3);
10229 format %{ "mneg $dst, $src1, $src2" %}
10230
10231 ins_encode %{
10232 __ mnegw(as_Register($dst$$reg),
10233 as_Register($src1$$reg),
10234 as_Register($src2$$reg));
10235 %}
10236
10237 ins_pipe(imac_reg_reg);
10238 %}
10239
10240 // Combined Long Multiply & Add/Sub
10241
10242 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10243 match(Set dst (AddL src3 (MulL src1 src2)));
10244
10245 ins_cost(INSN_COST * 5);
10246 format %{ "madd $dst, $src1, $src2, $src3" %}
10247
10248 ins_encode %{
10249 __ madd(as_Register($dst$$reg),
10250 as_Register($src1$$reg),
10251 as_Register($src2$$reg),
10252 as_Register($src3$$reg));
10253 %}
10254
10255 ins_pipe(lmac_reg_reg);
10256 %}
10257
10258 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10259 match(Set dst (SubL src3 (MulL src1 src2)));
10260
10261 ins_cost(INSN_COST * 5);
10262 format %{ "msub $dst, $src1, $src2, $src3" %}
10263
10264 ins_encode %{
10265 __ msub(as_Register($dst$$reg),
10266 as_Register($src1$$reg),
10267 as_Register($src2$$reg),
10268 as_Register($src3$$reg));
10269 %}
10270
10271 ins_pipe(lmac_reg_reg);
10272 %}
10273
10274 // Combined Long Multiply & Neg
10275
10276 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10277 match(Set dst (MulL (SubL zero src1) src2));
10278
10279 ins_cost(INSN_COST * 5);
10280 format %{ "mneg $dst, $src1, $src2" %}
10281
10282 ins_encode %{
10283 __ mneg(as_Register($dst$$reg),
10284 as_Register($src1$$reg),
10285 as_Register($src2$$reg));
10286 %}
10287
10288 ins_pipe(lmac_reg_reg);
10289 %}
10290
10291 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10292
10293 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10294 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10295
10296 ins_cost(INSN_COST * 3);
10297 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10298
10299 ins_encode %{
10300 __ smaddl(as_Register($dst$$reg),
10301 as_Register($src1$$reg),
10302 as_Register($src2$$reg),
10303 as_Register($src3$$reg));
10304 %}
10305
10306 ins_pipe(imac_reg_reg);
10307 %}
10308
10309 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10310 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10311
10312 ins_cost(INSN_COST * 3);
10313 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10314
10315 ins_encode %{
10316 __ smsubl(as_Register($dst$$reg),
10317 as_Register($src1$$reg),
10318 as_Register($src2$$reg),
10319 as_Register($src3$$reg));
10320 %}
10321
10322 ins_pipe(imac_reg_reg);
10323 %}
10324
10325 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10326 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10327
10328 ins_cost(INSN_COST * 3);
10329 format %{ "smnegl $dst, $src1, $src2" %}
10330
10331 ins_encode %{
10332 __ smnegl(as_Register($dst$$reg),
10333 as_Register($src1$$reg),
10334 as_Register($src2$$reg));
10335 %}
10336
10337 ins_pipe(imac_reg_reg);
10338 %}
10339
10340 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10341
10342 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10343 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10344
10345 ins_cost(INSN_COST * 5);
10346 format %{ "mulw rscratch1, $src1, $src2\n\t"
10347 "maddw $dst, $src3, $src4, rscratch1" %}
10348
10349 ins_encode %{
10350 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10351 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10352
10353 ins_pipe(imac_reg_reg);
10354 %}
10355
10356 // Integer Divide
10357
10358 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10359 match(Set dst (DivI src1 src2));
10360
10361 ins_cost(INSN_COST * 19);
10362 format %{ "sdivw $dst, $src1, $src2" %}
10363
10364 ins_encode(aarch64_enc_divw(dst, src1, src2));
10365 ins_pipe(idiv_reg_reg);
10366 %}
10367
10368 // Long Divide
10369
10370 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10371 match(Set dst (DivL src1 src2));
10372
10373 ins_cost(INSN_COST * 35);
10374 format %{ "sdiv $dst, $src1, $src2" %}
10375
10376 ins_encode(aarch64_enc_div(dst, src1, src2));
10377 ins_pipe(ldiv_reg_reg);
10378 %}
10379
10380 // Integer Remainder
10381
10382 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10383 match(Set dst (ModI src1 src2));
10384
10385 ins_cost(INSN_COST * 22);
10386 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10387 "msubw $dst, rscratch1, $src2, $src1" %}
10388
10389 ins_encode(aarch64_enc_modw(dst, src1, src2));
10390 ins_pipe(idiv_reg_reg);
10391 %}
10392
10393 // Long Remainder
10394
10395 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10396 match(Set dst (ModL src1 src2));
10397
10398 ins_cost(INSN_COST * 38);
10399 format %{ "sdiv rscratch1, $src1, $src2\n"
10400 "msub $dst, rscratch1, $src2, $src1" %}
10401
10402 ins_encode(aarch64_enc_mod(dst, src1, src2));
10403 ins_pipe(ldiv_reg_reg);
10404 %}
10405
10406 // Unsigned Integer Divide
10407
10408 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10409 match(Set dst (UDivI src1 src2));
10410
10411 ins_cost(INSN_COST * 19);
10412 format %{ "udivw $dst, $src1, $src2" %}
10413
10414 ins_encode %{
10415 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10416 %}
10417
10418 ins_pipe(idiv_reg_reg);
10419 %}
10420
10421 // Unsigned Long Divide
10422
10423 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10424 match(Set dst (UDivL src1 src2));
10425
10426 ins_cost(INSN_COST * 35);
10427 format %{ "udiv $dst, $src1, $src2" %}
10428
10429 ins_encode %{
10430 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10431 %}
10432
10433 ins_pipe(ldiv_reg_reg);
10434 %}
10435
10436 // Unsigned Integer Remainder
10437
10438 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10439 match(Set dst (UModI src1 src2));
10440
10441 ins_cost(INSN_COST * 22);
10442 format %{ "udivw rscratch1, $src1, $src2\n\t"
10443 "msubw $dst, rscratch1, $src2, $src1" %}
10444
10445 ins_encode %{
10446 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10447 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10448 %}
10449
10450 ins_pipe(idiv_reg_reg);
10451 %}
10452
10453 // Unsigned Long Remainder
10454
10455 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10456 match(Set dst (UModL src1 src2));
10457
10458 ins_cost(INSN_COST * 38);
10459 format %{ "udiv rscratch1, $src1, $src2\n"
10460 "msub $dst, rscratch1, $src2, $src1" %}
10461
10462 ins_encode %{
10463 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10464 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10465 %}
10466
10467 ins_pipe(ldiv_reg_reg);
10468 %}
10469
10470 // Integer Shifts
10471
10472 // Shift Left Register
10473 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10474 match(Set dst (LShiftI src1 src2));
10475
10476 ins_cost(INSN_COST * 2);
10477 format %{ "lslvw $dst, $src1, $src2" %}
10478
10479 ins_encode %{
10480 __ lslvw(as_Register($dst$$reg),
10481 as_Register($src1$$reg),
10482 as_Register($src2$$reg));
10483 %}
10484
10485 ins_pipe(ialu_reg_reg_vshift);
10486 %}
10487
10488 // Shift Left Immediate
10489 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10490 match(Set dst (LShiftI src1 src2));
10491
10492 ins_cost(INSN_COST);
10493 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10494
10495 ins_encode %{
10496 __ lslw(as_Register($dst$$reg),
10497 as_Register($src1$$reg),
10498 $src2$$constant & 0x1f);
10499 %}
10500
10501 ins_pipe(ialu_reg_shift);
10502 %}
10503
10504 // Shift Right Logical Register
10505 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10506 match(Set dst (URShiftI src1 src2));
10507
10508 ins_cost(INSN_COST * 2);
10509 format %{ "lsrvw $dst, $src1, $src2" %}
10510
10511 ins_encode %{
10512 __ lsrvw(as_Register($dst$$reg),
10513 as_Register($src1$$reg),
10514 as_Register($src2$$reg));
10515 %}
10516
10517 ins_pipe(ialu_reg_reg_vshift);
10518 %}
10519
10520 // Shift Right Logical Immediate
10521 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10522 match(Set dst (URShiftI src1 src2));
10523
10524 ins_cost(INSN_COST);
10525 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10526
10527 ins_encode %{
10528 __ lsrw(as_Register($dst$$reg),
10529 as_Register($src1$$reg),
10530 $src2$$constant & 0x1f);
10531 %}
10532
10533 ins_pipe(ialu_reg_shift);
10534 %}
10535
10536 // Shift Right Arithmetic Register
10537 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10538 match(Set dst (RShiftI src1 src2));
10539
10540 ins_cost(INSN_COST * 2);
10541 format %{ "asrvw $dst, $src1, $src2" %}
10542
10543 ins_encode %{
10544 __ asrvw(as_Register($dst$$reg),
10545 as_Register($src1$$reg),
10546 as_Register($src2$$reg));
10547 %}
10548
10549 ins_pipe(ialu_reg_reg_vshift);
10550 %}
10551
10552 // Shift Right Arithmetic Immediate
10553 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10554 match(Set dst (RShiftI src1 src2));
10555
10556 ins_cost(INSN_COST);
10557 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10558
10559 ins_encode %{
10560 __ asrw(as_Register($dst$$reg),
10561 as_Register($src1$$reg),
10562 $src2$$constant & 0x1f);
10563 %}
10564
10565 ins_pipe(ialu_reg_shift);
10566 %}
10567
10568 // Combined Int Mask and Right Shift (using UBFM)
10569 // TODO
10570
10571 // Long Shifts
10572
10573 // Shift Left Register
10574 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10575 match(Set dst (LShiftL src1 src2));
10576
10577 ins_cost(INSN_COST * 2);
10578 format %{ "lslv $dst, $src1, $src2" %}
10579
10580 ins_encode %{
10581 __ lslv(as_Register($dst$$reg),
10582 as_Register($src1$$reg),
10583 as_Register($src2$$reg));
10584 %}
10585
10586 ins_pipe(ialu_reg_reg_vshift);
10587 %}
10588
10589 // Shift Left Immediate
10590 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10591 match(Set dst (LShiftL src1 src2));
10592
10593 ins_cost(INSN_COST);
10594 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10595
10596 ins_encode %{
10597 __ lsl(as_Register($dst$$reg),
10598 as_Register($src1$$reg),
10599 $src2$$constant & 0x3f);
10600 %}
10601
10602 ins_pipe(ialu_reg_shift);
10603 %}
10604
10605 // Shift Right Logical Register
10606 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10607 match(Set dst (URShiftL src1 src2));
10608
10609 ins_cost(INSN_COST * 2);
10610 format %{ "lsrv $dst, $src1, $src2" %}
10611
10612 ins_encode %{
10613 __ lsrv(as_Register($dst$$reg),
10614 as_Register($src1$$reg),
10615 as_Register($src2$$reg));
10616 %}
10617
10618 ins_pipe(ialu_reg_reg_vshift);
10619 %}
10620
10621 // Shift Right Logical Immediate
10622 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10623 match(Set dst (URShiftL src1 src2));
10624
10625 ins_cost(INSN_COST);
10626 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10627
10628 ins_encode %{
10629 __ lsr(as_Register($dst$$reg),
10630 as_Register($src1$$reg),
10631 $src2$$constant & 0x3f);
10632 %}
10633
10634 ins_pipe(ialu_reg_shift);
10635 %}
10636
10637 // A special-case pattern for card table stores.
10638 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10639 match(Set dst (URShiftL (CastP2X src1) src2));
10640
10641 ins_cost(INSN_COST);
10642 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10643
10644 ins_encode %{
10645 __ lsr(as_Register($dst$$reg),
10646 as_Register($src1$$reg),
10647 $src2$$constant & 0x3f);
10648 %}
10649
10650 ins_pipe(ialu_reg_shift);
10651 %}
10652
10653 // Shift Right Arithmetic Register
10654 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10655 match(Set dst (RShiftL src1 src2));
10656
10657 ins_cost(INSN_COST * 2);
10658 format %{ "asrv $dst, $src1, $src2" %}
10659
10660 ins_encode %{
10661 __ asrv(as_Register($dst$$reg),
10662 as_Register($src1$$reg),
10663 as_Register($src2$$reg));
10664 %}
10665
10666 ins_pipe(ialu_reg_reg_vshift);
10667 %}
10668
10669 // Shift Right Arithmetic Immediate
10670 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10671 match(Set dst (RShiftL src1 src2));
10672
10673 ins_cost(INSN_COST);
10674 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10675
10676 ins_encode %{
10677 __ asr(as_Register($dst$$reg),
10678 as_Register($src1$$reg),
10679 $src2$$constant & 0x3f);
10680 %}
10681
10682 ins_pipe(ialu_reg_shift);
10683 %}
10684
10685 // BEGIN This section of the file is automatically generated. Do not edit --------------
10686 // This section is generated from aarch64_ad.m4
10687
10688 // This pattern is automatically generated from aarch64_ad.m4
10689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10690 instruct regL_not_reg(iRegLNoSp dst,
10691 iRegL src1, immL_M1 m1,
10692 rFlagsReg cr) %{
10693 match(Set dst (XorL src1 m1));
10694 ins_cost(INSN_COST);
10695 format %{ "eon $dst, $src1, zr" %}
10696
10697 ins_encode %{
10698 __ eon(as_Register($dst$$reg),
10699 as_Register($src1$$reg),
10700 zr,
10701 Assembler::LSL, 0);
10702 %}
10703
10704 ins_pipe(ialu_reg);
10705 %}
10706
10707 // This pattern is automatically generated from aarch64_ad.m4
10708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10709 instruct regI_not_reg(iRegINoSp dst,
10710 iRegIorL2I src1, immI_M1 m1,
10711 rFlagsReg cr) %{
10712 match(Set dst (XorI src1 m1));
10713 ins_cost(INSN_COST);
10714 format %{ "eonw $dst, $src1, zr" %}
10715
10716 ins_encode %{
10717 __ eonw(as_Register($dst$$reg),
10718 as_Register($src1$$reg),
10719 zr,
10720 Assembler::LSL, 0);
10721 %}
10722
10723 ins_pipe(ialu_reg);
10724 %}
10725
10726 // This pattern is automatically generated from aarch64_ad.m4
10727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10728 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10729 immI0 zero, iRegIorL2I src1, immI src2) %{
10730 match(Set dst (SubI zero (URShiftI src1 src2)));
10731
10732 ins_cost(1.9 * INSN_COST);
10733 format %{ "negw $dst, $src1, LSR $src2" %}
10734
10735 ins_encode %{
10736 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10737 Assembler::LSR, $src2$$constant & 0x1f);
10738 %}
10739
10740 ins_pipe(ialu_reg_shift);
10741 %}
10742
10743 // This pattern is automatically generated from aarch64_ad.m4
10744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10745 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10746 immI0 zero, iRegIorL2I src1, immI src2) %{
10747 match(Set dst (SubI zero (RShiftI src1 src2)));
10748
10749 ins_cost(1.9 * INSN_COST);
10750 format %{ "negw $dst, $src1, ASR $src2" %}
10751
10752 ins_encode %{
10753 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10754 Assembler::ASR, $src2$$constant & 0x1f);
10755 %}
10756
10757 ins_pipe(ialu_reg_shift);
10758 %}
10759
10760 // This pattern is automatically generated from aarch64_ad.m4
10761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10762 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10763 immI0 zero, iRegIorL2I src1, immI src2) %{
10764 match(Set dst (SubI zero (LShiftI src1 src2)));
10765
10766 ins_cost(1.9 * INSN_COST);
10767 format %{ "negw $dst, $src1, LSL $src2" %}
10768
10769 ins_encode %{
10770 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10771 Assembler::LSL, $src2$$constant & 0x1f);
10772 %}
10773
10774 ins_pipe(ialu_reg_shift);
10775 %}
10776
10777 // This pattern is automatically generated from aarch64_ad.m4
10778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10779 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10780 immL0 zero, iRegL src1, immI src2) %{
10781 match(Set dst (SubL zero (URShiftL src1 src2)));
10782
10783 ins_cost(1.9 * INSN_COST);
10784 format %{ "neg $dst, $src1, LSR $src2" %}
10785
10786 ins_encode %{
10787 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10788 Assembler::LSR, $src2$$constant & 0x3f);
10789 %}
10790
10791 ins_pipe(ialu_reg_shift);
10792 %}
10793
10794 // This pattern is automatically generated from aarch64_ad.m4
10795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10796 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10797 immL0 zero, iRegL src1, immI src2) %{
10798 match(Set dst (SubL zero (RShiftL src1 src2)));
10799
10800 ins_cost(1.9 * INSN_COST);
10801 format %{ "neg $dst, $src1, ASR $src2" %}
10802
10803 ins_encode %{
10804 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10805 Assembler::ASR, $src2$$constant & 0x3f);
10806 %}
10807
10808 ins_pipe(ialu_reg_shift);
10809 %}
10810
10811 // This pattern is automatically generated from aarch64_ad.m4
10812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10813 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10814 immL0 zero, iRegL src1, immI src2) %{
10815 match(Set dst (SubL zero (LShiftL src1 src2)));
10816
10817 ins_cost(1.9 * INSN_COST);
10818 format %{ "neg $dst, $src1, LSL $src2" %}
10819
10820 ins_encode %{
10821 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10822 Assembler::LSL, $src2$$constant & 0x3f);
10823 %}
10824
10825 ins_pipe(ialu_reg_shift);
10826 %}
10827
10828 // This pattern is automatically generated from aarch64_ad.m4
10829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10830 instruct AndI_reg_not_reg(iRegINoSp dst,
10831 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10832 match(Set dst (AndI src1 (XorI src2 m1)));
10833 ins_cost(INSN_COST);
10834 format %{ "bicw $dst, $src1, $src2" %}
10835
10836 ins_encode %{
10837 __ bicw(as_Register($dst$$reg),
10838 as_Register($src1$$reg),
10839 as_Register($src2$$reg),
10840 Assembler::LSL, 0);
10841 %}
10842
10843 ins_pipe(ialu_reg_reg);
10844 %}
10845
10846 // This pattern is automatically generated from aarch64_ad.m4
10847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10848 instruct AndL_reg_not_reg(iRegLNoSp dst,
10849 iRegL src1, iRegL src2, immL_M1 m1) %{
10850 match(Set dst (AndL src1 (XorL src2 m1)));
10851 ins_cost(INSN_COST);
10852 format %{ "bic $dst, $src1, $src2" %}
10853
10854 ins_encode %{
10855 __ bic(as_Register($dst$$reg),
10856 as_Register($src1$$reg),
10857 as_Register($src2$$reg),
10858 Assembler::LSL, 0);
10859 %}
10860
10861 ins_pipe(ialu_reg_reg);
10862 %}
10863
10864 // This pattern is automatically generated from aarch64_ad.m4
10865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10866 instruct OrI_reg_not_reg(iRegINoSp dst,
10867 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10868 match(Set dst (OrI src1 (XorI src2 m1)));
10869 ins_cost(INSN_COST);
10870 format %{ "ornw $dst, $src1, $src2" %}
10871
10872 ins_encode %{
10873 __ ornw(as_Register($dst$$reg),
10874 as_Register($src1$$reg),
10875 as_Register($src2$$reg),
10876 Assembler::LSL, 0);
10877 %}
10878
10879 ins_pipe(ialu_reg_reg);
10880 %}
10881
10882 // This pattern is automatically generated from aarch64_ad.m4
10883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10884 instruct OrL_reg_not_reg(iRegLNoSp dst,
10885 iRegL src1, iRegL src2, immL_M1 m1) %{
10886 match(Set dst (OrL src1 (XorL src2 m1)));
10887 ins_cost(INSN_COST);
10888 format %{ "orn $dst, $src1, $src2" %}
10889
10890 ins_encode %{
10891 __ orn(as_Register($dst$$reg),
10892 as_Register($src1$$reg),
10893 as_Register($src2$$reg),
10894 Assembler::LSL, 0);
10895 %}
10896
10897 ins_pipe(ialu_reg_reg);
10898 %}
10899
10900 // This pattern is automatically generated from aarch64_ad.m4
10901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10902 instruct XorI_reg_not_reg(iRegINoSp dst,
10903 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10904 match(Set dst (XorI m1 (XorI src2 src1)));
10905 ins_cost(INSN_COST);
10906 format %{ "eonw $dst, $src1, $src2" %}
10907
10908 ins_encode %{
10909 __ eonw(as_Register($dst$$reg),
10910 as_Register($src1$$reg),
10911 as_Register($src2$$reg),
10912 Assembler::LSL, 0);
10913 %}
10914
10915 ins_pipe(ialu_reg_reg);
10916 %}
10917
10918 // This pattern is automatically generated from aarch64_ad.m4
10919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10920 instruct XorL_reg_not_reg(iRegLNoSp dst,
10921 iRegL src1, iRegL src2, immL_M1 m1) %{
10922 match(Set dst (XorL m1 (XorL src2 src1)));
10923 ins_cost(INSN_COST);
10924 format %{ "eon $dst, $src1, $src2" %}
10925
10926 ins_encode %{
10927 __ eon(as_Register($dst$$reg),
10928 as_Register($src1$$reg),
10929 as_Register($src2$$reg),
10930 Assembler::LSL, 0);
10931 %}
10932
10933 ins_pipe(ialu_reg_reg);
10934 %}
10935
10936 // This pattern is automatically generated from aarch64_ad.m4
10937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10938 // val & (-1 ^ (val >>> shift)) ==> bicw
10939 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10940 iRegIorL2I src1, iRegIorL2I src2,
10941 immI src3, immI_M1 src4) %{
10942 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10943 ins_cost(1.9 * INSN_COST);
10944 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10945
10946 ins_encode %{
10947 __ bicw(as_Register($dst$$reg),
10948 as_Register($src1$$reg),
10949 as_Register($src2$$reg),
10950 Assembler::LSR,
10951 $src3$$constant & 0x1f);
10952 %}
10953
10954 ins_pipe(ialu_reg_reg_shift);
10955 %}
10956
10957 // This pattern is automatically generated from aarch64_ad.m4
10958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10959 // val & (-1 ^ (val >>> shift)) ==> bic
10960 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10961 iRegL src1, iRegL src2,
10962 immI src3, immL_M1 src4) %{
10963 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10964 ins_cost(1.9 * INSN_COST);
10965 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10966
10967 ins_encode %{
10968 __ bic(as_Register($dst$$reg),
10969 as_Register($src1$$reg),
10970 as_Register($src2$$reg),
10971 Assembler::LSR,
10972 $src3$$constant & 0x3f);
10973 %}
10974
10975 ins_pipe(ialu_reg_reg_shift);
10976 %}
10977
10978 // This pattern is automatically generated from aarch64_ad.m4
10979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10980 // val & (-1 ^ (val >> shift)) ==> bicw
10981 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10982 iRegIorL2I src1, iRegIorL2I src2,
10983 immI src3, immI_M1 src4) %{
10984 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10985 ins_cost(1.9 * INSN_COST);
10986 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10987
10988 ins_encode %{
10989 __ bicw(as_Register($dst$$reg),
10990 as_Register($src1$$reg),
10991 as_Register($src2$$reg),
10992 Assembler::ASR,
10993 $src3$$constant & 0x1f);
10994 %}
10995
10996 ins_pipe(ialu_reg_reg_shift);
10997 %}
10998
10999 // This pattern is automatically generated from aarch64_ad.m4
11000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11001 // val & (-1 ^ (val >> shift)) ==> bic
11002 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11003 iRegL src1, iRegL src2,
11004 immI src3, immL_M1 src4) %{
11005 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11006 ins_cost(1.9 * INSN_COST);
11007 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11008
11009 ins_encode %{
11010 __ bic(as_Register($dst$$reg),
11011 as_Register($src1$$reg),
11012 as_Register($src2$$reg),
11013 Assembler::ASR,
11014 $src3$$constant & 0x3f);
11015 %}
11016
11017 ins_pipe(ialu_reg_reg_shift);
11018 %}
11019
11020 // This pattern is automatically generated from aarch64_ad.m4
11021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11022 // val & (-1 ^ (val ror shift)) ==> bicw
11023 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11024 iRegIorL2I src1, iRegIorL2I src2,
11025 immI src3, immI_M1 src4) %{
11026 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11027 ins_cost(1.9 * INSN_COST);
11028 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11029
11030 ins_encode %{
11031 __ bicw(as_Register($dst$$reg),
11032 as_Register($src1$$reg),
11033 as_Register($src2$$reg),
11034 Assembler::ROR,
11035 $src3$$constant & 0x1f);
11036 %}
11037
11038 ins_pipe(ialu_reg_reg_shift);
11039 %}
11040
11041 // This pattern is automatically generated from aarch64_ad.m4
11042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11043 // val & (-1 ^ (val ror shift)) ==> bic
11044 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11045 iRegL src1, iRegL src2,
11046 immI src3, immL_M1 src4) %{
11047 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11048 ins_cost(1.9 * INSN_COST);
11049 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11050
11051 ins_encode %{
11052 __ bic(as_Register($dst$$reg),
11053 as_Register($src1$$reg),
11054 as_Register($src2$$reg),
11055 Assembler::ROR,
11056 $src3$$constant & 0x3f);
11057 %}
11058
11059 ins_pipe(ialu_reg_reg_shift);
11060 %}
11061
11062 // This pattern is automatically generated from aarch64_ad.m4
11063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11064 // val & (-1 ^ (val << shift)) ==> bicw
11065 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11066 iRegIorL2I src1, iRegIorL2I src2,
11067 immI src3, immI_M1 src4) %{
11068 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11069 ins_cost(1.9 * INSN_COST);
11070 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11071
11072 ins_encode %{
11073 __ bicw(as_Register($dst$$reg),
11074 as_Register($src1$$reg),
11075 as_Register($src2$$reg),
11076 Assembler::LSL,
11077 $src3$$constant & 0x1f);
11078 %}
11079
11080 ins_pipe(ialu_reg_reg_shift);
11081 %}
11082
11083 // This pattern is automatically generated from aarch64_ad.m4
11084 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11085 // val & (-1 ^ (val << shift)) ==> bic
11086 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11087 iRegL src1, iRegL src2,
11088 immI src3, immL_M1 src4) %{
11089 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11090 ins_cost(1.9 * INSN_COST);
11091 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11092
11093 ins_encode %{
11094 __ bic(as_Register($dst$$reg),
11095 as_Register($src1$$reg),
11096 as_Register($src2$$reg),
11097 Assembler::LSL,
11098 $src3$$constant & 0x3f);
11099 %}
11100
11101 ins_pipe(ialu_reg_reg_shift);
11102 %}
11103
11104 // This pattern is automatically generated from aarch64_ad.m4
11105 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11106 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11107 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11108 iRegIorL2I src1, iRegIorL2I src2,
11109 immI src3, immI_M1 src4) %{
11110 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11111 ins_cost(1.9 * INSN_COST);
11112 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11113
11114 ins_encode %{
11115 __ eonw(as_Register($dst$$reg),
11116 as_Register($src1$$reg),
11117 as_Register($src2$$reg),
11118 Assembler::LSR,
11119 $src3$$constant & 0x1f);
11120 %}
11121
11122 ins_pipe(ialu_reg_reg_shift);
11123 %}
11124
11125 // This pattern is automatically generated from aarch64_ad.m4
11126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11127 // val ^ (-1 ^ (val >>> shift)) ==> eon
11128 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11129 iRegL src1, iRegL src2,
11130 immI src3, immL_M1 src4) %{
11131 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11132 ins_cost(1.9 * INSN_COST);
11133 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11134
11135 ins_encode %{
11136 __ eon(as_Register($dst$$reg),
11137 as_Register($src1$$reg),
11138 as_Register($src2$$reg),
11139 Assembler::LSR,
11140 $src3$$constant & 0x3f);
11141 %}
11142
11143 ins_pipe(ialu_reg_reg_shift);
11144 %}
11145
11146 // This pattern is automatically generated from aarch64_ad.m4
11147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11148 // val ^ (-1 ^ (val >> shift)) ==> eonw
11149 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11150 iRegIorL2I src1, iRegIorL2I src2,
11151 immI src3, immI_M1 src4) %{
11152 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11153 ins_cost(1.9 * INSN_COST);
11154 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11155
11156 ins_encode %{
11157 __ eonw(as_Register($dst$$reg),
11158 as_Register($src1$$reg),
11159 as_Register($src2$$reg),
11160 Assembler::ASR,
11161 $src3$$constant & 0x1f);
11162 %}
11163
11164 ins_pipe(ialu_reg_reg_shift);
11165 %}
11166
11167 // This pattern is automatically generated from aarch64_ad.m4
11168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11169 // val ^ (-1 ^ (val >> shift)) ==> eon
11170 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11171 iRegL src1, iRegL src2,
11172 immI src3, immL_M1 src4) %{
11173 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11174 ins_cost(1.9 * INSN_COST);
11175 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11176
11177 ins_encode %{
11178 __ eon(as_Register($dst$$reg),
11179 as_Register($src1$$reg),
11180 as_Register($src2$$reg),
11181 Assembler::ASR,
11182 $src3$$constant & 0x3f);
11183 %}
11184
11185 ins_pipe(ialu_reg_reg_shift);
11186 %}
11187
11188 // This pattern is automatically generated from aarch64_ad.m4
11189 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11190 // val ^ (-1 ^ (val ror shift)) ==> eonw
11191 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11192 iRegIorL2I src1, iRegIorL2I src2,
11193 immI src3, immI_M1 src4) %{
11194 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11195 ins_cost(1.9 * INSN_COST);
11196 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11197
11198 ins_encode %{
11199 __ eonw(as_Register($dst$$reg),
11200 as_Register($src1$$reg),
11201 as_Register($src2$$reg),
11202 Assembler::ROR,
11203 $src3$$constant & 0x1f);
11204 %}
11205
11206 ins_pipe(ialu_reg_reg_shift);
11207 %}
11208
11209 // This pattern is automatically generated from aarch64_ad.m4
11210 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11211 // val ^ (-1 ^ (val ror shift)) ==> eon
11212 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11213 iRegL src1, iRegL src2,
11214 immI src3, immL_M1 src4) %{
11215 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11216 ins_cost(1.9 * INSN_COST);
11217 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11218
11219 ins_encode %{
11220 __ eon(as_Register($dst$$reg),
11221 as_Register($src1$$reg),
11222 as_Register($src2$$reg),
11223 Assembler::ROR,
11224 $src3$$constant & 0x3f);
11225 %}
11226
11227 ins_pipe(ialu_reg_reg_shift);
11228 %}
11229
11230 // This pattern is automatically generated from aarch64_ad.m4
11231 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11232 // val ^ (-1 ^ (val << shift)) ==> eonw
11233 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11234 iRegIorL2I src1, iRegIorL2I src2,
11235 immI src3, immI_M1 src4) %{
11236 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11237 ins_cost(1.9 * INSN_COST);
11238 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11239
11240 ins_encode %{
11241 __ eonw(as_Register($dst$$reg),
11242 as_Register($src1$$reg),
11243 as_Register($src2$$reg),
11244 Assembler::LSL,
11245 $src3$$constant & 0x1f);
11246 %}
11247
11248 ins_pipe(ialu_reg_reg_shift);
11249 %}
11250
11251 // This pattern is automatically generated from aarch64_ad.m4
11252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11253 // val ^ (-1 ^ (val << shift)) ==> eon
11254 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11255 iRegL src1, iRegL src2,
11256 immI src3, immL_M1 src4) %{
11257 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11258 ins_cost(1.9 * INSN_COST);
11259 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11260
11261 ins_encode %{
11262 __ eon(as_Register($dst$$reg),
11263 as_Register($src1$$reg),
11264 as_Register($src2$$reg),
11265 Assembler::LSL,
11266 $src3$$constant & 0x3f);
11267 %}
11268
11269 ins_pipe(ialu_reg_reg_shift);
11270 %}
11271
11272 // This pattern is automatically generated from aarch64_ad.m4
11273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11274 // val | (-1 ^ (val >>> shift)) ==> ornw
11275 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11276 iRegIorL2I src1, iRegIorL2I src2,
11277 immI src3, immI_M1 src4) %{
11278 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11279 ins_cost(1.9 * INSN_COST);
11280 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11281
11282 ins_encode %{
11283 __ ornw(as_Register($dst$$reg),
11284 as_Register($src1$$reg),
11285 as_Register($src2$$reg),
11286 Assembler::LSR,
11287 $src3$$constant & 0x1f);
11288 %}
11289
11290 ins_pipe(ialu_reg_reg_shift);
11291 %}
11292
11293 // This pattern is automatically generated from aarch64_ad.m4
11294 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11295 // val | (-1 ^ (val >>> shift)) ==> orn
11296 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11297 iRegL src1, iRegL src2,
11298 immI src3, immL_M1 src4) %{
11299 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11300 ins_cost(1.9 * INSN_COST);
11301 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11302
11303 ins_encode %{
11304 __ orn(as_Register($dst$$reg),
11305 as_Register($src1$$reg),
11306 as_Register($src2$$reg),
11307 Assembler::LSR,
11308 $src3$$constant & 0x3f);
11309 %}
11310
11311 ins_pipe(ialu_reg_reg_shift);
11312 %}
11313
11314 // This pattern is automatically generated from aarch64_ad.m4
11315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11316 // val | (-1 ^ (val >> shift)) ==> ornw
11317 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11318 iRegIorL2I src1, iRegIorL2I src2,
11319 immI src3, immI_M1 src4) %{
11320 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11321 ins_cost(1.9 * INSN_COST);
11322 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11323
11324 ins_encode %{
11325 __ ornw(as_Register($dst$$reg),
11326 as_Register($src1$$reg),
11327 as_Register($src2$$reg),
11328 Assembler::ASR,
11329 $src3$$constant & 0x1f);
11330 %}
11331
11332 ins_pipe(ialu_reg_reg_shift);
11333 %}
11334
11335 // This pattern is automatically generated from aarch64_ad.m4
11336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11337 // val | (-1 ^ (val >> shift)) ==> orn
11338 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11339 iRegL src1, iRegL src2,
11340 immI src3, immL_M1 src4) %{
11341 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11342 ins_cost(1.9 * INSN_COST);
11343 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11344
11345 ins_encode %{
11346 __ orn(as_Register($dst$$reg),
11347 as_Register($src1$$reg),
11348 as_Register($src2$$reg),
11349 Assembler::ASR,
11350 $src3$$constant & 0x3f);
11351 %}
11352
11353 ins_pipe(ialu_reg_reg_shift);
11354 %}
11355
11356 // This pattern is automatically generated from aarch64_ad.m4
11357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11358 // val | (-1 ^ (val ror shift)) ==> ornw
11359 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11360 iRegIorL2I src1, iRegIorL2I src2,
11361 immI src3, immI_M1 src4) %{
11362 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11363 ins_cost(1.9 * INSN_COST);
11364 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11365
11366 ins_encode %{
11367 __ ornw(as_Register($dst$$reg),
11368 as_Register($src1$$reg),
11369 as_Register($src2$$reg),
11370 Assembler::ROR,
11371 $src3$$constant & 0x1f);
11372 %}
11373
11374 ins_pipe(ialu_reg_reg_shift);
11375 %}
11376
11377 // This pattern is automatically generated from aarch64_ad.m4
11378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11379 // val | (-1 ^ (val ror shift)) ==> orn
11380 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11381 iRegL src1, iRegL src2,
11382 immI src3, immL_M1 src4) %{
11383 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11384 ins_cost(1.9 * INSN_COST);
11385 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11386
11387 ins_encode %{
11388 __ orn(as_Register($dst$$reg),
11389 as_Register($src1$$reg),
11390 as_Register($src2$$reg),
11391 Assembler::ROR,
11392 $src3$$constant & 0x3f);
11393 %}
11394
11395 ins_pipe(ialu_reg_reg_shift);
11396 %}
11397
11398 // This pattern is automatically generated from aarch64_ad.m4
11399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11400 // val | (-1 ^ (val << shift)) ==> ornw
11401 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11402 iRegIorL2I src1, iRegIorL2I src2,
11403 immI src3, immI_M1 src4) %{
11404 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11405 ins_cost(1.9 * INSN_COST);
11406 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11407
11408 ins_encode %{
11409 __ ornw(as_Register($dst$$reg),
11410 as_Register($src1$$reg),
11411 as_Register($src2$$reg),
11412 Assembler::LSL,
11413 $src3$$constant & 0x1f);
11414 %}
11415
11416 ins_pipe(ialu_reg_reg_shift);
11417 %}
11418
11419 // This pattern is automatically generated from aarch64_ad.m4
11420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11421 // val | (-1 ^ (val << shift)) ==> orn
11422 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11423 iRegL src1, iRegL src2,
11424 immI src3, immL_M1 src4) %{
11425 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11426 ins_cost(1.9 * INSN_COST);
11427 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11428
11429 ins_encode %{
11430 __ orn(as_Register($dst$$reg),
11431 as_Register($src1$$reg),
11432 as_Register($src2$$reg),
11433 Assembler::LSL,
11434 $src3$$constant & 0x3f);
11435 %}
11436
11437 ins_pipe(ialu_reg_reg_shift);
11438 %}
11439
11440 // This pattern is automatically generated from aarch64_ad.m4
11441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11442 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11443 iRegIorL2I src1, iRegIorL2I src2,
11444 immI src3) %{
11445 match(Set dst (AndI src1 (URShiftI src2 src3)));
11446
11447 ins_cost(1.9 * INSN_COST);
11448 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11449
11450 ins_encode %{
11451 __ andw(as_Register($dst$$reg),
11452 as_Register($src1$$reg),
11453 as_Register($src2$$reg),
11454 Assembler::LSR,
11455 $src3$$constant & 0x1f);
11456 %}
11457
11458 ins_pipe(ialu_reg_reg_shift);
11459 %}
11460
11461 // This pattern is automatically generated from aarch64_ad.m4
11462 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11463 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11464 iRegL src1, iRegL src2,
11465 immI src3) %{
11466 match(Set dst (AndL src1 (URShiftL src2 src3)));
11467
11468 ins_cost(1.9 * INSN_COST);
11469 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11470
11471 ins_encode %{
11472 __ andr(as_Register($dst$$reg),
11473 as_Register($src1$$reg),
11474 as_Register($src2$$reg),
11475 Assembler::LSR,
11476 $src3$$constant & 0x3f);
11477 %}
11478
11479 ins_pipe(ialu_reg_reg_shift);
11480 %}
11481
11482 // This pattern is automatically generated from aarch64_ad.m4
11483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11484 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11485 iRegIorL2I src1, iRegIorL2I src2,
11486 immI src3) %{
11487 match(Set dst (AndI src1 (RShiftI src2 src3)));
11488
11489 ins_cost(1.9 * INSN_COST);
11490 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11491
11492 ins_encode %{
11493 __ andw(as_Register($dst$$reg),
11494 as_Register($src1$$reg),
11495 as_Register($src2$$reg),
11496 Assembler::ASR,
11497 $src3$$constant & 0x1f);
11498 %}
11499
11500 ins_pipe(ialu_reg_reg_shift);
11501 %}
11502
11503 // This pattern is automatically generated from aarch64_ad.m4
11504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11505 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11506 iRegL src1, iRegL src2,
11507 immI src3) %{
11508 match(Set dst (AndL src1 (RShiftL src2 src3)));
11509
11510 ins_cost(1.9 * INSN_COST);
11511 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11512
11513 ins_encode %{
11514 __ andr(as_Register($dst$$reg),
11515 as_Register($src1$$reg),
11516 as_Register($src2$$reg),
11517 Assembler::ASR,
11518 $src3$$constant & 0x3f);
11519 %}
11520
11521 ins_pipe(ialu_reg_reg_shift);
11522 %}
11523
11524 // This pattern is automatically generated from aarch64_ad.m4
11525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11526 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11527 iRegIorL2I src1, iRegIorL2I src2,
11528 immI src3) %{
11529 match(Set dst (AndI src1 (LShiftI src2 src3)));
11530
11531 ins_cost(1.9 * INSN_COST);
11532 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11533
11534 ins_encode %{
11535 __ andw(as_Register($dst$$reg),
11536 as_Register($src1$$reg),
11537 as_Register($src2$$reg),
11538 Assembler::LSL,
11539 $src3$$constant & 0x1f);
11540 %}
11541
11542 ins_pipe(ialu_reg_reg_shift);
11543 %}
11544
11545 // This pattern is automatically generated from aarch64_ad.m4
11546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11547 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11548 iRegL src1, iRegL src2,
11549 immI src3) %{
11550 match(Set dst (AndL src1 (LShiftL src2 src3)));
11551
11552 ins_cost(1.9 * INSN_COST);
11553 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11554
11555 ins_encode %{
11556 __ andr(as_Register($dst$$reg),
11557 as_Register($src1$$reg),
11558 as_Register($src2$$reg),
11559 Assembler::LSL,
11560 $src3$$constant & 0x3f);
11561 %}
11562
11563 ins_pipe(ialu_reg_reg_shift);
11564 %}
11565
11566 // This pattern is automatically generated from aarch64_ad.m4
11567 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11568 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11569 iRegIorL2I src1, iRegIorL2I src2,
11570 immI src3) %{
11571 match(Set dst (AndI src1 (RotateRight src2 src3)));
11572
11573 ins_cost(1.9 * INSN_COST);
11574 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11575
11576 ins_encode %{
11577 __ andw(as_Register($dst$$reg),
11578 as_Register($src1$$reg),
11579 as_Register($src2$$reg),
11580 Assembler::ROR,
11581 $src3$$constant & 0x1f);
11582 %}
11583
11584 ins_pipe(ialu_reg_reg_shift);
11585 %}
11586
11587 // This pattern is automatically generated from aarch64_ad.m4
11588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11589 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11590 iRegL src1, iRegL src2,
11591 immI src3) %{
11592 match(Set dst (AndL src1 (RotateRight src2 src3)));
11593
11594 ins_cost(1.9 * INSN_COST);
11595 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11596
11597 ins_encode %{
11598 __ andr(as_Register($dst$$reg),
11599 as_Register($src1$$reg),
11600 as_Register($src2$$reg),
11601 Assembler::ROR,
11602 $src3$$constant & 0x3f);
11603 %}
11604
11605 ins_pipe(ialu_reg_reg_shift);
11606 %}
11607
11608 // This pattern is automatically generated from aarch64_ad.m4
11609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11610 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11611 iRegIorL2I src1, iRegIorL2I src2,
11612 immI src3) %{
11613 match(Set dst (XorI src1 (URShiftI src2 src3)));
11614
11615 ins_cost(1.9 * INSN_COST);
11616 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11617
11618 ins_encode %{
11619 __ eorw(as_Register($dst$$reg),
11620 as_Register($src1$$reg),
11621 as_Register($src2$$reg),
11622 Assembler::LSR,
11623 $src3$$constant & 0x1f);
11624 %}
11625
11626 ins_pipe(ialu_reg_reg_shift);
11627 %}
11628
11629 // This pattern is automatically generated from aarch64_ad.m4
11630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11631 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11632 iRegL src1, iRegL src2,
11633 immI src3) %{
11634 match(Set dst (XorL src1 (URShiftL src2 src3)));
11635
11636 ins_cost(1.9 * INSN_COST);
11637 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11638
11639 ins_encode %{
11640 __ eor(as_Register($dst$$reg),
11641 as_Register($src1$$reg),
11642 as_Register($src2$$reg),
11643 Assembler::LSR,
11644 $src3$$constant & 0x3f);
11645 %}
11646
11647 ins_pipe(ialu_reg_reg_shift);
11648 %}
11649
11650 // This pattern is automatically generated from aarch64_ad.m4
11651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11652 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11653 iRegIorL2I src1, iRegIorL2I src2,
11654 immI src3) %{
11655 match(Set dst (XorI src1 (RShiftI src2 src3)));
11656
11657 ins_cost(1.9 * INSN_COST);
11658 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11659
11660 ins_encode %{
11661 __ eorw(as_Register($dst$$reg),
11662 as_Register($src1$$reg),
11663 as_Register($src2$$reg),
11664 Assembler::ASR,
11665 $src3$$constant & 0x1f);
11666 %}
11667
11668 ins_pipe(ialu_reg_reg_shift);
11669 %}
11670
11671 // This pattern is automatically generated from aarch64_ad.m4
11672 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11673 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11674 iRegL src1, iRegL src2,
11675 immI src3) %{
11676 match(Set dst (XorL src1 (RShiftL src2 src3)));
11677
11678 ins_cost(1.9 * INSN_COST);
11679 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11680
11681 ins_encode %{
11682 __ eor(as_Register($dst$$reg),
11683 as_Register($src1$$reg),
11684 as_Register($src2$$reg),
11685 Assembler::ASR,
11686 $src3$$constant & 0x3f);
11687 %}
11688
11689 ins_pipe(ialu_reg_reg_shift);
11690 %}
11691
11692 // This pattern is automatically generated from aarch64_ad.m4
11693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11694 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11695 iRegIorL2I src1, iRegIorL2I src2,
11696 immI src3) %{
11697 match(Set dst (XorI src1 (LShiftI src2 src3)));
11698
11699 ins_cost(1.9 * INSN_COST);
11700 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11701
11702 ins_encode %{
11703 __ eorw(as_Register($dst$$reg),
11704 as_Register($src1$$reg),
11705 as_Register($src2$$reg),
11706 Assembler::LSL,
11707 $src3$$constant & 0x1f);
11708 %}
11709
11710 ins_pipe(ialu_reg_reg_shift);
11711 %}
11712
11713 // This pattern is automatically generated from aarch64_ad.m4
11714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11715 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11716 iRegL src1, iRegL src2,
11717 immI src3) %{
11718 match(Set dst (XorL src1 (LShiftL src2 src3)));
11719
11720 ins_cost(1.9 * INSN_COST);
11721 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11722
11723 ins_encode %{
11724 __ eor(as_Register($dst$$reg),
11725 as_Register($src1$$reg),
11726 as_Register($src2$$reg),
11727 Assembler::LSL,
11728 $src3$$constant & 0x3f);
11729 %}
11730
11731 ins_pipe(ialu_reg_reg_shift);
11732 %}
11733
11734 // This pattern is automatically generated from aarch64_ad.m4
11735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11736 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11737 iRegIorL2I src1, iRegIorL2I src2,
11738 immI src3) %{
11739 match(Set dst (XorI src1 (RotateRight src2 src3)));
11740
11741 ins_cost(1.9 * INSN_COST);
11742 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11743
11744 ins_encode %{
11745 __ eorw(as_Register($dst$$reg),
11746 as_Register($src1$$reg),
11747 as_Register($src2$$reg),
11748 Assembler::ROR,
11749 $src3$$constant & 0x1f);
11750 %}
11751
11752 ins_pipe(ialu_reg_reg_shift);
11753 %}
11754
11755 // This pattern is automatically generated from aarch64_ad.m4
11756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11757 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11758 iRegL src1, iRegL src2,
11759 immI src3) %{
11760 match(Set dst (XorL src1 (RotateRight src2 src3)));
11761
11762 ins_cost(1.9 * INSN_COST);
11763 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11764
11765 ins_encode %{
11766 __ eor(as_Register($dst$$reg),
11767 as_Register($src1$$reg),
11768 as_Register($src2$$reg),
11769 Assembler::ROR,
11770 $src3$$constant & 0x3f);
11771 %}
11772
11773 ins_pipe(ialu_reg_reg_shift);
11774 %}
11775
11776 // This pattern is automatically generated from aarch64_ad.m4
11777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11778 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11779 iRegIorL2I src1, iRegIorL2I src2,
11780 immI src3) %{
11781 match(Set dst (OrI src1 (URShiftI src2 src3)));
11782
11783 ins_cost(1.9 * INSN_COST);
11784 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11785
11786 ins_encode %{
11787 __ orrw(as_Register($dst$$reg),
11788 as_Register($src1$$reg),
11789 as_Register($src2$$reg),
11790 Assembler::LSR,
11791 $src3$$constant & 0x1f);
11792 %}
11793
11794 ins_pipe(ialu_reg_reg_shift);
11795 %}
11796
11797 // This pattern is automatically generated from aarch64_ad.m4
11798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11799 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11800 iRegL src1, iRegL src2,
11801 immI src3) %{
11802 match(Set dst (OrL src1 (URShiftL src2 src3)));
11803
11804 ins_cost(1.9 * INSN_COST);
11805 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11806
11807 ins_encode %{
11808 __ orr(as_Register($dst$$reg),
11809 as_Register($src1$$reg),
11810 as_Register($src2$$reg),
11811 Assembler::LSR,
11812 $src3$$constant & 0x3f);
11813 %}
11814
11815 ins_pipe(ialu_reg_reg_shift);
11816 %}
11817
11818 // This pattern is automatically generated from aarch64_ad.m4
11819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11820 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11821 iRegIorL2I src1, iRegIorL2I src2,
11822 immI src3) %{
11823 match(Set dst (OrI src1 (RShiftI src2 src3)));
11824
11825 ins_cost(1.9 * INSN_COST);
11826 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11827
11828 ins_encode %{
11829 __ orrw(as_Register($dst$$reg),
11830 as_Register($src1$$reg),
11831 as_Register($src2$$reg),
11832 Assembler::ASR,
11833 $src3$$constant & 0x1f);
11834 %}
11835
11836 ins_pipe(ialu_reg_reg_shift);
11837 %}
11838
11839 // This pattern is automatically generated from aarch64_ad.m4
11840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11841 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11842 iRegL src1, iRegL src2,
11843 immI src3) %{
11844 match(Set dst (OrL src1 (RShiftL src2 src3)));
11845
11846 ins_cost(1.9 * INSN_COST);
11847 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11848
11849 ins_encode %{
11850 __ orr(as_Register($dst$$reg),
11851 as_Register($src1$$reg),
11852 as_Register($src2$$reg),
11853 Assembler::ASR,
11854 $src3$$constant & 0x3f);
11855 %}
11856
11857 ins_pipe(ialu_reg_reg_shift);
11858 %}
11859
11860 // This pattern is automatically generated from aarch64_ad.m4
11861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11862 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11863 iRegIorL2I src1, iRegIorL2I src2,
11864 immI src3) %{
11865 match(Set dst (OrI src1 (LShiftI src2 src3)));
11866
11867 ins_cost(1.9 * INSN_COST);
11868 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11869
11870 ins_encode %{
11871 __ orrw(as_Register($dst$$reg),
11872 as_Register($src1$$reg),
11873 as_Register($src2$$reg),
11874 Assembler::LSL,
11875 $src3$$constant & 0x1f);
11876 %}
11877
11878 ins_pipe(ialu_reg_reg_shift);
11879 %}
11880
11881 // This pattern is automatically generated from aarch64_ad.m4
11882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11883 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11884 iRegL src1, iRegL src2,
11885 immI src3) %{
11886 match(Set dst (OrL src1 (LShiftL src2 src3)));
11887
11888 ins_cost(1.9 * INSN_COST);
11889 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11890
11891 ins_encode %{
11892 __ orr(as_Register($dst$$reg),
11893 as_Register($src1$$reg),
11894 as_Register($src2$$reg),
11895 Assembler::LSL,
11896 $src3$$constant & 0x3f);
11897 %}
11898
11899 ins_pipe(ialu_reg_reg_shift);
11900 %}
11901
11902 // This pattern is automatically generated from aarch64_ad.m4
11903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11904 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11905 iRegIorL2I src1, iRegIorL2I src2,
11906 immI src3) %{
11907 match(Set dst (OrI src1 (RotateRight src2 src3)));
11908
11909 ins_cost(1.9 * INSN_COST);
11910 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11911
11912 ins_encode %{
11913 __ orrw(as_Register($dst$$reg),
11914 as_Register($src1$$reg),
11915 as_Register($src2$$reg),
11916 Assembler::ROR,
11917 $src3$$constant & 0x1f);
11918 %}
11919
11920 ins_pipe(ialu_reg_reg_shift);
11921 %}
11922
11923 // This pattern is automatically generated from aarch64_ad.m4
11924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11925 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11926 iRegL src1, iRegL src2,
11927 immI src3) %{
11928 match(Set dst (OrL src1 (RotateRight src2 src3)));
11929
11930 ins_cost(1.9 * INSN_COST);
11931 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11932
11933 ins_encode %{
11934 __ orr(as_Register($dst$$reg),
11935 as_Register($src1$$reg),
11936 as_Register($src2$$reg),
11937 Assembler::ROR,
11938 $src3$$constant & 0x3f);
11939 %}
11940
11941 ins_pipe(ialu_reg_reg_shift);
11942 %}
11943
11944 // This pattern is automatically generated from aarch64_ad.m4
11945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11946 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11947 iRegIorL2I src1, iRegIorL2I src2,
11948 immI src3) %{
11949 match(Set dst (AddI src1 (URShiftI src2 src3)));
11950
11951 ins_cost(1.9 * INSN_COST);
11952 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11953
11954 ins_encode %{
11955 __ addw(as_Register($dst$$reg),
11956 as_Register($src1$$reg),
11957 as_Register($src2$$reg),
11958 Assembler::LSR,
11959 $src3$$constant & 0x1f);
11960 %}
11961
11962 ins_pipe(ialu_reg_reg_shift);
11963 %}
11964
11965 // This pattern is automatically generated from aarch64_ad.m4
11966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11967 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11968 iRegL src1, iRegL src2,
11969 immI src3) %{
11970 match(Set dst (AddL src1 (URShiftL src2 src3)));
11971
11972 ins_cost(1.9 * INSN_COST);
11973 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11974
11975 ins_encode %{
11976 __ add(as_Register($dst$$reg),
11977 as_Register($src1$$reg),
11978 as_Register($src2$$reg),
11979 Assembler::LSR,
11980 $src3$$constant & 0x3f);
11981 %}
11982
11983 ins_pipe(ialu_reg_reg_shift);
11984 %}
11985
11986 // This pattern is automatically generated from aarch64_ad.m4
11987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11988 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11989 iRegIorL2I src1, iRegIorL2I src2,
11990 immI src3) %{
11991 match(Set dst (AddI src1 (RShiftI src2 src3)));
11992
11993 ins_cost(1.9 * INSN_COST);
11994 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11995
11996 ins_encode %{
11997 __ addw(as_Register($dst$$reg),
11998 as_Register($src1$$reg),
11999 as_Register($src2$$reg),
12000 Assembler::ASR,
12001 $src3$$constant & 0x1f);
12002 %}
12003
12004 ins_pipe(ialu_reg_reg_shift);
12005 %}
12006
12007 // This pattern is automatically generated from aarch64_ad.m4
12008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12009 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12010 iRegL src1, iRegL src2,
12011 immI src3) %{
12012 match(Set dst (AddL src1 (RShiftL src2 src3)));
12013
12014 ins_cost(1.9 * INSN_COST);
12015 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12016
12017 ins_encode %{
12018 __ add(as_Register($dst$$reg),
12019 as_Register($src1$$reg),
12020 as_Register($src2$$reg),
12021 Assembler::ASR,
12022 $src3$$constant & 0x3f);
12023 %}
12024
12025 ins_pipe(ialu_reg_reg_shift);
12026 %}
12027
12028 // This pattern is automatically generated from aarch64_ad.m4
12029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12030 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12031 iRegIorL2I src1, iRegIorL2I src2,
12032 immI src3) %{
12033 match(Set dst (AddI src1 (LShiftI src2 src3)));
12034
12035 ins_cost(1.9 * INSN_COST);
12036 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12037
12038 ins_encode %{
12039 __ addw(as_Register($dst$$reg),
12040 as_Register($src1$$reg),
12041 as_Register($src2$$reg),
12042 Assembler::LSL,
12043 $src3$$constant & 0x1f);
12044 %}
12045
12046 ins_pipe(ialu_reg_reg_shift);
12047 %}
12048
12049 // This pattern is automatically generated from aarch64_ad.m4
12050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12051 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12052 iRegL src1, iRegL src2,
12053 immI src3) %{
12054 match(Set dst (AddL src1 (LShiftL src2 src3)));
12055
12056 ins_cost(1.9 * INSN_COST);
12057 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12058
12059 ins_encode %{
12060 __ add(as_Register($dst$$reg),
12061 as_Register($src1$$reg),
12062 as_Register($src2$$reg),
12063 Assembler::LSL,
12064 $src3$$constant & 0x3f);
12065 %}
12066
12067 ins_pipe(ialu_reg_reg_shift);
12068 %}
12069
12070 // This pattern is automatically generated from aarch64_ad.m4
12071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12072 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12073 iRegIorL2I src1, iRegIorL2I src2,
12074 immI src3) %{
12075 match(Set dst (SubI src1 (URShiftI src2 src3)));
12076
12077 ins_cost(1.9 * INSN_COST);
12078 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12079
12080 ins_encode %{
12081 __ subw(as_Register($dst$$reg),
12082 as_Register($src1$$reg),
12083 as_Register($src2$$reg),
12084 Assembler::LSR,
12085 $src3$$constant & 0x1f);
12086 %}
12087
12088 ins_pipe(ialu_reg_reg_shift);
12089 %}
12090
12091 // This pattern is automatically generated from aarch64_ad.m4
12092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12093 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12094 iRegL src1, iRegL src2,
12095 immI src3) %{
12096 match(Set dst (SubL src1 (URShiftL src2 src3)));
12097
12098 ins_cost(1.9 * INSN_COST);
12099 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12100
12101 ins_encode %{
12102 __ sub(as_Register($dst$$reg),
12103 as_Register($src1$$reg),
12104 as_Register($src2$$reg),
12105 Assembler::LSR,
12106 $src3$$constant & 0x3f);
12107 %}
12108
12109 ins_pipe(ialu_reg_reg_shift);
12110 %}
12111
12112 // This pattern is automatically generated from aarch64_ad.m4
12113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12114 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12115 iRegIorL2I src1, iRegIorL2I src2,
12116 immI src3) %{
12117 match(Set dst (SubI src1 (RShiftI src2 src3)));
12118
12119 ins_cost(1.9 * INSN_COST);
12120 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12121
12122 ins_encode %{
12123 __ subw(as_Register($dst$$reg),
12124 as_Register($src1$$reg),
12125 as_Register($src2$$reg),
12126 Assembler::ASR,
12127 $src3$$constant & 0x1f);
12128 %}
12129
12130 ins_pipe(ialu_reg_reg_shift);
12131 %}
12132
12133 // This pattern is automatically generated from aarch64_ad.m4
12134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12135 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12136 iRegL src1, iRegL src2,
12137 immI src3) %{
12138 match(Set dst (SubL src1 (RShiftL src2 src3)));
12139
12140 ins_cost(1.9 * INSN_COST);
12141 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12142
12143 ins_encode %{
12144 __ sub(as_Register($dst$$reg),
12145 as_Register($src1$$reg),
12146 as_Register($src2$$reg),
12147 Assembler::ASR,
12148 $src3$$constant & 0x3f);
12149 %}
12150
12151 ins_pipe(ialu_reg_reg_shift);
12152 %}
12153
12154 // This pattern is automatically generated from aarch64_ad.m4
12155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12156 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12157 iRegIorL2I src1, iRegIorL2I src2,
12158 immI src3) %{
12159 match(Set dst (SubI src1 (LShiftI src2 src3)));
12160
12161 ins_cost(1.9 * INSN_COST);
12162 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12163
12164 ins_encode %{
12165 __ subw(as_Register($dst$$reg),
12166 as_Register($src1$$reg),
12167 as_Register($src2$$reg),
12168 Assembler::LSL,
12169 $src3$$constant & 0x1f);
12170 %}
12171
12172 ins_pipe(ialu_reg_reg_shift);
12173 %}
12174
12175 // This pattern is automatically generated from aarch64_ad.m4
12176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12177 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12178 iRegL src1, iRegL src2,
12179 immI src3) %{
12180 match(Set dst (SubL src1 (LShiftL src2 src3)));
12181
12182 ins_cost(1.9 * INSN_COST);
12183 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12184
12185 ins_encode %{
12186 __ sub(as_Register($dst$$reg),
12187 as_Register($src1$$reg),
12188 as_Register($src2$$reg),
12189 Assembler::LSL,
12190 $src3$$constant & 0x3f);
12191 %}
12192
12193 ins_pipe(ialu_reg_reg_shift);
12194 %}
12195
12196 // This pattern is automatically generated from aarch64_ad.m4
12197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12198
12199 // Shift Left followed by Shift Right.
12200 // This idiom is used by the compiler for the i2b bytecode etc.
12201 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12202 %{
12203 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12204 ins_cost(INSN_COST * 2);
12205 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12206 ins_encode %{
12207 int lshift = $lshift_count$$constant & 63;
12208 int rshift = $rshift_count$$constant & 63;
12209 int s = 63 - lshift;
12210 int r = (rshift - lshift) & 63;
12211 __ sbfm(as_Register($dst$$reg),
12212 as_Register($src$$reg),
12213 r, s);
12214 %}
12215
12216 ins_pipe(ialu_reg_shift);
12217 %}
12218
12219 // This pattern is automatically generated from aarch64_ad.m4
12220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12221
12222 // Shift Left followed by Shift Right.
12223 // This idiom is used by the compiler for the i2b bytecode etc.
12224 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12225 %{
12226 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12227 ins_cost(INSN_COST * 2);
12228 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12229 ins_encode %{
12230 int lshift = $lshift_count$$constant & 31;
12231 int rshift = $rshift_count$$constant & 31;
12232 int s = 31 - lshift;
12233 int r = (rshift - lshift) & 31;
12234 __ sbfmw(as_Register($dst$$reg),
12235 as_Register($src$$reg),
12236 r, s);
12237 %}
12238
12239 ins_pipe(ialu_reg_shift);
12240 %}
12241
12242 // This pattern is automatically generated from aarch64_ad.m4
12243 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12244
12245 // Shift Left followed by Shift Right.
12246 // This idiom is used by the compiler for the i2b bytecode etc.
12247 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12248 %{
12249 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12250 ins_cost(INSN_COST * 2);
12251 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12252 ins_encode %{
12253 int lshift = $lshift_count$$constant & 63;
12254 int rshift = $rshift_count$$constant & 63;
12255 int s = 63 - lshift;
12256 int r = (rshift - lshift) & 63;
12257 __ ubfm(as_Register($dst$$reg),
12258 as_Register($src$$reg),
12259 r, s);
12260 %}
12261
12262 ins_pipe(ialu_reg_shift);
12263 %}
12264
12265 // This pattern is automatically generated from aarch64_ad.m4
12266 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12267
12268 // Shift Left followed by Shift Right.
12269 // This idiom is used by the compiler for the i2b bytecode etc.
12270 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12271 %{
12272 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12273 ins_cost(INSN_COST * 2);
12274 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12275 ins_encode %{
12276 int lshift = $lshift_count$$constant & 31;
12277 int rshift = $rshift_count$$constant & 31;
12278 int s = 31 - lshift;
12279 int r = (rshift - lshift) & 31;
12280 __ ubfmw(as_Register($dst$$reg),
12281 as_Register($src$$reg),
12282 r, s);
12283 %}
12284
12285 ins_pipe(ialu_reg_shift);
12286 %}
12287
12288 // Bitfield extract with shift & mask
12289
12290 // This pattern is automatically generated from aarch64_ad.m4
12291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12292 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12293 %{
12294 match(Set dst (AndI (URShiftI src rshift) mask));
12295 // Make sure we are not going to exceed what ubfxw can do.
12296 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12297
12298 ins_cost(INSN_COST);
12299 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12300 ins_encode %{
12301 int rshift = $rshift$$constant & 31;
12302 intptr_t mask = $mask$$constant;
12303 int width = exact_log2(mask+1);
12304 __ ubfxw(as_Register($dst$$reg),
12305 as_Register($src$$reg), rshift, width);
12306 %}
12307 ins_pipe(ialu_reg_shift);
12308 %}
12309
12310 // This pattern is automatically generated from aarch64_ad.m4
12311 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12312 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12313 %{
12314 match(Set dst (AndL (URShiftL src rshift) mask));
12315 // Make sure we are not going to exceed what ubfx can do.
12316 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12317
12318 ins_cost(INSN_COST);
12319 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12320 ins_encode %{
12321 int rshift = $rshift$$constant & 63;
12322 intptr_t mask = $mask$$constant;
12323 int width = exact_log2_long(mask+1);
12324 __ ubfx(as_Register($dst$$reg),
12325 as_Register($src$$reg), rshift, width);
12326 %}
12327 ins_pipe(ialu_reg_shift);
12328 %}
12329
12330
12331 // This pattern is automatically generated from aarch64_ad.m4
12332 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12333
12334 // We can use ubfx when extending an And with a mask when we know mask
12335 // is positive. We know that because immI_bitmask guarantees it.
12336 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12337 %{
12338 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12339 // Make sure we are not going to exceed what ubfxw can do.
12340 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12341
12342 ins_cost(INSN_COST * 2);
12343 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12344 ins_encode %{
12345 int rshift = $rshift$$constant & 31;
12346 intptr_t mask = $mask$$constant;
12347 int width = exact_log2(mask+1);
12348 __ ubfx(as_Register($dst$$reg),
12349 as_Register($src$$reg), rshift, width);
12350 %}
12351 ins_pipe(ialu_reg_shift);
12352 %}
12353
12354
12355 // This pattern is automatically generated from aarch64_ad.m4
12356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12357
12358 // We can use ubfiz when masking by a positive number and then left shifting the result.
12359 // We know that the mask is positive because immI_bitmask guarantees it.
12360 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12361 %{
12362 match(Set dst (LShiftI (AndI src mask) lshift));
12363 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12364
12365 ins_cost(INSN_COST);
12366 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12367 ins_encode %{
12368 int lshift = $lshift$$constant & 31;
12369 intptr_t mask = $mask$$constant;
12370 int width = exact_log2(mask+1);
12371 __ ubfizw(as_Register($dst$$reg),
12372 as_Register($src$$reg), lshift, width);
12373 %}
12374 ins_pipe(ialu_reg_shift);
12375 %}
12376
12377 // This pattern is automatically generated from aarch64_ad.m4
12378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12379
12380 // We can use ubfiz when masking by a positive number and then left shifting the result.
12381 // We know that the mask is positive because immL_bitmask guarantees it.
12382 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12383 %{
12384 match(Set dst (LShiftL (AndL src mask) lshift));
12385 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12386
12387 ins_cost(INSN_COST);
12388 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12389 ins_encode %{
12390 int lshift = $lshift$$constant & 63;
12391 intptr_t mask = $mask$$constant;
12392 int width = exact_log2_long(mask+1);
12393 __ ubfiz(as_Register($dst$$reg),
12394 as_Register($src$$reg), lshift, width);
12395 %}
12396 ins_pipe(ialu_reg_shift);
12397 %}
12398
12399 // This pattern is automatically generated from aarch64_ad.m4
12400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12401
12402 // We can use ubfiz when masking by a positive number and then left shifting the result.
12403 // We know that the mask is positive because immI_bitmask guarantees it.
12404 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12405 %{
12406 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12407 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12408
12409 ins_cost(INSN_COST);
12410 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12411 ins_encode %{
12412 int lshift = $lshift$$constant & 31;
12413 intptr_t mask = $mask$$constant;
12414 int width = exact_log2(mask+1);
12415 __ ubfizw(as_Register($dst$$reg),
12416 as_Register($src$$reg), lshift, width);
12417 %}
12418 ins_pipe(ialu_reg_shift);
12419 %}
12420
12421 // This pattern is automatically generated from aarch64_ad.m4
12422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12423
12424 // We can use ubfiz when masking by a positive number and then left shifting the result.
12425 // We know that the mask is positive because immL_bitmask guarantees it.
12426 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12427 %{
12428 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12429 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12430
12431 ins_cost(INSN_COST);
12432 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12433 ins_encode %{
12434 int lshift = $lshift$$constant & 63;
12435 intptr_t mask = $mask$$constant;
12436 int width = exact_log2_long(mask+1);
12437 __ ubfiz(as_Register($dst$$reg),
12438 as_Register($src$$reg), lshift, width);
12439 %}
12440 ins_pipe(ialu_reg_shift);
12441 %}
12442
12443
12444 // This pattern is automatically generated from aarch64_ad.m4
12445 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12446
12447 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12448 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12449 %{
12450 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12451 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12452
12453 ins_cost(INSN_COST);
12454 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12455 ins_encode %{
12456 int lshift = $lshift$$constant & 63;
12457 intptr_t mask = $mask$$constant;
12458 int width = exact_log2(mask+1);
12459 __ ubfiz(as_Register($dst$$reg),
12460 as_Register($src$$reg), lshift, width);
12461 %}
12462 ins_pipe(ialu_reg_shift);
12463 %}
12464
12465 // This pattern is automatically generated from aarch64_ad.m4
12466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12467
12468 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12469 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12470 %{
12471 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12472 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12473
12474 ins_cost(INSN_COST);
12475 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12476 ins_encode %{
12477 int lshift = $lshift$$constant & 31;
12478 intptr_t mask = $mask$$constant;
12479 int width = exact_log2(mask+1);
12480 __ ubfiz(as_Register($dst$$reg),
12481 as_Register($src$$reg), lshift, width);
12482 %}
12483 ins_pipe(ialu_reg_shift);
12484 %}
12485
12486 // This pattern is automatically generated from aarch64_ad.m4
12487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12488
12489 // Can skip int2long conversions after AND with small bitmask
12490 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12491 %{
12492 match(Set dst (ConvI2L (AndI src msk)));
12493 ins_cost(INSN_COST);
12494 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12495 ins_encode %{
12496 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12497 %}
12498 ins_pipe(ialu_reg_shift);
12499 %}
12500
12501
12502 // Rotations
12503
12504 // This pattern is automatically generated from aarch64_ad.m4
12505 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12506 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12507 %{
12508 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12509 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12510
12511 ins_cost(INSN_COST);
12512 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12513
12514 ins_encode %{
12515 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12516 $rshift$$constant & 63);
12517 %}
12518 ins_pipe(ialu_reg_reg_extr);
12519 %}
12520
12521
12522 // This pattern is automatically generated from aarch64_ad.m4
12523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12524 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12525 %{
12526 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12527 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12528
12529 ins_cost(INSN_COST);
12530 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12531
12532 ins_encode %{
12533 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12534 $rshift$$constant & 31);
12535 %}
12536 ins_pipe(ialu_reg_reg_extr);
12537 %}
12538
12539
12540 // This pattern is automatically generated from aarch64_ad.m4
12541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12542 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12543 %{
12544 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12545 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12546
12547 ins_cost(INSN_COST);
12548 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12549
12550 ins_encode %{
12551 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12552 $rshift$$constant & 63);
12553 %}
12554 ins_pipe(ialu_reg_reg_extr);
12555 %}
12556
12557
12558 // This pattern is automatically generated from aarch64_ad.m4
12559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12560 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12561 %{
12562 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12563 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12564
12565 ins_cost(INSN_COST);
12566 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12567
12568 ins_encode %{
12569 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12570 $rshift$$constant & 31);
12571 %}
12572 ins_pipe(ialu_reg_reg_extr);
12573 %}
12574
12575 // This pattern is automatically generated from aarch64_ad.m4
12576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12577 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12578 %{
12579 match(Set dst (RotateRight src shift));
12580
12581 ins_cost(INSN_COST);
12582 format %{ "ror $dst, $src, $shift" %}
12583
12584 ins_encode %{
12585 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12586 $shift$$constant & 0x1f);
12587 %}
12588 ins_pipe(ialu_reg_reg_vshift);
12589 %}
12590
12591 // This pattern is automatically generated from aarch64_ad.m4
12592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12593 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12594 %{
12595 match(Set dst (RotateRight src shift));
12596
12597 ins_cost(INSN_COST);
12598 format %{ "ror $dst, $src, $shift" %}
12599
12600 ins_encode %{
12601 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12602 $shift$$constant & 0x3f);
12603 %}
12604 ins_pipe(ialu_reg_reg_vshift);
12605 %}
12606
12607 // This pattern is automatically generated from aarch64_ad.m4
12608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12609 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12610 %{
12611 match(Set dst (RotateRight src shift));
12612
12613 ins_cost(INSN_COST);
12614 format %{ "ror $dst, $src, $shift" %}
12615
12616 ins_encode %{
12617 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12618 %}
12619 ins_pipe(ialu_reg_reg_vshift);
12620 %}
12621
12622 // This pattern is automatically generated from aarch64_ad.m4
12623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12624 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12625 %{
12626 match(Set dst (RotateRight src shift));
12627
12628 ins_cost(INSN_COST);
12629 format %{ "ror $dst, $src, $shift" %}
12630
12631 ins_encode %{
12632 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12633 %}
12634 ins_pipe(ialu_reg_reg_vshift);
12635 %}
12636
12637 // This pattern is automatically generated from aarch64_ad.m4
12638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12639 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12640 %{
12641 match(Set dst (RotateLeft src shift));
12642
12643 ins_cost(INSN_COST);
12644 format %{ "rol $dst, $src, $shift" %}
12645
12646 ins_encode %{
12647 __ subw(rscratch1, zr, as_Register($shift$$reg));
12648 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12649 %}
12650 ins_pipe(ialu_reg_reg_vshift);
12651 %}
12652
12653 // This pattern is automatically generated from aarch64_ad.m4
12654 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12655 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12656 %{
12657 match(Set dst (RotateLeft src shift));
12658
12659 ins_cost(INSN_COST);
12660 format %{ "rol $dst, $src, $shift" %}
12661
12662 ins_encode %{
12663 __ subw(rscratch1, zr, as_Register($shift$$reg));
12664 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12665 %}
12666 ins_pipe(ialu_reg_reg_vshift);
12667 %}
12668
12669
12670 // Add/subtract (extended)
12671
12672 // This pattern is automatically generated from aarch64_ad.m4
12673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12674 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12675 %{
12676 match(Set dst (AddL src1 (ConvI2L src2)));
12677 ins_cost(INSN_COST);
12678 format %{ "add $dst, $src1, $src2, sxtw" %}
12679
12680 ins_encode %{
12681 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12682 as_Register($src2$$reg), ext::sxtw);
12683 %}
12684 ins_pipe(ialu_reg_reg);
12685 %}
12686
12687 // This pattern is automatically generated from aarch64_ad.m4
12688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12689 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12690 %{
12691 match(Set dst (SubL src1 (ConvI2L src2)));
12692 ins_cost(INSN_COST);
12693 format %{ "sub $dst, $src1, $src2, sxtw" %}
12694
12695 ins_encode %{
12696 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12697 as_Register($src2$$reg), ext::sxtw);
12698 %}
12699 ins_pipe(ialu_reg_reg);
12700 %}
12701
12702 // This pattern is automatically generated from aarch64_ad.m4
12703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12704 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12705 %{
12706 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12707 ins_cost(INSN_COST);
12708 format %{ "add $dst, $src1, $src2, sxth" %}
12709
12710 ins_encode %{
12711 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12712 as_Register($src2$$reg), ext::sxth);
12713 %}
12714 ins_pipe(ialu_reg_reg);
12715 %}
12716
12717 // This pattern is automatically generated from aarch64_ad.m4
12718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12719 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12720 %{
12721 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12722 ins_cost(INSN_COST);
12723 format %{ "add $dst, $src1, $src2, sxtb" %}
12724
12725 ins_encode %{
12726 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12727 as_Register($src2$$reg), ext::sxtb);
12728 %}
12729 ins_pipe(ialu_reg_reg);
12730 %}
12731
12732 // This pattern is automatically generated from aarch64_ad.m4
12733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12734 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12735 %{
12736 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12737 ins_cost(INSN_COST);
12738 format %{ "add $dst, $src1, $src2, uxtb" %}
12739
12740 ins_encode %{
12741 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12742 as_Register($src2$$reg), ext::uxtb);
12743 %}
12744 ins_pipe(ialu_reg_reg);
12745 %}
12746
12747 // This pattern is automatically generated from aarch64_ad.m4
12748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12749 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12750 %{
12751 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12752 ins_cost(INSN_COST);
12753 format %{ "add $dst, $src1, $src2, sxth" %}
12754
12755 ins_encode %{
12756 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12757 as_Register($src2$$reg), ext::sxth);
12758 %}
12759 ins_pipe(ialu_reg_reg);
12760 %}
12761
12762 // This pattern is automatically generated from aarch64_ad.m4
12763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12764 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12765 %{
12766 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12767 ins_cost(INSN_COST);
12768 format %{ "add $dst, $src1, $src2, sxtw" %}
12769
12770 ins_encode %{
12771 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12772 as_Register($src2$$reg), ext::sxtw);
12773 %}
12774 ins_pipe(ialu_reg_reg);
12775 %}
12776
12777 // This pattern is automatically generated from aarch64_ad.m4
12778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12779 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12780 %{
12781 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12782 ins_cost(INSN_COST);
12783 format %{ "add $dst, $src1, $src2, sxtb" %}
12784
12785 ins_encode %{
12786 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12787 as_Register($src2$$reg), ext::sxtb);
12788 %}
12789 ins_pipe(ialu_reg_reg);
12790 %}
12791
12792 // This pattern is automatically generated from aarch64_ad.m4
12793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12794 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12795 %{
12796 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12797 ins_cost(INSN_COST);
12798 format %{ "add $dst, $src1, $src2, uxtb" %}
12799
12800 ins_encode %{
12801 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12802 as_Register($src2$$reg), ext::uxtb);
12803 %}
12804 ins_pipe(ialu_reg_reg);
12805 %}
12806
12807 // This pattern is automatically generated from aarch64_ad.m4
12808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12809 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12810 %{
12811 match(Set dst (AddI src1 (AndI src2 mask)));
12812 ins_cost(INSN_COST);
12813 format %{ "addw $dst, $src1, $src2, uxtb" %}
12814
12815 ins_encode %{
12816 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12817 as_Register($src2$$reg), ext::uxtb);
12818 %}
12819 ins_pipe(ialu_reg_reg);
12820 %}
12821
12822 // This pattern is automatically generated from aarch64_ad.m4
12823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12824 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12825 %{
12826 match(Set dst (AddI src1 (AndI src2 mask)));
12827 ins_cost(INSN_COST);
12828 format %{ "addw $dst, $src1, $src2, uxth" %}
12829
12830 ins_encode %{
12831 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12832 as_Register($src2$$reg), ext::uxth);
12833 %}
12834 ins_pipe(ialu_reg_reg);
12835 %}
12836
12837 // This pattern is automatically generated from aarch64_ad.m4
12838 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12839 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12840 %{
12841 match(Set dst (AddL src1 (AndL src2 mask)));
12842 ins_cost(INSN_COST);
12843 format %{ "add $dst, $src1, $src2, uxtb" %}
12844
12845 ins_encode %{
12846 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12847 as_Register($src2$$reg), ext::uxtb);
12848 %}
12849 ins_pipe(ialu_reg_reg);
12850 %}
12851
12852 // This pattern is automatically generated from aarch64_ad.m4
12853 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12854 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12855 %{
12856 match(Set dst (AddL src1 (AndL src2 mask)));
12857 ins_cost(INSN_COST);
12858 format %{ "add $dst, $src1, $src2, uxth" %}
12859
12860 ins_encode %{
12861 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12862 as_Register($src2$$reg), ext::uxth);
12863 %}
12864 ins_pipe(ialu_reg_reg);
12865 %}
12866
12867 // This pattern is automatically generated from aarch64_ad.m4
12868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12869 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12870 %{
12871 match(Set dst (AddL src1 (AndL src2 mask)));
12872 ins_cost(INSN_COST);
12873 format %{ "add $dst, $src1, $src2, uxtw" %}
12874
12875 ins_encode %{
12876 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12877 as_Register($src2$$reg), ext::uxtw);
12878 %}
12879 ins_pipe(ialu_reg_reg);
12880 %}
12881
12882 // This pattern is automatically generated from aarch64_ad.m4
12883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12884 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12885 %{
12886 match(Set dst (SubI src1 (AndI src2 mask)));
12887 ins_cost(INSN_COST);
12888 format %{ "subw $dst, $src1, $src2, uxtb" %}
12889
12890 ins_encode %{
12891 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12892 as_Register($src2$$reg), ext::uxtb);
12893 %}
12894 ins_pipe(ialu_reg_reg);
12895 %}
12896
12897 // This pattern is automatically generated from aarch64_ad.m4
12898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12899 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12900 %{
12901 match(Set dst (SubI src1 (AndI src2 mask)));
12902 ins_cost(INSN_COST);
12903 format %{ "subw $dst, $src1, $src2, uxth" %}
12904
12905 ins_encode %{
12906 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12907 as_Register($src2$$reg), ext::uxth);
12908 %}
12909 ins_pipe(ialu_reg_reg);
12910 %}
12911
12912 // This pattern is automatically generated from aarch64_ad.m4
12913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12914 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12915 %{
12916 match(Set dst (SubL src1 (AndL src2 mask)));
12917 ins_cost(INSN_COST);
12918 format %{ "sub $dst, $src1, $src2, uxtb" %}
12919
12920 ins_encode %{
12921 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12922 as_Register($src2$$reg), ext::uxtb);
12923 %}
12924 ins_pipe(ialu_reg_reg);
12925 %}
12926
12927 // This pattern is automatically generated from aarch64_ad.m4
12928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12929 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12930 %{
12931 match(Set dst (SubL src1 (AndL src2 mask)));
12932 ins_cost(INSN_COST);
12933 format %{ "sub $dst, $src1, $src2, uxth" %}
12934
12935 ins_encode %{
12936 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12937 as_Register($src2$$reg), ext::uxth);
12938 %}
12939 ins_pipe(ialu_reg_reg);
12940 %}
12941
12942 // This pattern is automatically generated from aarch64_ad.m4
12943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12944 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12945 %{
12946 match(Set dst (SubL src1 (AndL src2 mask)));
12947 ins_cost(INSN_COST);
12948 format %{ "sub $dst, $src1, $src2, uxtw" %}
12949
12950 ins_encode %{
12951 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12952 as_Register($src2$$reg), ext::uxtw);
12953 %}
12954 ins_pipe(ialu_reg_reg);
12955 %}
12956
12957
12958 // This pattern is automatically generated from aarch64_ad.m4
12959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12960 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12961 %{
12962 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12963 ins_cost(1.9 * INSN_COST);
12964 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12965
12966 ins_encode %{
12967 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12968 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12969 %}
12970 ins_pipe(ialu_reg_reg_shift);
12971 %}
12972
12973 // This pattern is automatically generated from aarch64_ad.m4
12974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12975 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12976 %{
12977 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12978 ins_cost(1.9 * INSN_COST);
12979 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12980
12981 ins_encode %{
12982 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12983 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12984 %}
12985 ins_pipe(ialu_reg_reg_shift);
12986 %}
12987
12988 // This pattern is automatically generated from aarch64_ad.m4
12989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12990 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12991 %{
12992 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12993 ins_cost(1.9 * INSN_COST);
12994 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12995
12996 ins_encode %{
12997 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12998 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12999 %}
13000 ins_pipe(ialu_reg_reg_shift);
13001 %}
13002
13003 // This pattern is automatically generated from aarch64_ad.m4
13004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13005 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13006 %{
13007 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13008 ins_cost(1.9 * INSN_COST);
13009 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13010
13011 ins_encode %{
13012 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13013 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13014 %}
13015 ins_pipe(ialu_reg_reg_shift);
13016 %}
13017
13018 // This pattern is automatically generated from aarch64_ad.m4
13019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13020 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13021 %{
13022 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13023 ins_cost(1.9 * INSN_COST);
13024 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13025
13026 ins_encode %{
13027 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13028 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13029 %}
13030 ins_pipe(ialu_reg_reg_shift);
13031 %}
13032
13033 // This pattern is automatically generated from aarch64_ad.m4
13034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13035 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13036 %{
13037 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13038 ins_cost(1.9 * INSN_COST);
13039 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13040
13041 ins_encode %{
13042 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13043 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13044 %}
13045 ins_pipe(ialu_reg_reg_shift);
13046 %}
13047
13048 // This pattern is automatically generated from aarch64_ad.m4
13049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13050 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13051 %{
13052 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13053 ins_cost(1.9 * INSN_COST);
13054 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13055
13056 ins_encode %{
13057 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13058 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13059 %}
13060 ins_pipe(ialu_reg_reg_shift);
13061 %}
13062
13063 // This pattern is automatically generated from aarch64_ad.m4
13064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13065 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13066 %{
13067 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13068 ins_cost(1.9 * INSN_COST);
13069 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13070
13071 ins_encode %{
13072 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13073 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13074 %}
13075 ins_pipe(ialu_reg_reg_shift);
13076 %}
13077
13078 // This pattern is automatically generated from aarch64_ad.m4
13079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13080 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13081 %{
13082 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13083 ins_cost(1.9 * INSN_COST);
13084 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13085
13086 ins_encode %{
13087 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13088 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13089 %}
13090 ins_pipe(ialu_reg_reg_shift);
13091 %}
13092
13093 // This pattern is automatically generated from aarch64_ad.m4
13094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13095 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13096 %{
13097 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13098 ins_cost(1.9 * INSN_COST);
13099 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13100
13101 ins_encode %{
13102 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13103 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13104 %}
13105 ins_pipe(ialu_reg_reg_shift);
13106 %}
13107
13108 // This pattern is automatically generated from aarch64_ad.m4
13109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13110 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13111 %{
13112 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13113 ins_cost(1.9 * INSN_COST);
13114 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13115
13116 ins_encode %{
13117 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13118 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13119 %}
13120 ins_pipe(ialu_reg_reg_shift);
13121 %}
13122
13123 // This pattern is automatically generated from aarch64_ad.m4
13124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13125 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13126 %{
13127 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13128 ins_cost(1.9 * INSN_COST);
13129 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13130
13131 ins_encode %{
13132 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13133 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13134 %}
13135 ins_pipe(ialu_reg_reg_shift);
13136 %}
13137
13138 // This pattern is automatically generated from aarch64_ad.m4
13139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13140 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13141 %{
13142 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13143 ins_cost(1.9 * INSN_COST);
13144 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13145
13146 ins_encode %{
13147 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13148 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13149 %}
13150 ins_pipe(ialu_reg_reg_shift);
13151 %}
13152
13153 // This pattern is automatically generated from aarch64_ad.m4
13154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13155 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13156 %{
13157 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13158 ins_cost(1.9 * INSN_COST);
13159 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13160
13161 ins_encode %{
13162 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13163 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13164 %}
13165 ins_pipe(ialu_reg_reg_shift);
13166 %}
13167
13168 // This pattern is automatically generated from aarch64_ad.m4
13169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13170 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13171 %{
13172 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13173 ins_cost(1.9 * INSN_COST);
13174 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13175
13176 ins_encode %{
13177 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13178 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13179 %}
13180 ins_pipe(ialu_reg_reg_shift);
13181 %}
13182
13183 // This pattern is automatically generated from aarch64_ad.m4
13184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13185 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13186 %{
13187 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13188 ins_cost(1.9 * INSN_COST);
13189 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13190
13191 ins_encode %{
13192 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13193 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13194 %}
13195 ins_pipe(ialu_reg_reg_shift);
13196 %}
13197
13198 // This pattern is automatically generated from aarch64_ad.m4
13199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13200 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13201 %{
13202 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13203 ins_cost(1.9 * INSN_COST);
13204 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13205
13206 ins_encode %{
13207 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13208 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13209 %}
13210 ins_pipe(ialu_reg_reg_shift);
13211 %}
13212
13213 // This pattern is automatically generated from aarch64_ad.m4
13214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13215 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13216 %{
13217 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13218 ins_cost(1.9 * INSN_COST);
13219 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13220
13221 ins_encode %{
13222 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13223 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13224 %}
13225 ins_pipe(ialu_reg_reg_shift);
13226 %}
13227
13228 // This pattern is automatically generated from aarch64_ad.m4
13229 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13230 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13231 %{
13232 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13233 ins_cost(1.9 * INSN_COST);
13234 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13235
13236 ins_encode %{
13237 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13238 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13239 %}
13240 ins_pipe(ialu_reg_reg_shift);
13241 %}
13242
13243 // This pattern is automatically generated from aarch64_ad.m4
13244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13245 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13246 %{
13247 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13248 ins_cost(1.9 * INSN_COST);
13249 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13250
13251 ins_encode %{
13252 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13253 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13254 %}
13255 ins_pipe(ialu_reg_reg_shift);
13256 %}
13257
13258 // This pattern is automatically generated from aarch64_ad.m4
13259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13260 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13261 %{
13262 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13263 ins_cost(1.9 * INSN_COST);
13264 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13265
13266 ins_encode %{
13267 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13268 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13269 %}
13270 ins_pipe(ialu_reg_reg_shift);
13271 %}
13272
13273 // This pattern is automatically generated from aarch64_ad.m4
13274 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13275 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13276 %{
13277 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13278 ins_cost(1.9 * INSN_COST);
13279 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13280
13281 ins_encode %{
13282 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13283 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13284 %}
13285 ins_pipe(ialu_reg_reg_shift);
13286 %}
13287
13288 // This pattern is automatically generated from aarch64_ad.m4
13289 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13290 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13291 %{
13292 effect(DEF dst, USE src1, USE src2, USE cr);
13293 ins_cost(INSN_COST * 2);
13294 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13295
13296 ins_encode %{
13297 __ cselw($dst$$Register,
13298 $src1$$Register,
13299 $src2$$Register,
13300 Assembler::LT);
13301 %}
13302 ins_pipe(icond_reg_reg);
13303 %}
13304
13305 // This pattern is automatically generated from aarch64_ad.m4
13306 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13307 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13308 %{
13309 effect(DEF dst, USE src1, USE src2, USE cr);
13310 ins_cost(INSN_COST * 2);
13311 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13312
13313 ins_encode %{
13314 __ cselw($dst$$Register,
13315 $src1$$Register,
13316 $src2$$Register,
13317 Assembler::GT);
13318 %}
13319 ins_pipe(icond_reg_reg);
13320 %}
13321
13322 // This pattern is automatically generated from aarch64_ad.m4
13323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13324 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13325 %{
13326 effect(DEF dst, USE src1, USE cr);
13327 ins_cost(INSN_COST * 2);
13328 format %{ "cselw $dst, $src1, zr lt\t" %}
13329
13330 ins_encode %{
13331 __ cselw($dst$$Register,
13332 $src1$$Register,
13333 zr,
13334 Assembler::LT);
13335 %}
13336 ins_pipe(icond_reg);
13337 %}
13338
13339 // This pattern is automatically generated from aarch64_ad.m4
13340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13341 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13342 %{
13343 effect(DEF dst, USE src1, USE cr);
13344 ins_cost(INSN_COST * 2);
13345 format %{ "cselw $dst, $src1, zr gt\t" %}
13346
13347 ins_encode %{
13348 __ cselw($dst$$Register,
13349 $src1$$Register,
13350 zr,
13351 Assembler::GT);
13352 %}
13353 ins_pipe(icond_reg);
13354 %}
13355
13356 // This pattern is automatically generated from aarch64_ad.m4
13357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13358 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13359 %{
13360 effect(DEF dst, USE src1, USE cr);
13361 ins_cost(INSN_COST * 2);
13362 format %{ "csincw $dst, $src1, zr le\t" %}
13363
13364 ins_encode %{
13365 __ csincw($dst$$Register,
13366 $src1$$Register,
13367 zr,
13368 Assembler::LE);
13369 %}
13370 ins_pipe(icond_reg);
13371 %}
13372
13373 // This pattern is automatically generated from aarch64_ad.m4
13374 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13375 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13376 %{
13377 effect(DEF dst, USE src1, USE cr);
13378 ins_cost(INSN_COST * 2);
13379 format %{ "csincw $dst, $src1, zr gt\t" %}
13380
13381 ins_encode %{
13382 __ csincw($dst$$Register,
13383 $src1$$Register,
13384 zr,
13385 Assembler::GT);
13386 %}
13387 ins_pipe(icond_reg);
13388 %}
13389
13390 // This pattern is automatically generated from aarch64_ad.m4
13391 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13392 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13393 %{
13394 effect(DEF dst, USE src1, USE cr);
13395 ins_cost(INSN_COST * 2);
13396 format %{ "csinvw $dst, $src1, zr lt\t" %}
13397
13398 ins_encode %{
13399 __ csinvw($dst$$Register,
13400 $src1$$Register,
13401 zr,
13402 Assembler::LT);
13403 %}
13404 ins_pipe(icond_reg);
13405 %}
13406
13407 // This pattern is automatically generated from aarch64_ad.m4
13408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13409 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13410 %{
13411 effect(DEF dst, USE src1, USE cr);
13412 ins_cost(INSN_COST * 2);
13413 format %{ "csinvw $dst, $src1, zr ge\t" %}
13414
13415 ins_encode %{
13416 __ csinvw($dst$$Register,
13417 $src1$$Register,
13418 zr,
13419 Assembler::GE);
13420 %}
13421 ins_pipe(icond_reg);
13422 %}
13423
13424 // This pattern is automatically generated from aarch64_ad.m4
13425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13426 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13427 %{
13428 match(Set dst (MinI src imm));
13429 ins_cost(INSN_COST * 3);
13430 expand %{
13431 rFlagsReg cr;
13432 compI_reg_imm0(cr, src);
13433 cmovI_reg_imm0_lt(dst, src, cr);
13434 %}
13435 %}
13436
13437 // This pattern is automatically generated from aarch64_ad.m4
13438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13439 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13440 %{
13441 match(Set dst (MinI imm src));
13442 ins_cost(INSN_COST * 3);
13443 expand %{
13444 rFlagsReg cr;
13445 compI_reg_imm0(cr, src);
13446 cmovI_reg_imm0_lt(dst, src, cr);
13447 %}
13448 %}
13449
13450 // This pattern is automatically generated from aarch64_ad.m4
13451 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13452 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13453 %{
13454 match(Set dst (MinI src imm));
13455 ins_cost(INSN_COST * 3);
13456 expand %{
13457 rFlagsReg cr;
13458 compI_reg_imm0(cr, src);
13459 cmovI_reg_imm1_le(dst, src, cr);
13460 %}
13461 %}
13462
13463 // This pattern is automatically generated from aarch64_ad.m4
13464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13465 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13466 %{
13467 match(Set dst (MinI imm src));
13468 ins_cost(INSN_COST * 3);
13469 expand %{
13470 rFlagsReg cr;
13471 compI_reg_imm0(cr, src);
13472 cmovI_reg_imm1_le(dst, src, cr);
13473 %}
13474 %}
13475
13476 // This pattern is automatically generated from aarch64_ad.m4
13477 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13478 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13479 %{
13480 match(Set dst (MinI src imm));
13481 ins_cost(INSN_COST * 3);
13482 expand %{
13483 rFlagsReg cr;
13484 compI_reg_imm0(cr, src);
13485 cmovI_reg_immM1_lt(dst, src, cr);
13486 %}
13487 %}
13488
13489 // This pattern is automatically generated from aarch64_ad.m4
13490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13491 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13492 %{
13493 match(Set dst (MinI imm src));
13494 ins_cost(INSN_COST * 3);
13495 expand %{
13496 rFlagsReg cr;
13497 compI_reg_imm0(cr, src);
13498 cmovI_reg_immM1_lt(dst, src, cr);
13499 %}
13500 %}
13501
13502 // This pattern is automatically generated from aarch64_ad.m4
13503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13504 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13505 %{
13506 match(Set dst (MaxI src imm));
13507 ins_cost(INSN_COST * 3);
13508 expand %{
13509 rFlagsReg cr;
13510 compI_reg_imm0(cr, src);
13511 cmovI_reg_imm0_gt(dst, src, cr);
13512 %}
13513 %}
13514
13515 // This pattern is automatically generated from aarch64_ad.m4
13516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13517 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13518 %{
13519 match(Set dst (MaxI imm src));
13520 ins_cost(INSN_COST * 3);
13521 expand %{
13522 rFlagsReg cr;
13523 compI_reg_imm0(cr, src);
13524 cmovI_reg_imm0_gt(dst, src, cr);
13525 %}
13526 %}
13527
13528 // This pattern is automatically generated from aarch64_ad.m4
13529 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13530 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13531 %{
13532 match(Set dst (MaxI src imm));
13533 ins_cost(INSN_COST * 3);
13534 expand %{
13535 rFlagsReg cr;
13536 compI_reg_imm0(cr, src);
13537 cmovI_reg_imm1_gt(dst, src, cr);
13538 %}
13539 %}
13540
13541 // This pattern is automatically generated from aarch64_ad.m4
13542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13543 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13544 %{
13545 match(Set dst (MaxI imm src));
13546 ins_cost(INSN_COST * 3);
13547 expand %{
13548 rFlagsReg cr;
13549 compI_reg_imm0(cr, src);
13550 cmovI_reg_imm1_gt(dst, src, cr);
13551 %}
13552 %}
13553
13554 // This pattern is automatically generated from aarch64_ad.m4
13555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13556 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13557 %{
13558 match(Set dst (MaxI src imm));
13559 ins_cost(INSN_COST * 3);
13560 expand %{
13561 rFlagsReg cr;
13562 compI_reg_imm0(cr, src);
13563 cmovI_reg_immM1_ge(dst, src, cr);
13564 %}
13565 %}
13566
13567 // This pattern is automatically generated from aarch64_ad.m4
13568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13569 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13570 %{
13571 match(Set dst (MaxI imm src));
13572 ins_cost(INSN_COST * 3);
13573 expand %{
13574 rFlagsReg cr;
13575 compI_reg_imm0(cr, src);
13576 cmovI_reg_immM1_ge(dst, src, cr);
13577 %}
13578 %}
13579
13580 // This pattern is automatically generated from aarch64_ad.m4
13581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13582 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13583 %{
13584 match(Set dst (ReverseI src));
13585 ins_cost(INSN_COST);
13586 format %{ "rbitw $dst, $src" %}
13587 ins_encode %{
13588 __ rbitw($dst$$Register, $src$$Register);
13589 %}
13590 ins_pipe(ialu_reg);
13591 %}
13592
13593 // This pattern is automatically generated from aarch64_ad.m4
13594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13595 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13596 %{
13597 match(Set dst (ReverseL src));
13598 ins_cost(INSN_COST);
13599 format %{ "rbit $dst, $src" %}
13600 ins_encode %{
13601 __ rbit($dst$$Register, $src$$Register);
13602 %}
13603 ins_pipe(ialu_reg);
13604 %}
13605
13606
13607 // END This section of the file is automatically generated. Do not edit --------------
13608
13609
13610 // ============================================================================
13611 // Floating Point Arithmetic Instructions
13612
13613 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13614 match(Set dst (AddF src1 src2));
13615
13616 ins_cost(INSN_COST * 5);
13617 format %{ "fadds $dst, $src1, $src2" %}
13618
13619 ins_encode %{
13620 __ fadds(as_FloatRegister($dst$$reg),
13621 as_FloatRegister($src1$$reg),
13622 as_FloatRegister($src2$$reg));
13623 %}
13624
13625 ins_pipe(fp_dop_reg_reg_s);
13626 %}
13627
13628 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13629 match(Set dst (AddD src1 src2));
13630
13631 ins_cost(INSN_COST * 5);
13632 format %{ "faddd $dst, $src1, $src2" %}
13633
13634 ins_encode %{
13635 __ faddd(as_FloatRegister($dst$$reg),
13636 as_FloatRegister($src1$$reg),
13637 as_FloatRegister($src2$$reg));
13638 %}
13639
13640 ins_pipe(fp_dop_reg_reg_d);
13641 %}
13642
13643 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13644 match(Set dst (SubF src1 src2));
13645
13646 ins_cost(INSN_COST * 5);
13647 format %{ "fsubs $dst, $src1, $src2" %}
13648
13649 ins_encode %{
13650 __ fsubs(as_FloatRegister($dst$$reg),
13651 as_FloatRegister($src1$$reg),
13652 as_FloatRegister($src2$$reg));
13653 %}
13654
13655 ins_pipe(fp_dop_reg_reg_s);
13656 %}
13657
13658 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13659 match(Set dst (SubD src1 src2));
13660
13661 ins_cost(INSN_COST * 5);
13662 format %{ "fsubd $dst, $src1, $src2" %}
13663
13664 ins_encode %{
13665 __ fsubd(as_FloatRegister($dst$$reg),
13666 as_FloatRegister($src1$$reg),
13667 as_FloatRegister($src2$$reg));
13668 %}
13669
13670 ins_pipe(fp_dop_reg_reg_d);
13671 %}
13672
13673 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13674 match(Set dst (MulF src1 src2));
13675
13676 ins_cost(INSN_COST * 6);
13677 format %{ "fmuls $dst, $src1, $src2" %}
13678
13679 ins_encode %{
13680 __ fmuls(as_FloatRegister($dst$$reg),
13681 as_FloatRegister($src1$$reg),
13682 as_FloatRegister($src2$$reg));
13683 %}
13684
13685 ins_pipe(fp_dop_reg_reg_s);
13686 %}
13687
13688 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13689 match(Set dst (MulD src1 src2));
13690
13691 ins_cost(INSN_COST * 6);
13692 format %{ "fmuld $dst, $src1, $src2" %}
13693
13694 ins_encode %{
13695 __ fmuld(as_FloatRegister($dst$$reg),
13696 as_FloatRegister($src1$$reg),
13697 as_FloatRegister($src2$$reg));
13698 %}
13699
13700 ins_pipe(fp_dop_reg_reg_d);
13701 %}
13702
13703 // src1 * src2 + src3
13704 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13705 match(Set dst (FmaF src3 (Binary src1 src2)));
13706
13707 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13708
13709 ins_encode %{
13710 assert(UseFMA, "Needs FMA instructions support.");
13711 __ fmadds(as_FloatRegister($dst$$reg),
13712 as_FloatRegister($src1$$reg),
13713 as_FloatRegister($src2$$reg),
13714 as_FloatRegister($src3$$reg));
13715 %}
13716
13717 ins_pipe(pipe_class_default);
13718 %}
13719
13720 // src1 * src2 + src3
13721 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13722 match(Set dst (FmaD src3 (Binary src1 src2)));
13723
13724 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13725
13726 ins_encode %{
13727 assert(UseFMA, "Needs FMA instructions support.");
13728 __ fmaddd(as_FloatRegister($dst$$reg),
13729 as_FloatRegister($src1$$reg),
13730 as_FloatRegister($src2$$reg),
13731 as_FloatRegister($src3$$reg));
13732 %}
13733
13734 ins_pipe(pipe_class_default);
13735 %}
13736
13737 // src1 * (-src2) + src3
13738 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13739 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13740 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13741
13742 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13743
13744 ins_encode %{
13745 assert(UseFMA, "Needs FMA instructions support.");
13746 __ fmsubs(as_FloatRegister($dst$$reg),
13747 as_FloatRegister($src1$$reg),
13748 as_FloatRegister($src2$$reg),
13749 as_FloatRegister($src3$$reg));
13750 %}
13751
13752 ins_pipe(pipe_class_default);
13753 %}
13754
13755 // src1 * (-src2) + src3
13756 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13757 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13758 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13759
13760 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13761
13762 ins_encode %{
13763 assert(UseFMA, "Needs FMA instructions support.");
13764 __ fmsubd(as_FloatRegister($dst$$reg),
13765 as_FloatRegister($src1$$reg),
13766 as_FloatRegister($src2$$reg),
13767 as_FloatRegister($src3$$reg));
13768 %}
13769
13770 ins_pipe(pipe_class_default);
13771 %}
13772
13773 // src1 * (-src2) - src3
13774 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13775 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13776 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13777
13778 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13779
13780 ins_encode %{
13781 assert(UseFMA, "Needs FMA instructions support.");
13782 __ fnmadds(as_FloatRegister($dst$$reg),
13783 as_FloatRegister($src1$$reg),
13784 as_FloatRegister($src2$$reg),
13785 as_FloatRegister($src3$$reg));
13786 %}
13787
13788 ins_pipe(pipe_class_default);
13789 %}
13790
13791 // src1 * (-src2) - src3
13792 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13793 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13794 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13795
13796 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13797
13798 ins_encode %{
13799 assert(UseFMA, "Needs FMA instructions support.");
13800 __ fnmaddd(as_FloatRegister($dst$$reg),
13801 as_FloatRegister($src1$$reg),
13802 as_FloatRegister($src2$$reg),
13803 as_FloatRegister($src3$$reg));
13804 %}
13805
13806 ins_pipe(pipe_class_default);
13807 %}
13808
13809 // src1 * src2 - src3
13810 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13811 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13812
13813 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13814
13815 ins_encode %{
13816 assert(UseFMA, "Needs FMA instructions support.");
13817 __ fnmsubs(as_FloatRegister($dst$$reg),
13818 as_FloatRegister($src1$$reg),
13819 as_FloatRegister($src2$$reg),
13820 as_FloatRegister($src3$$reg));
13821 %}
13822
13823 ins_pipe(pipe_class_default);
13824 %}
13825
13826 // src1 * src2 - src3
13827 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13828 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13829
13830 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13831
13832 ins_encode %{
13833 assert(UseFMA, "Needs FMA instructions support.");
13834 // n.b. insn name should be fnmsubd
13835 __ fnmsub(as_FloatRegister($dst$$reg),
13836 as_FloatRegister($src1$$reg),
13837 as_FloatRegister($src2$$reg),
13838 as_FloatRegister($src3$$reg));
13839 %}
13840
13841 ins_pipe(pipe_class_default);
13842 %}
13843
13844
13845 // Math.max(FF)F
13846 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13847 match(Set dst (MaxF src1 src2));
13848
13849 format %{ "fmaxs $dst, $src1, $src2" %}
13850 ins_encode %{
13851 __ fmaxs(as_FloatRegister($dst$$reg),
13852 as_FloatRegister($src1$$reg),
13853 as_FloatRegister($src2$$reg));
13854 %}
13855
13856 ins_pipe(fp_dop_reg_reg_s);
13857 %}
13858
13859 // Math.min(FF)F
13860 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13861 match(Set dst (MinF src1 src2));
13862
13863 format %{ "fmins $dst, $src1, $src2" %}
13864 ins_encode %{
13865 __ fmins(as_FloatRegister($dst$$reg),
13866 as_FloatRegister($src1$$reg),
13867 as_FloatRegister($src2$$reg));
13868 %}
13869
13870 ins_pipe(fp_dop_reg_reg_s);
13871 %}
13872
13873 // Math.max(DD)D
13874 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13875 match(Set dst (MaxD src1 src2));
13876
13877 format %{ "fmaxd $dst, $src1, $src2" %}
13878 ins_encode %{
13879 __ fmaxd(as_FloatRegister($dst$$reg),
13880 as_FloatRegister($src1$$reg),
13881 as_FloatRegister($src2$$reg));
13882 %}
13883
13884 ins_pipe(fp_dop_reg_reg_d);
13885 %}
13886
13887 // Math.min(DD)D
13888 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13889 match(Set dst (MinD src1 src2));
13890
13891 format %{ "fmind $dst, $src1, $src2" %}
13892 ins_encode %{
13893 __ fmind(as_FloatRegister($dst$$reg),
13894 as_FloatRegister($src1$$reg),
13895 as_FloatRegister($src2$$reg));
13896 %}
13897
13898 ins_pipe(fp_dop_reg_reg_d);
13899 %}
13900
13901
13902 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13903 match(Set dst (DivF src1 src2));
13904
13905 ins_cost(INSN_COST * 18);
13906 format %{ "fdivs $dst, $src1, $src2" %}
13907
13908 ins_encode %{
13909 __ fdivs(as_FloatRegister($dst$$reg),
13910 as_FloatRegister($src1$$reg),
13911 as_FloatRegister($src2$$reg));
13912 %}
13913
13914 ins_pipe(fp_div_s);
13915 %}
13916
13917 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13918 match(Set dst (DivD src1 src2));
13919
13920 ins_cost(INSN_COST * 32);
13921 format %{ "fdivd $dst, $src1, $src2" %}
13922
13923 ins_encode %{
13924 __ fdivd(as_FloatRegister($dst$$reg),
13925 as_FloatRegister($src1$$reg),
13926 as_FloatRegister($src2$$reg));
13927 %}
13928
13929 ins_pipe(fp_div_d);
13930 %}
13931
13932 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13933 match(Set dst (NegF src));
13934
13935 ins_cost(INSN_COST * 3);
13936 format %{ "fneg $dst, $src" %}
13937
13938 ins_encode %{
13939 __ fnegs(as_FloatRegister($dst$$reg),
13940 as_FloatRegister($src$$reg));
13941 %}
13942
13943 ins_pipe(fp_uop_s);
13944 %}
13945
13946 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13947 match(Set dst (NegD src));
13948
13949 ins_cost(INSN_COST * 3);
13950 format %{ "fnegd $dst, $src" %}
13951
13952 ins_encode %{
13953 __ fnegd(as_FloatRegister($dst$$reg),
13954 as_FloatRegister($src$$reg));
13955 %}
13956
13957 ins_pipe(fp_uop_d);
13958 %}
13959
13960 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13961 %{
13962 match(Set dst (AbsI src));
13963
13964 effect(KILL cr);
13965 ins_cost(INSN_COST * 2);
13966 format %{ "cmpw $src, zr\n\t"
13967 "cnegw $dst, $src, Assembler::LT\t# int abs"
13968 %}
13969
13970 ins_encode %{
13971 __ cmpw(as_Register($src$$reg), zr);
13972 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13973 %}
13974 ins_pipe(pipe_class_default);
13975 %}
13976
13977 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13978 %{
13979 match(Set dst (AbsL src));
13980
13981 effect(KILL cr);
13982 ins_cost(INSN_COST * 2);
13983 format %{ "cmp $src, zr\n\t"
13984 "cneg $dst, $src, Assembler::LT\t# long abs"
13985 %}
13986
13987 ins_encode %{
13988 __ cmp(as_Register($src$$reg), zr);
13989 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13990 %}
13991 ins_pipe(pipe_class_default);
13992 %}
13993
13994 instruct absF_reg(vRegF dst, vRegF src) %{
13995 match(Set dst (AbsF src));
13996
13997 ins_cost(INSN_COST * 3);
13998 format %{ "fabss $dst, $src" %}
13999 ins_encode %{
14000 __ fabss(as_FloatRegister($dst$$reg),
14001 as_FloatRegister($src$$reg));
14002 %}
14003
14004 ins_pipe(fp_uop_s);
14005 %}
14006
14007 instruct absD_reg(vRegD dst, vRegD src) %{
14008 match(Set dst (AbsD src));
14009
14010 ins_cost(INSN_COST * 3);
14011 format %{ "fabsd $dst, $src" %}
14012 ins_encode %{
14013 __ fabsd(as_FloatRegister($dst$$reg),
14014 as_FloatRegister($src$$reg));
14015 %}
14016
14017 ins_pipe(fp_uop_d);
14018 %}
14019
14020 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14021 match(Set dst (AbsF (SubF src1 src2)));
14022
14023 ins_cost(INSN_COST * 3);
14024 format %{ "fabds $dst, $src1, $src2" %}
14025 ins_encode %{
14026 __ fabds(as_FloatRegister($dst$$reg),
14027 as_FloatRegister($src1$$reg),
14028 as_FloatRegister($src2$$reg));
14029 %}
14030
14031 ins_pipe(fp_uop_s);
14032 %}
14033
14034 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14035 match(Set dst (AbsD (SubD src1 src2)));
14036
14037 ins_cost(INSN_COST * 3);
14038 format %{ "fabdd $dst, $src1, $src2" %}
14039 ins_encode %{
14040 __ fabdd(as_FloatRegister($dst$$reg),
14041 as_FloatRegister($src1$$reg),
14042 as_FloatRegister($src2$$reg));
14043 %}
14044
14045 ins_pipe(fp_uop_d);
14046 %}
14047
14048 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14049 match(Set dst (SqrtD src));
14050
14051 ins_cost(INSN_COST * 50);
14052 format %{ "fsqrtd $dst, $src" %}
14053 ins_encode %{
14054 __ fsqrtd(as_FloatRegister($dst$$reg),
14055 as_FloatRegister($src$$reg));
14056 %}
14057
14058 ins_pipe(fp_div_s);
14059 %}
14060
14061 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14062 match(Set dst (SqrtF src));
14063
14064 ins_cost(INSN_COST * 50);
14065 format %{ "fsqrts $dst, $src" %}
14066 ins_encode %{
14067 __ fsqrts(as_FloatRegister($dst$$reg),
14068 as_FloatRegister($src$$reg));
14069 %}
14070
14071 ins_pipe(fp_div_d);
14072 %}
14073
14074 // Math.rint, floor, ceil
14075 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14076 match(Set dst (RoundDoubleMode src rmode));
14077 format %{ "frint $dst, $src, $rmode" %}
14078 ins_encode %{
14079 switch ($rmode$$constant) {
14080 case RoundDoubleModeNode::rmode_rint:
14081 __ frintnd(as_FloatRegister($dst$$reg),
14082 as_FloatRegister($src$$reg));
14083 break;
14084 case RoundDoubleModeNode::rmode_floor:
14085 __ frintmd(as_FloatRegister($dst$$reg),
14086 as_FloatRegister($src$$reg));
14087 break;
14088 case RoundDoubleModeNode::rmode_ceil:
14089 __ frintpd(as_FloatRegister($dst$$reg),
14090 as_FloatRegister($src$$reg));
14091 break;
14092 }
14093 %}
14094 ins_pipe(fp_uop_d);
14095 %}
14096
14097 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14098 match(Set dst (CopySignD src1 (Binary src2 zero)));
14099 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14100 format %{ "CopySignD $dst $src1 $src2" %}
14101 ins_encode %{
14102 FloatRegister dst = as_FloatRegister($dst$$reg),
14103 src1 = as_FloatRegister($src1$$reg),
14104 src2 = as_FloatRegister($src2$$reg),
14105 zero = as_FloatRegister($zero$$reg);
14106 __ fnegd(dst, zero);
14107 __ bsl(dst, __ T8B, src2, src1);
14108 %}
14109 ins_pipe(fp_uop_d);
14110 %}
14111
14112 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14113 match(Set dst (CopySignF src1 src2));
14114 effect(TEMP_DEF dst, USE src1, USE src2);
14115 format %{ "CopySignF $dst $src1 $src2" %}
14116 ins_encode %{
14117 FloatRegister dst = as_FloatRegister($dst$$reg),
14118 src1 = as_FloatRegister($src1$$reg),
14119 src2 = as_FloatRegister($src2$$reg);
14120 __ movi(dst, __ T2S, 0x80, 24);
14121 __ bsl(dst, __ T8B, src2, src1);
14122 %}
14123 ins_pipe(fp_uop_d);
14124 %}
14125
14126 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14127 match(Set dst (SignumD src (Binary zero one)));
14128 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14129 format %{ "signumD $dst, $src" %}
14130 ins_encode %{
14131 FloatRegister src = as_FloatRegister($src$$reg),
14132 dst = as_FloatRegister($dst$$reg),
14133 zero = as_FloatRegister($zero$$reg),
14134 one = as_FloatRegister($one$$reg);
14135 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14136 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14137 // Bit selection instruction gets bit from "one" for each enabled bit in
14138 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14139 // NaN the whole "src" will be copied because "dst" is zero. For all other
14140 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14141 // from "src", and all other bits are copied from 1.0.
14142 __ bsl(dst, __ T8B, one, src);
14143 %}
14144 ins_pipe(fp_uop_d);
14145 %}
14146
14147 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14148 match(Set dst (SignumF src (Binary zero one)));
14149 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14150 format %{ "signumF $dst, $src" %}
14151 ins_encode %{
14152 FloatRegister src = as_FloatRegister($src$$reg),
14153 dst = as_FloatRegister($dst$$reg),
14154 zero = as_FloatRegister($zero$$reg),
14155 one = as_FloatRegister($one$$reg);
14156 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14157 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14158 // Bit selection instruction gets bit from "one" for each enabled bit in
14159 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14160 // NaN the whole "src" will be copied because "dst" is zero. For all other
14161 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14162 // from "src", and all other bits are copied from 1.0.
14163 __ bsl(dst, __ T8B, one, src);
14164 %}
14165 ins_pipe(fp_uop_d);
14166 %}
14167
14168 instruct onspinwait() %{
14169 match(OnSpinWait);
14170 ins_cost(INSN_COST);
14171
14172 format %{ "onspinwait" %}
14173
14174 ins_encode %{
14175 __ spin_wait();
14176 %}
14177 ins_pipe(pipe_class_empty);
14178 %}
14179
14180 // ============================================================================
14181 // Logical Instructions
14182
14183 // Integer Logical Instructions
14184
14185 // And Instructions
14186
14187
14188 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14189 match(Set dst (AndI src1 src2));
14190
14191 format %{ "andw $dst, $src1, $src2\t# int" %}
14192
14193 ins_cost(INSN_COST);
14194 ins_encode %{
14195 __ andw(as_Register($dst$$reg),
14196 as_Register($src1$$reg),
14197 as_Register($src2$$reg));
14198 %}
14199
14200 ins_pipe(ialu_reg_reg);
14201 %}
14202
14203 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14204 match(Set dst (AndI src1 src2));
14205
14206 format %{ "andsw $dst, $src1, $src2\t# int" %}
14207
14208 ins_cost(INSN_COST);
14209 ins_encode %{
14210 __ andw(as_Register($dst$$reg),
14211 as_Register($src1$$reg),
14212 (uint64_t)($src2$$constant));
14213 %}
14214
14215 ins_pipe(ialu_reg_imm);
14216 %}
14217
14218 // Or Instructions
14219
14220 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14221 match(Set dst (OrI src1 src2));
14222
14223 format %{ "orrw $dst, $src1, $src2\t# int" %}
14224
14225 ins_cost(INSN_COST);
14226 ins_encode %{
14227 __ orrw(as_Register($dst$$reg),
14228 as_Register($src1$$reg),
14229 as_Register($src2$$reg));
14230 %}
14231
14232 ins_pipe(ialu_reg_reg);
14233 %}
14234
14235 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14236 match(Set dst (OrI src1 src2));
14237
14238 format %{ "orrw $dst, $src1, $src2\t# int" %}
14239
14240 ins_cost(INSN_COST);
14241 ins_encode %{
14242 __ orrw(as_Register($dst$$reg),
14243 as_Register($src1$$reg),
14244 (uint64_t)($src2$$constant));
14245 %}
14246
14247 ins_pipe(ialu_reg_imm);
14248 %}
14249
14250 // Xor Instructions
14251
14252 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14253 match(Set dst (XorI src1 src2));
14254
14255 format %{ "eorw $dst, $src1, $src2\t# int" %}
14256
14257 ins_cost(INSN_COST);
14258 ins_encode %{
14259 __ eorw(as_Register($dst$$reg),
14260 as_Register($src1$$reg),
14261 as_Register($src2$$reg));
14262 %}
14263
14264 ins_pipe(ialu_reg_reg);
14265 %}
14266
14267 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14268 match(Set dst (XorI src1 src2));
14269
14270 format %{ "eorw $dst, $src1, $src2\t# int" %}
14271
14272 ins_cost(INSN_COST);
14273 ins_encode %{
14274 __ eorw(as_Register($dst$$reg),
14275 as_Register($src1$$reg),
14276 (uint64_t)($src2$$constant));
14277 %}
14278
14279 ins_pipe(ialu_reg_imm);
14280 %}
14281
14282 // Long Logical Instructions
14283 // TODO
14284
14285 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14286 match(Set dst (AndL src1 src2));
14287
14288 format %{ "and $dst, $src1, $src2\t# int" %}
14289
14290 ins_cost(INSN_COST);
14291 ins_encode %{
14292 __ andr(as_Register($dst$$reg),
14293 as_Register($src1$$reg),
14294 as_Register($src2$$reg));
14295 %}
14296
14297 ins_pipe(ialu_reg_reg);
14298 %}
14299
14300 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14301 match(Set dst (AndL src1 src2));
14302
14303 format %{ "and $dst, $src1, $src2\t# int" %}
14304
14305 ins_cost(INSN_COST);
14306 ins_encode %{
14307 __ andr(as_Register($dst$$reg),
14308 as_Register($src1$$reg),
14309 (uint64_t)($src2$$constant));
14310 %}
14311
14312 ins_pipe(ialu_reg_imm);
14313 %}
14314
14315 // Or Instructions
14316
14317 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14318 match(Set dst (OrL src1 src2));
14319
14320 format %{ "orr $dst, $src1, $src2\t# int" %}
14321
14322 ins_cost(INSN_COST);
14323 ins_encode %{
14324 __ orr(as_Register($dst$$reg),
14325 as_Register($src1$$reg),
14326 as_Register($src2$$reg));
14327 %}
14328
14329 ins_pipe(ialu_reg_reg);
14330 %}
14331
14332 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14333 match(Set dst (OrL src1 src2));
14334
14335 format %{ "orr $dst, $src1, $src2\t# int" %}
14336
14337 ins_cost(INSN_COST);
14338 ins_encode %{
14339 __ orr(as_Register($dst$$reg),
14340 as_Register($src1$$reg),
14341 (uint64_t)($src2$$constant));
14342 %}
14343
14344 ins_pipe(ialu_reg_imm);
14345 %}
14346
14347 // Xor Instructions
14348
14349 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14350 match(Set dst (XorL src1 src2));
14351
14352 format %{ "eor $dst, $src1, $src2\t# int" %}
14353
14354 ins_cost(INSN_COST);
14355 ins_encode %{
14356 __ eor(as_Register($dst$$reg),
14357 as_Register($src1$$reg),
14358 as_Register($src2$$reg));
14359 %}
14360
14361 ins_pipe(ialu_reg_reg);
14362 %}
14363
14364 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14365 match(Set dst (XorL src1 src2));
14366
14367 ins_cost(INSN_COST);
14368 format %{ "eor $dst, $src1, $src2\t# int" %}
14369
14370 ins_encode %{
14371 __ eor(as_Register($dst$$reg),
14372 as_Register($src1$$reg),
14373 (uint64_t)($src2$$constant));
14374 %}
14375
14376 ins_pipe(ialu_reg_imm);
14377 %}
14378
14379 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14380 %{
14381 match(Set dst (ConvI2L src));
14382
14383 ins_cost(INSN_COST);
14384 format %{ "sxtw $dst, $src\t# i2l" %}
14385 ins_encode %{
14386 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14387 %}
14388 ins_pipe(ialu_reg_shift);
14389 %}
14390
14391 // this pattern occurs in bigmath arithmetic
14392 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14393 %{
14394 match(Set dst (AndL (ConvI2L src) mask));
14395
14396 ins_cost(INSN_COST);
14397 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14398 ins_encode %{
14399 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14400 %}
14401
14402 ins_pipe(ialu_reg_shift);
14403 %}
14404
14405 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14406 match(Set dst (ConvL2I src));
14407
14408 ins_cost(INSN_COST);
14409 format %{ "movw $dst, $src \t// l2i" %}
14410
14411 ins_encode %{
14412 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14413 %}
14414
14415 ins_pipe(ialu_reg);
14416 %}
14417
14418 instruct convD2F_reg(vRegF dst, vRegD src) %{
14419 match(Set dst (ConvD2F src));
14420
14421 ins_cost(INSN_COST * 5);
14422 format %{ "fcvtd $dst, $src \t// d2f" %}
14423
14424 ins_encode %{
14425 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14426 %}
14427
14428 ins_pipe(fp_d2f);
14429 %}
14430
14431 instruct convF2D_reg(vRegD dst, vRegF src) %{
14432 match(Set dst (ConvF2D src));
14433
14434 ins_cost(INSN_COST * 5);
14435 format %{ "fcvts $dst, $src \t// f2d" %}
14436
14437 ins_encode %{
14438 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14439 %}
14440
14441 ins_pipe(fp_f2d);
14442 %}
14443
14444 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14445 match(Set dst (ConvF2I src));
14446
14447 ins_cost(INSN_COST * 5);
14448 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14449
14450 ins_encode %{
14451 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14452 %}
14453
14454 ins_pipe(fp_f2i);
14455 %}
14456
14457 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14458 match(Set dst (ConvF2L src));
14459
14460 ins_cost(INSN_COST * 5);
14461 format %{ "fcvtzs $dst, $src \t// f2l" %}
14462
14463 ins_encode %{
14464 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14465 %}
14466
14467 ins_pipe(fp_f2l);
14468 %}
14469
14470 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14471 match(Set dst (ConvF2HF src));
14472 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14473 "smov $dst, $tmp\t# move result from $tmp to $dst"
14474 %}
14475 effect(TEMP tmp);
14476 ins_encode %{
14477 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14478 %}
14479 ins_pipe(pipe_slow);
14480 %}
14481
14482 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14483 match(Set dst (ConvHF2F src));
14484 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14485 "fcvt $dst, $tmp\t# convert half to single precision"
14486 %}
14487 effect(TEMP tmp);
14488 ins_encode %{
14489 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14490 %}
14491 ins_pipe(pipe_slow);
14492 %}
14493
14494 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14495 match(Set dst (ConvI2F src));
14496
14497 ins_cost(INSN_COST * 5);
14498 format %{ "scvtfws $dst, $src \t// i2f" %}
14499
14500 ins_encode %{
14501 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14502 %}
14503
14504 ins_pipe(fp_i2f);
14505 %}
14506
14507 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14508 match(Set dst (ConvL2F src));
14509
14510 ins_cost(INSN_COST * 5);
14511 format %{ "scvtfs $dst, $src \t// l2f" %}
14512
14513 ins_encode %{
14514 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14515 %}
14516
14517 ins_pipe(fp_l2f);
14518 %}
14519
14520 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14521 match(Set dst (ConvD2I src));
14522
14523 ins_cost(INSN_COST * 5);
14524 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14525
14526 ins_encode %{
14527 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14528 %}
14529
14530 ins_pipe(fp_d2i);
14531 %}
14532
14533 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14534 match(Set dst (ConvD2L src));
14535
14536 ins_cost(INSN_COST * 5);
14537 format %{ "fcvtzd $dst, $src \t// d2l" %}
14538
14539 ins_encode %{
14540 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14541 %}
14542
14543 ins_pipe(fp_d2l);
14544 %}
14545
14546 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14547 match(Set dst (ConvI2D src));
14548
14549 ins_cost(INSN_COST * 5);
14550 format %{ "scvtfwd $dst, $src \t// i2d" %}
14551
14552 ins_encode %{
14553 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14554 %}
14555
14556 ins_pipe(fp_i2d);
14557 %}
14558
14559 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14560 match(Set dst (ConvL2D src));
14561
14562 ins_cost(INSN_COST * 5);
14563 format %{ "scvtfd $dst, $src \t// l2d" %}
14564
14565 ins_encode %{
14566 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14567 %}
14568
14569 ins_pipe(fp_l2d);
14570 %}
14571
14572 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14573 %{
14574 match(Set dst (RoundD src));
14575 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14576 format %{ "java_round_double $dst,$src"%}
14577 ins_encode %{
14578 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14579 as_FloatRegister($ftmp$$reg));
14580 %}
14581 ins_pipe(pipe_slow);
14582 %}
14583
14584 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14585 %{
14586 match(Set dst (RoundF src));
14587 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14588 format %{ "java_round_float $dst,$src"%}
14589 ins_encode %{
14590 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14591 as_FloatRegister($ftmp$$reg));
14592 %}
14593 ins_pipe(pipe_slow);
14594 %}
14595
14596 // stack <-> reg and reg <-> reg shuffles with no conversion
14597
14598 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14599
14600 match(Set dst (MoveF2I src));
14601
14602 effect(DEF dst, USE src);
14603
14604 ins_cost(4 * INSN_COST);
14605
14606 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14607
14608 ins_encode %{
14609 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14610 %}
14611
14612 ins_pipe(iload_reg_reg);
14613
14614 %}
14615
14616 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14617
14618 match(Set dst (MoveI2F src));
14619
14620 effect(DEF dst, USE src);
14621
14622 ins_cost(4 * INSN_COST);
14623
14624 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14625
14626 ins_encode %{
14627 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14628 %}
14629
14630 ins_pipe(pipe_class_memory);
14631
14632 %}
14633
14634 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14635
14636 match(Set dst (MoveD2L src));
14637
14638 effect(DEF dst, USE src);
14639
14640 ins_cost(4 * INSN_COST);
14641
14642 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14643
14644 ins_encode %{
14645 __ ldr($dst$$Register, Address(sp, $src$$disp));
14646 %}
14647
14648 ins_pipe(iload_reg_reg);
14649
14650 %}
14651
14652 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14653
14654 match(Set dst (MoveL2D src));
14655
14656 effect(DEF dst, USE src);
14657
14658 ins_cost(4 * INSN_COST);
14659
14660 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14661
14662 ins_encode %{
14663 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14664 %}
14665
14666 ins_pipe(pipe_class_memory);
14667
14668 %}
14669
14670 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14671
14672 match(Set dst (MoveF2I src));
14673
14674 effect(DEF dst, USE src);
14675
14676 ins_cost(INSN_COST);
14677
14678 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14679
14680 ins_encode %{
14681 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14682 %}
14683
14684 ins_pipe(pipe_class_memory);
14685
14686 %}
14687
14688 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14689
14690 match(Set dst (MoveI2F src));
14691
14692 effect(DEF dst, USE src);
14693
14694 ins_cost(INSN_COST);
14695
14696 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14697
14698 ins_encode %{
14699 __ strw($src$$Register, Address(sp, $dst$$disp));
14700 %}
14701
14702 ins_pipe(istore_reg_reg);
14703
14704 %}
14705
14706 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14707
14708 match(Set dst (MoveD2L src));
14709
14710 effect(DEF dst, USE src);
14711
14712 ins_cost(INSN_COST);
14713
14714 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14715
14716 ins_encode %{
14717 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14718 %}
14719
14720 ins_pipe(pipe_class_memory);
14721
14722 %}
14723
14724 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14725
14726 match(Set dst (MoveL2D src));
14727
14728 effect(DEF dst, USE src);
14729
14730 ins_cost(INSN_COST);
14731
14732 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14733
14734 ins_encode %{
14735 __ str($src$$Register, Address(sp, $dst$$disp));
14736 %}
14737
14738 ins_pipe(istore_reg_reg);
14739
14740 %}
14741
14742 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14743
14744 match(Set dst (MoveF2I src));
14745
14746 effect(DEF dst, USE src);
14747
14748 ins_cost(INSN_COST);
14749
14750 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14751
14752 ins_encode %{
14753 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14754 %}
14755
14756 ins_pipe(fp_f2i);
14757
14758 %}
14759
14760 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14761
14762 match(Set dst (MoveI2F src));
14763
14764 effect(DEF dst, USE src);
14765
14766 ins_cost(INSN_COST);
14767
14768 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14769
14770 ins_encode %{
14771 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14772 %}
14773
14774 ins_pipe(fp_i2f);
14775
14776 %}
14777
14778 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14779
14780 match(Set dst (MoveD2L src));
14781
14782 effect(DEF dst, USE src);
14783
14784 ins_cost(INSN_COST);
14785
14786 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14787
14788 ins_encode %{
14789 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14790 %}
14791
14792 ins_pipe(fp_d2l);
14793
14794 %}
14795
14796 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14797
14798 match(Set dst (MoveL2D src));
14799
14800 effect(DEF dst, USE src);
14801
14802 ins_cost(INSN_COST);
14803
14804 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14805
14806 ins_encode %{
14807 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14808 %}
14809
14810 ins_pipe(fp_l2d);
14811
14812 %}
14813
14814 // ============================================================================
14815 // clearing of an array
14816
14817 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14818 %{
14819 match(Set dummy (ClearArray (Binary cnt base) zero));
14820 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14821
14822 ins_cost(4 * INSN_COST);
14823 format %{ "ClearArray $cnt, $base" %}
14824
14825 ins_encode %{
14826 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14827 if (tpc == nullptr) {
14828 ciEnv::current()->record_failure("CodeCache is full");
14829 return;
14830 }
14831 %}
14832
14833 ins_pipe(pipe_class_memory);
14834 %}
14835
14836 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14837 %{
14838 predicate(((ClearArrayNode*)n)->word_copy_only());
14839 match(Set dummy (ClearArray (Binary cnt base) val));
14840 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14841
14842 ins_cost(4 * INSN_COST);
14843 format %{ "ClearArray $cnt, $base, $val" %}
14844
14845 ins_encode %{
14846 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14847 %}
14848
14849 ins_pipe(pipe_class_memory);
14850 %}
14851
14852 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14853 %{
14854 predicate((uint64_t)n->in(2)->get_long()
14855 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14856 && !((ClearArrayNode*)n)->word_copy_only());
14857 match(Set dummy (ClearArray cnt base));
14858 effect(TEMP temp, USE_KILL base, KILL cr);
14859
14860 ins_cost(4 * INSN_COST);
14861 format %{ "ClearArray $cnt, $base" %}
14862
14863 ins_encode %{
14864 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14865 if (tpc == nullptr) {
14866 ciEnv::current()->record_failure("CodeCache is full");
14867 return;
14868 }
14869 %}
14870
14871 ins_pipe(pipe_class_memory);
14872 %}
14873
14874 // ============================================================================
14875 // Overflow Math Instructions
14876
14877 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14878 %{
14879 match(Set cr (OverflowAddI op1 op2));
14880
14881 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14882 ins_cost(INSN_COST);
14883 ins_encode %{
14884 __ cmnw($op1$$Register, $op2$$Register);
14885 %}
14886
14887 ins_pipe(icmp_reg_reg);
14888 %}
14889
14890 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14891 %{
14892 match(Set cr (OverflowAddI op1 op2));
14893
14894 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14895 ins_cost(INSN_COST);
14896 ins_encode %{
14897 __ cmnw($op1$$Register, $op2$$constant);
14898 %}
14899
14900 ins_pipe(icmp_reg_imm);
14901 %}
14902
14903 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14904 %{
14905 match(Set cr (OverflowAddL op1 op2));
14906
14907 format %{ "cmn $op1, $op2\t# overflow check long" %}
14908 ins_cost(INSN_COST);
14909 ins_encode %{
14910 __ cmn($op1$$Register, $op2$$Register);
14911 %}
14912
14913 ins_pipe(icmp_reg_reg);
14914 %}
14915
14916 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14917 %{
14918 match(Set cr (OverflowAddL op1 op2));
14919
14920 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14921 ins_cost(INSN_COST);
14922 ins_encode %{
14923 __ adds(zr, $op1$$Register, $op2$$constant);
14924 %}
14925
14926 ins_pipe(icmp_reg_imm);
14927 %}
14928
14929 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14930 %{
14931 match(Set cr (OverflowSubI op1 op2));
14932
14933 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14934 ins_cost(INSN_COST);
14935 ins_encode %{
14936 __ cmpw($op1$$Register, $op2$$Register);
14937 %}
14938
14939 ins_pipe(icmp_reg_reg);
14940 %}
14941
14942 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14943 %{
14944 match(Set cr (OverflowSubI op1 op2));
14945
14946 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14947 ins_cost(INSN_COST);
14948 ins_encode %{
14949 __ cmpw($op1$$Register, $op2$$constant);
14950 %}
14951
14952 ins_pipe(icmp_reg_imm);
14953 %}
14954
14955 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14956 %{
14957 match(Set cr (OverflowSubL op1 op2));
14958
14959 format %{ "cmp $op1, $op2\t# overflow check long" %}
14960 ins_cost(INSN_COST);
14961 ins_encode %{
14962 __ cmp($op1$$Register, $op2$$Register);
14963 %}
14964
14965 ins_pipe(icmp_reg_reg);
14966 %}
14967
14968 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14969 %{
14970 match(Set cr (OverflowSubL op1 op2));
14971
14972 format %{ "cmp $op1, $op2\t# overflow check long" %}
14973 ins_cost(INSN_COST);
14974 ins_encode %{
14975 __ subs(zr, $op1$$Register, $op2$$constant);
14976 %}
14977
14978 ins_pipe(icmp_reg_imm);
14979 %}
14980
14981 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14982 %{
14983 match(Set cr (OverflowSubI zero op1));
14984
14985 format %{ "cmpw zr, $op1\t# overflow check int" %}
14986 ins_cost(INSN_COST);
14987 ins_encode %{
14988 __ cmpw(zr, $op1$$Register);
14989 %}
14990
14991 ins_pipe(icmp_reg_imm);
14992 %}
14993
14994 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14995 %{
14996 match(Set cr (OverflowSubL zero op1));
14997
14998 format %{ "cmp zr, $op1\t# overflow check long" %}
14999 ins_cost(INSN_COST);
15000 ins_encode %{
15001 __ cmp(zr, $op1$$Register);
15002 %}
15003
15004 ins_pipe(icmp_reg_imm);
15005 %}
15006
15007 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15008 %{
15009 match(Set cr (OverflowMulI op1 op2));
15010
15011 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15012 "cmp rscratch1, rscratch1, sxtw\n\t"
15013 "movw rscratch1, #0x80000000\n\t"
15014 "cselw rscratch1, rscratch1, zr, NE\n\t"
15015 "cmpw rscratch1, #1" %}
15016 ins_cost(5 * INSN_COST);
15017 ins_encode %{
15018 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15019 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15020 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15021 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15022 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15023 %}
15024
15025 ins_pipe(pipe_slow);
15026 %}
15027
15028 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15029 %{
15030 match(If cmp (OverflowMulI op1 op2));
15031 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15032 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15033 effect(USE labl, KILL cr);
15034
15035 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15036 "cmp rscratch1, rscratch1, sxtw\n\t"
15037 "b$cmp $labl" %}
15038 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15039 ins_encode %{
15040 Label* L = $labl$$label;
15041 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15042 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15043 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15044 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15045 %}
15046
15047 ins_pipe(pipe_serial);
15048 %}
15049
15050 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15051 %{
15052 match(Set cr (OverflowMulL op1 op2));
15053
15054 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15055 "smulh rscratch2, $op1, $op2\n\t"
15056 "cmp rscratch2, rscratch1, ASR #63\n\t"
15057 "movw rscratch1, #0x80000000\n\t"
15058 "cselw rscratch1, rscratch1, zr, NE\n\t"
15059 "cmpw rscratch1, #1" %}
15060 ins_cost(6 * INSN_COST);
15061 ins_encode %{
15062 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15063 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15064 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15065 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15066 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15067 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15068 %}
15069
15070 ins_pipe(pipe_slow);
15071 %}
15072
15073 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15074 %{
15075 match(If cmp (OverflowMulL op1 op2));
15076 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15077 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15078 effect(USE labl, KILL cr);
15079
15080 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15081 "smulh rscratch2, $op1, $op2\n\t"
15082 "cmp rscratch2, rscratch1, ASR #63\n\t"
15083 "b$cmp $labl" %}
15084 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15085 ins_encode %{
15086 Label* L = $labl$$label;
15087 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15088 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15089 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15090 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15091 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15092 %}
15093
15094 ins_pipe(pipe_serial);
15095 %}
15096
15097 // ============================================================================
15098 // Compare Instructions
15099
15100 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15101 %{
15102 match(Set cr (CmpI op1 op2));
15103
15104 effect(DEF cr, USE op1, USE op2);
15105
15106 ins_cost(INSN_COST);
15107 format %{ "cmpw $op1, $op2" %}
15108
15109 ins_encode(aarch64_enc_cmpw(op1, op2));
15110
15111 ins_pipe(icmp_reg_reg);
15112 %}
15113
15114 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15115 %{
15116 match(Set cr (CmpI op1 zero));
15117
15118 effect(DEF cr, USE op1);
15119
15120 ins_cost(INSN_COST);
15121 format %{ "cmpw $op1, 0" %}
15122
15123 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15124
15125 ins_pipe(icmp_reg_imm);
15126 %}
15127
15128 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15129 %{
15130 match(Set cr (CmpI op1 op2));
15131
15132 effect(DEF cr, USE op1);
15133
15134 ins_cost(INSN_COST);
15135 format %{ "cmpw $op1, $op2" %}
15136
15137 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15138
15139 ins_pipe(icmp_reg_imm);
15140 %}
15141
15142 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15143 %{
15144 match(Set cr (CmpI op1 op2));
15145
15146 effect(DEF cr, USE op1);
15147
15148 ins_cost(INSN_COST * 2);
15149 format %{ "cmpw $op1, $op2" %}
15150
15151 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15152
15153 ins_pipe(icmp_reg_imm);
15154 %}
15155
15156 // Unsigned compare Instructions; really, same as signed compare
15157 // except it should only be used to feed an If or a CMovI which takes a
15158 // cmpOpU.
15159
15160 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15161 %{
15162 match(Set cr (CmpU op1 op2));
15163
15164 effect(DEF cr, USE op1, USE op2);
15165
15166 ins_cost(INSN_COST);
15167 format %{ "cmpw $op1, $op2\t# unsigned" %}
15168
15169 ins_encode(aarch64_enc_cmpw(op1, op2));
15170
15171 ins_pipe(icmp_reg_reg);
15172 %}
15173
15174 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15175 %{
15176 match(Set cr (CmpU op1 zero));
15177
15178 effect(DEF cr, USE op1);
15179
15180 ins_cost(INSN_COST);
15181 format %{ "cmpw $op1, #0\t# unsigned" %}
15182
15183 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15184
15185 ins_pipe(icmp_reg_imm);
15186 %}
15187
15188 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15189 %{
15190 match(Set cr (CmpU op1 op2));
15191
15192 effect(DEF cr, USE op1);
15193
15194 ins_cost(INSN_COST);
15195 format %{ "cmpw $op1, $op2\t# unsigned" %}
15196
15197 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15198
15199 ins_pipe(icmp_reg_imm);
15200 %}
15201
15202 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15203 %{
15204 match(Set cr (CmpU op1 op2));
15205
15206 effect(DEF cr, USE op1);
15207
15208 ins_cost(INSN_COST * 2);
15209 format %{ "cmpw $op1, $op2\t# unsigned" %}
15210
15211 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15212
15213 ins_pipe(icmp_reg_imm);
15214 %}
15215
15216 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15217 %{
15218 match(Set cr (CmpL op1 op2));
15219
15220 effect(DEF cr, USE op1, USE op2);
15221
15222 ins_cost(INSN_COST);
15223 format %{ "cmp $op1, $op2" %}
15224
15225 ins_encode(aarch64_enc_cmp(op1, op2));
15226
15227 ins_pipe(icmp_reg_reg);
15228 %}
15229
15230 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15231 %{
15232 match(Set cr (CmpL op1 zero));
15233
15234 effect(DEF cr, USE op1);
15235
15236 ins_cost(INSN_COST);
15237 format %{ "tst $op1" %}
15238
15239 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15240
15241 ins_pipe(icmp_reg_imm);
15242 %}
15243
15244 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15245 %{
15246 match(Set cr (CmpL op1 op2));
15247
15248 effect(DEF cr, USE op1);
15249
15250 ins_cost(INSN_COST);
15251 format %{ "cmp $op1, $op2" %}
15252
15253 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15254
15255 ins_pipe(icmp_reg_imm);
15256 %}
15257
15258 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15259 %{
15260 match(Set cr (CmpL op1 op2));
15261
15262 effect(DEF cr, USE op1);
15263
15264 ins_cost(INSN_COST * 2);
15265 format %{ "cmp $op1, $op2" %}
15266
15267 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15268
15269 ins_pipe(icmp_reg_imm);
15270 %}
15271
15272 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15273 %{
15274 match(Set cr (CmpUL op1 op2));
15275
15276 effect(DEF cr, USE op1, USE op2);
15277
15278 ins_cost(INSN_COST);
15279 format %{ "cmp $op1, $op2" %}
15280
15281 ins_encode(aarch64_enc_cmp(op1, op2));
15282
15283 ins_pipe(icmp_reg_reg);
15284 %}
15285
15286 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15287 %{
15288 match(Set cr (CmpUL op1 zero));
15289
15290 effect(DEF cr, USE op1);
15291
15292 ins_cost(INSN_COST);
15293 format %{ "tst $op1" %}
15294
15295 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15296
15297 ins_pipe(icmp_reg_imm);
15298 %}
15299
15300 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15301 %{
15302 match(Set cr (CmpUL op1 op2));
15303
15304 effect(DEF cr, USE op1);
15305
15306 ins_cost(INSN_COST);
15307 format %{ "cmp $op1, $op2" %}
15308
15309 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15310
15311 ins_pipe(icmp_reg_imm);
15312 %}
15313
15314 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15315 %{
15316 match(Set cr (CmpUL op1 op2));
15317
15318 effect(DEF cr, USE op1);
15319
15320 ins_cost(INSN_COST * 2);
15321 format %{ "cmp $op1, $op2" %}
15322
15323 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15324
15325 ins_pipe(icmp_reg_imm);
15326 %}
15327
15328 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15329 %{
15330 match(Set cr (CmpP op1 op2));
15331
15332 effect(DEF cr, USE op1, USE op2);
15333
15334 ins_cost(INSN_COST);
15335 format %{ "cmp $op1, $op2\t // ptr" %}
15336
15337 ins_encode(aarch64_enc_cmpp(op1, op2));
15338
15339 ins_pipe(icmp_reg_reg);
15340 %}
15341
15342 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15343 %{
15344 match(Set cr (CmpN op1 op2));
15345
15346 effect(DEF cr, USE op1, USE op2);
15347
15348 ins_cost(INSN_COST);
15349 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15350
15351 ins_encode(aarch64_enc_cmpn(op1, op2));
15352
15353 ins_pipe(icmp_reg_reg);
15354 %}
15355
15356 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15357 %{
15358 match(Set cr (CmpP op1 zero));
15359
15360 effect(DEF cr, USE op1, USE zero);
15361
15362 ins_cost(INSN_COST);
15363 format %{ "cmp $op1, 0\t // ptr" %}
15364
15365 ins_encode(aarch64_enc_testp(op1));
15366
15367 ins_pipe(icmp_reg_imm);
15368 %}
15369
15370 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15371 %{
15372 match(Set cr (CmpN op1 zero));
15373
15374 effect(DEF cr, USE op1, USE zero);
15375
15376 ins_cost(INSN_COST);
15377 format %{ "cmp $op1, 0\t // compressed ptr" %}
15378
15379 ins_encode(aarch64_enc_testn(op1));
15380
15381 ins_pipe(icmp_reg_imm);
15382 %}
15383
15384 // FP comparisons
15385 //
15386 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15387 // using normal cmpOp. See declaration of rFlagsReg for details.
15388
15389 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15390 %{
15391 match(Set cr (CmpF src1 src2));
15392
15393 ins_cost(3 * INSN_COST);
15394 format %{ "fcmps $src1, $src2" %}
15395
15396 ins_encode %{
15397 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15398 %}
15399
15400 ins_pipe(pipe_class_compare);
15401 %}
15402
15403 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15404 %{
15405 match(Set cr (CmpF src1 src2));
15406
15407 ins_cost(3 * INSN_COST);
15408 format %{ "fcmps $src1, 0.0" %}
15409
15410 ins_encode %{
15411 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15412 %}
15413
15414 ins_pipe(pipe_class_compare);
15415 %}
15416 // FROM HERE
15417
15418 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15419 %{
15420 match(Set cr (CmpD src1 src2));
15421
15422 ins_cost(3 * INSN_COST);
15423 format %{ "fcmpd $src1, $src2" %}
15424
15425 ins_encode %{
15426 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15427 %}
15428
15429 ins_pipe(pipe_class_compare);
15430 %}
15431
15432 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15433 %{
15434 match(Set cr (CmpD src1 src2));
15435
15436 ins_cost(3 * INSN_COST);
15437 format %{ "fcmpd $src1, 0.0" %}
15438
15439 ins_encode %{
15440 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15441 %}
15442
15443 ins_pipe(pipe_class_compare);
15444 %}
15445
15446 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15447 %{
15448 match(Set dst (CmpF3 src1 src2));
15449 effect(KILL cr);
15450
15451 ins_cost(5 * INSN_COST);
15452 format %{ "fcmps $src1, $src2\n\t"
15453 "csinvw($dst, zr, zr, eq\n\t"
15454 "csnegw($dst, $dst, $dst, lt)"
15455 %}
15456
15457 ins_encode %{
15458 Label done;
15459 FloatRegister s1 = as_FloatRegister($src1$$reg);
15460 FloatRegister s2 = as_FloatRegister($src2$$reg);
15461 Register d = as_Register($dst$$reg);
15462 __ fcmps(s1, s2);
15463 // installs 0 if EQ else -1
15464 __ csinvw(d, zr, zr, Assembler::EQ);
15465 // keeps -1 if less or unordered else installs 1
15466 __ csnegw(d, d, d, Assembler::LT);
15467 __ bind(done);
15468 %}
15469
15470 ins_pipe(pipe_class_default);
15471
15472 %}
15473
15474 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15475 %{
15476 match(Set dst (CmpD3 src1 src2));
15477 effect(KILL cr);
15478
15479 ins_cost(5 * INSN_COST);
15480 format %{ "fcmpd $src1, $src2\n\t"
15481 "csinvw($dst, zr, zr, eq\n\t"
15482 "csnegw($dst, $dst, $dst, lt)"
15483 %}
15484
15485 ins_encode %{
15486 Label done;
15487 FloatRegister s1 = as_FloatRegister($src1$$reg);
15488 FloatRegister s2 = as_FloatRegister($src2$$reg);
15489 Register d = as_Register($dst$$reg);
15490 __ fcmpd(s1, s2);
15491 // installs 0 if EQ else -1
15492 __ csinvw(d, zr, zr, Assembler::EQ);
15493 // keeps -1 if less or unordered else installs 1
15494 __ csnegw(d, d, d, Assembler::LT);
15495 __ bind(done);
15496 %}
15497 ins_pipe(pipe_class_default);
15498
15499 %}
15500
15501 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15502 %{
15503 match(Set dst (CmpF3 src1 zero));
15504 effect(KILL cr);
15505
15506 ins_cost(5 * INSN_COST);
15507 format %{ "fcmps $src1, 0.0\n\t"
15508 "csinvw($dst, zr, zr, eq\n\t"
15509 "csnegw($dst, $dst, $dst, lt)"
15510 %}
15511
15512 ins_encode %{
15513 Label done;
15514 FloatRegister s1 = as_FloatRegister($src1$$reg);
15515 Register d = as_Register($dst$$reg);
15516 __ fcmps(s1, 0.0);
15517 // installs 0 if EQ else -1
15518 __ csinvw(d, zr, zr, Assembler::EQ);
15519 // keeps -1 if less or unordered else installs 1
15520 __ csnegw(d, d, d, Assembler::LT);
15521 __ bind(done);
15522 %}
15523
15524 ins_pipe(pipe_class_default);
15525
15526 %}
15527
15528 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15529 %{
15530 match(Set dst (CmpD3 src1 zero));
15531 effect(KILL cr);
15532
15533 ins_cost(5 * INSN_COST);
15534 format %{ "fcmpd $src1, 0.0\n\t"
15535 "csinvw($dst, zr, zr, eq\n\t"
15536 "csnegw($dst, $dst, $dst, lt)"
15537 %}
15538
15539 ins_encode %{
15540 Label done;
15541 FloatRegister s1 = as_FloatRegister($src1$$reg);
15542 Register d = as_Register($dst$$reg);
15543 __ fcmpd(s1, 0.0);
15544 // installs 0 if EQ else -1
15545 __ csinvw(d, zr, zr, Assembler::EQ);
15546 // keeps -1 if less or unordered else installs 1
15547 __ csnegw(d, d, d, Assembler::LT);
15548 __ bind(done);
15549 %}
15550 ins_pipe(pipe_class_default);
15551
15552 %}
15553
15554 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15555 %{
15556 match(Set dst (CmpLTMask p q));
15557 effect(KILL cr);
15558
15559 ins_cost(3 * INSN_COST);
15560
15561 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15562 "csetw $dst, lt\n\t"
15563 "subw $dst, zr, $dst"
15564 %}
15565
15566 ins_encode %{
15567 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15568 __ csetw(as_Register($dst$$reg), Assembler::LT);
15569 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15570 %}
15571
15572 ins_pipe(ialu_reg_reg);
15573 %}
15574
15575 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15576 %{
15577 match(Set dst (CmpLTMask src zero));
15578 effect(KILL cr);
15579
15580 ins_cost(INSN_COST);
15581
15582 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15583
15584 ins_encode %{
15585 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15586 %}
15587
15588 ins_pipe(ialu_reg_shift);
15589 %}
15590
15591 // ============================================================================
15592 // Max and Min
15593
15594 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15595
15596 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15597 %{
15598 effect(DEF cr, USE src);
15599 ins_cost(INSN_COST);
15600 format %{ "cmpw $src, 0" %}
15601
15602 ins_encode %{
15603 __ cmpw($src$$Register, 0);
15604 %}
15605 ins_pipe(icmp_reg_imm);
15606 %}
15607
15608 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15609 %{
15610 match(Set dst (MinI src1 src2));
15611 ins_cost(INSN_COST * 3);
15612
15613 expand %{
15614 rFlagsReg cr;
15615 compI_reg_reg(cr, src1, src2);
15616 cmovI_reg_reg_lt(dst, src1, src2, cr);
15617 %}
15618 %}
15619
15620 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15621 %{
15622 match(Set dst (MaxI src1 src2));
15623 ins_cost(INSN_COST * 3);
15624
15625 expand %{
15626 rFlagsReg cr;
15627 compI_reg_reg(cr, src1, src2);
15628 cmovI_reg_reg_gt(dst, src1, src2, cr);
15629 %}
15630 %}
15631
15632
15633 // ============================================================================
15634 // Branch Instructions
15635
15636 // Direct Branch.
15637 instruct branch(label lbl)
15638 %{
15639 match(Goto);
15640
15641 effect(USE lbl);
15642
15643 ins_cost(BRANCH_COST);
15644 format %{ "b $lbl" %}
15645
15646 ins_encode(aarch64_enc_b(lbl));
15647
15648 ins_pipe(pipe_branch);
15649 %}
15650
15651 // Conditional Near Branch
15652 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15653 %{
15654 // Same match rule as `branchConFar'.
15655 match(If cmp cr);
15656
15657 effect(USE lbl);
15658
15659 ins_cost(BRANCH_COST);
15660 // If set to 1 this indicates that the current instruction is a
15661 // short variant of a long branch. This avoids using this
15662 // instruction in first-pass matching. It will then only be used in
15663 // the `Shorten_branches' pass.
15664 // ins_short_branch(1);
15665 format %{ "b$cmp $lbl" %}
15666
15667 ins_encode(aarch64_enc_br_con(cmp, lbl));
15668
15669 ins_pipe(pipe_branch_cond);
15670 %}
15671
15672 // Conditional Near Branch Unsigned
15673 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15674 %{
15675 // Same match rule as `branchConFar'.
15676 match(If cmp cr);
15677
15678 effect(USE lbl);
15679
15680 ins_cost(BRANCH_COST);
15681 // If set to 1 this indicates that the current instruction is a
15682 // short variant of a long branch. This avoids using this
15683 // instruction in first-pass matching. It will then only be used in
15684 // the `Shorten_branches' pass.
15685 // ins_short_branch(1);
15686 format %{ "b$cmp $lbl\t# unsigned" %}
15687
15688 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15689
15690 ins_pipe(pipe_branch_cond);
15691 %}
15692
15693 // Make use of CBZ and CBNZ. These instructions, as well as being
15694 // shorter than (cmp; branch), have the additional benefit of not
15695 // killing the flags.
15696
15697 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15698 match(If cmp (CmpI op1 op2));
15699 effect(USE labl);
15700
15701 ins_cost(BRANCH_COST);
15702 format %{ "cbw$cmp $op1, $labl" %}
15703 ins_encode %{
15704 Label* L = $labl$$label;
15705 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15706 if (cond == Assembler::EQ)
15707 __ cbzw($op1$$Register, *L);
15708 else
15709 __ cbnzw($op1$$Register, *L);
15710 %}
15711 ins_pipe(pipe_cmp_branch);
15712 %}
15713
15714 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15715 match(If cmp (CmpL op1 op2));
15716 effect(USE labl);
15717
15718 ins_cost(BRANCH_COST);
15719 format %{ "cb$cmp $op1, $labl" %}
15720 ins_encode %{
15721 Label* L = $labl$$label;
15722 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15723 if (cond == Assembler::EQ)
15724 __ cbz($op1$$Register, *L);
15725 else
15726 __ cbnz($op1$$Register, *L);
15727 %}
15728 ins_pipe(pipe_cmp_branch);
15729 %}
15730
15731 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15732 match(If cmp (CmpP op1 op2));
15733 effect(USE labl);
15734
15735 ins_cost(BRANCH_COST);
15736 format %{ "cb$cmp $op1, $labl" %}
15737 ins_encode %{
15738 Label* L = $labl$$label;
15739 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15740 if (cond == Assembler::EQ)
15741 __ cbz($op1$$Register, *L);
15742 else
15743 __ cbnz($op1$$Register, *L);
15744 %}
15745 ins_pipe(pipe_cmp_branch);
15746 %}
15747
15748 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15749 match(If cmp (CmpN op1 op2));
15750 effect(USE labl);
15751
15752 ins_cost(BRANCH_COST);
15753 format %{ "cbw$cmp $op1, $labl" %}
15754 ins_encode %{
15755 Label* L = $labl$$label;
15756 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15757 if (cond == Assembler::EQ)
15758 __ cbzw($op1$$Register, *L);
15759 else
15760 __ cbnzw($op1$$Register, *L);
15761 %}
15762 ins_pipe(pipe_cmp_branch);
15763 %}
15764
15765 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15766 match(If cmp (CmpP (DecodeN oop) zero));
15767 effect(USE labl);
15768
15769 ins_cost(BRANCH_COST);
15770 format %{ "cb$cmp $oop, $labl" %}
15771 ins_encode %{
15772 Label* L = $labl$$label;
15773 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15774 if (cond == Assembler::EQ)
15775 __ cbzw($oop$$Register, *L);
15776 else
15777 __ cbnzw($oop$$Register, *L);
15778 %}
15779 ins_pipe(pipe_cmp_branch);
15780 %}
15781
15782 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15783 match(If cmp (CmpU op1 op2));
15784 effect(USE labl);
15785
15786 ins_cost(BRANCH_COST);
15787 format %{ "cbw$cmp $op1, $labl" %}
15788 ins_encode %{
15789 Label* L = $labl$$label;
15790 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15791 if (cond == Assembler::EQ || cond == Assembler::LS) {
15792 __ cbzw($op1$$Register, *L);
15793 } else {
15794 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15795 __ cbnzw($op1$$Register, *L);
15796 }
15797 %}
15798 ins_pipe(pipe_cmp_branch);
15799 %}
15800
15801 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15802 match(If cmp (CmpUL op1 op2));
15803 effect(USE labl);
15804
15805 ins_cost(BRANCH_COST);
15806 format %{ "cb$cmp $op1, $labl" %}
15807 ins_encode %{
15808 Label* L = $labl$$label;
15809 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15810 if (cond == Assembler::EQ || cond == Assembler::LS) {
15811 __ cbz($op1$$Register, *L);
15812 } else {
15813 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15814 __ cbnz($op1$$Register, *L);
15815 }
15816 %}
15817 ins_pipe(pipe_cmp_branch);
15818 %}
15819
15820 // Test bit and Branch
15821
15822 // Patterns for short (< 32KiB) variants
15823 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15824 match(If cmp (CmpL op1 op2));
15825 effect(USE labl);
15826
15827 ins_cost(BRANCH_COST);
15828 format %{ "cb$cmp $op1, $labl # long" %}
15829 ins_encode %{
15830 Label* L = $labl$$label;
15831 Assembler::Condition cond =
15832 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15833 __ tbr(cond, $op1$$Register, 63, *L);
15834 %}
15835 ins_pipe(pipe_cmp_branch);
15836 ins_short_branch(1);
15837 %}
15838
15839 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15840 match(If cmp (CmpI op1 op2));
15841 effect(USE labl);
15842
15843 ins_cost(BRANCH_COST);
15844 format %{ "cb$cmp $op1, $labl # int" %}
15845 ins_encode %{
15846 Label* L = $labl$$label;
15847 Assembler::Condition cond =
15848 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15849 __ tbr(cond, $op1$$Register, 31, *L);
15850 %}
15851 ins_pipe(pipe_cmp_branch);
15852 ins_short_branch(1);
15853 %}
15854
15855 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15856 match(If cmp (CmpL (AndL op1 op2) op3));
15857 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15858 effect(USE labl);
15859
15860 ins_cost(BRANCH_COST);
15861 format %{ "tb$cmp $op1, $op2, $labl" %}
15862 ins_encode %{
15863 Label* L = $labl$$label;
15864 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15865 int bit = exact_log2_long($op2$$constant);
15866 __ tbr(cond, $op1$$Register, bit, *L);
15867 %}
15868 ins_pipe(pipe_cmp_branch);
15869 ins_short_branch(1);
15870 %}
15871
15872 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15873 match(If cmp (CmpI (AndI op1 op2) op3));
15874 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15875 effect(USE labl);
15876
15877 ins_cost(BRANCH_COST);
15878 format %{ "tb$cmp $op1, $op2, $labl" %}
15879 ins_encode %{
15880 Label* L = $labl$$label;
15881 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15882 int bit = exact_log2((juint)$op2$$constant);
15883 __ tbr(cond, $op1$$Register, bit, *L);
15884 %}
15885 ins_pipe(pipe_cmp_branch);
15886 ins_short_branch(1);
15887 %}
15888
15889 // And far variants
15890 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15891 match(If cmp (CmpL op1 op2));
15892 effect(USE labl);
15893
15894 ins_cost(BRANCH_COST);
15895 format %{ "cb$cmp $op1, $labl # long" %}
15896 ins_encode %{
15897 Label* L = $labl$$label;
15898 Assembler::Condition cond =
15899 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15900 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15901 %}
15902 ins_pipe(pipe_cmp_branch);
15903 %}
15904
15905 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15906 match(If cmp (CmpI op1 op2));
15907 effect(USE labl);
15908
15909 ins_cost(BRANCH_COST);
15910 format %{ "cb$cmp $op1, $labl # int" %}
15911 ins_encode %{
15912 Label* L = $labl$$label;
15913 Assembler::Condition cond =
15914 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15915 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15916 %}
15917 ins_pipe(pipe_cmp_branch);
15918 %}
15919
15920 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15921 match(If cmp (CmpL (AndL op1 op2) op3));
15922 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15923 effect(USE labl);
15924
15925 ins_cost(BRANCH_COST);
15926 format %{ "tb$cmp $op1, $op2, $labl" %}
15927 ins_encode %{
15928 Label* L = $labl$$label;
15929 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15930 int bit = exact_log2_long($op2$$constant);
15931 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15932 %}
15933 ins_pipe(pipe_cmp_branch);
15934 %}
15935
15936 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15937 match(If cmp (CmpI (AndI op1 op2) op3));
15938 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15939 effect(USE labl);
15940
15941 ins_cost(BRANCH_COST);
15942 format %{ "tb$cmp $op1, $op2, $labl" %}
15943 ins_encode %{
15944 Label* L = $labl$$label;
15945 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15946 int bit = exact_log2((juint)$op2$$constant);
15947 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15948 %}
15949 ins_pipe(pipe_cmp_branch);
15950 %}
15951
15952 // Test bits
15953
15954 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15955 match(Set cr (CmpL (AndL op1 op2) op3));
15956 predicate(Assembler::operand_valid_for_logical_immediate
15957 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15958
15959 ins_cost(INSN_COST);
15960 format %{ "tst $op1, $op2 # long" %}
15961 ins_encode %{
15962 __ tst($op1$$Register, $op2$$constant);
15963 %}
15964 ins_pipe(ialu_reg_reg);
15965 %}
15966
15967 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15968 match(Set cr (CmpI (AndI op1 op2) op3));
15969 predicate(Assembler::operand_valid_for_logical_immediate
15970 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15971
15972 ins_cost(INSN_COST);
15973 format %{ "tst $op1, $op2 # int" %}
15974 ins_encode %{
15975 __ tstw($op1$$Register, $op2$$constant);
15976 %}
15977 ins_pipe(ialu_reg_reg);
15978 %}
15979
15980 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15981 match(Set cr (CmpL (AndL op1 op2) op3));
15982
15983 ins_cost(INSN_COST);
15984 format %{ "tst $op1, $op2 # long" %}
15985 ins_encode %{
15986 __ tst($op1$$Register, $op2$$Register);
15987 %}
15988 ins_pipe(ialu_reg_reg);
15989 %}
15990
15991 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15992 match(Set cr (CmpI (AndI op1 op2) op3));
15993
15994 ins_cost(INSN_COST);
15995 format %{ "tstw $op1, $op2 # int" %}
15996 ins_encode %{
15997 __ tstw($op1$$Register, $op2$$Register);
15998 %}
15999 ins_pipe(ialu_reg_reg);
16000 %}
16001
16002
16003 // Conditional Far Branch
16004 // Conditional Far Branch Unsigned
16005 // TODO: fixme
16006
16007 // counted loop end branch near
16008 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16009 %{
16010 match(CountedLoopEnd cmp cr);
16011
16012 effect(USE lbl);
16013
16014 ins_cost(BRANCH_COST);
16015 // short variant.
16016 // ins_short_branch(1);
16017 format %{ "b$cmp $lbl \t// counted loop end" %}
16018
16019 ins_encode(aarch64_enc_br_con(cmp, lbl));
16020
16021 ins_pipe(pipe_branch);
16022 %}
16023
16024 // counted loop end branch far
16025 // TODO: fixme
16026
16027 // ============================================================================
16028 // inlined locking and unlocking
16029
16030 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16031 %{
16032 predicate(LockingMode != LM_LIGHTWEIGHT);
16033 match(Set cr (FastLock object box));
16034 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16035
16036 ins_cost(5 * INSN_COST);
16037 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16038
16039 ins_encode %{
16040 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16041 %}
16042
16043 ins_pipe(pipe_serial);
16044 %}
16045
16046 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16047 %{
16048 predicate(LockingMode != LM_LIGHTWEIGHT);
16049 match(Set cr (FastUnlock object box));
16050 effect(TEMP tmp, TEMP tmp2);
16051
16052 ins_cost(5 * INSN_COST);
16053 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16054
16055 ins_encode %{
16056 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16057 %}
16058
16059 ins_pipe(pipe_serial);
16060 %}
16061
16062 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16063 %{
16064 predicate(LockingMode == LM_LIGHTWEIGHT);
16065 match(Set cr (FastLock object box));
16066 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16067
16068 ins_cost(5 * INSN_COST);
16069 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16070
16071 ins_encode %{
16072 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16073 %}
16074
16075 ins_pipe(pipe_serial);
16076 %}
16077
16078 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16079 %{
16080 predicate(LockingMode == LM_LIGHTWEIGHT);
16081 match(Set cr (FastUnlock object box));
16082 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16083
16084 ins_cost(5 * INSN_COST);
16085 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16086
16087 ins_encode %{
16088 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16089 %}
16090
16091 ins_pipe(pipe_serial);
16092 %}
16093
16094 // ============================================================================
16095 // Safepoint Instructions
16096
16097 // TODO
16098 // provide a near and far version of this code
16099
16100 instruct safePoint(rFlagsReg cr, iRegP poll)
16101 %{
16102 match(SafePoint poll);
16103 effect(KILL cr);
16104
16105 format %{
16106 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16107 %}
16108 ins_encode %{
16109 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16110 %}
16111 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16112 %}
16113
16114
16115 // ============================================================================
16116 // Procedure Call/Return Instructions
16117
16118 // Call Java Static Instruction
16119
16120 instruct CallStaticJavaDirect(method meth)
16121 %{
16122 match(CallStaticJava);
16123
16124 effect(USE meth);
16125
16126 ins_cost(CALL_COST);
16127
16128 format %{ "call,static $meth \t// ==> " %}
16129
16130 ins_encode(aarch64_enc_java_static_call(meth),
16131 aarch64_enc_call_epilog);
16132
16133 ins_pipe(pipe_class_call);
16134 %}
16135
16136 // TO HERE
16137
16138 // Call Java Dynamic Instruction
16139 instruct CallDynamicJavaDirect(method meth)
16140 %{
16141 match(CallDynamicJava);
16142
16143 effect(USE meth);
16144
16145 ins_cost(CALL_COST);
16146
16147 format %{ "CALL,dynamic $meth \t// ==> " %}
16148
16149 ins_encode(aarch64_enc_java_dynamic_call(meth),
16150 aarch64_enc_call_epilog);
16151
16152 ins_pipe(pipe_class_call);
16153 %}
16154
16155 // Call Runtime Instruction
16156
16157 instruct CallRuntimeDirect(method meth)
16158 %{
16159 match(CallRuntime);
16160
16161 effect(USE meth);
16162
16163 ins_cost(CALL_COST);
16164
16165 format %{ "CALL, runtime $meth" %}
16166
16167 ins_encode( aarch64_enc_java_to_runtime(meth) );
16168
16169 ins_pipe(pipe_class_call);
16170 %}
16171
16172 // Call Runtime Instruction
16173
16174 instruct CallLeafDirect(method meth)
16175 %{
16176 match(CallLeaf);
16177
16178 effect(USE meth);
16179
16180 ins_cost(CALL_COST);
16181
16182 format %{ "CALL, runtime leaf $meth" %}
16183
16184 ins_encode( aarch64_enc_java_to_runtime(meth) );
16185
16186 ins_pipe(pipe_class_call);
16187 %}
16188
16189 // Call Runtime Instruction without safepoint and with vector arguments
16190 instruct CallLeafDirectVector(method meth)
16191 %{
16192 match(CallLeafVector);
16193
16194 effect(USE meth);
16195
16196 ins_cost(CALL_COST);
16197
16198 format %{ "CALL, runtime leaf vector $meth" %}
16199
16200 ins_encode(aarch64_enc_java_to_runtime(meth));
16201
16202 ins_pipe(pipe_class_call);
16203 %}
16204
16205 // Call Runtime Instruction
16206
16207 // entry point is null, target holds the address to call
16208 instruct CallLeafNoFPIndirect(iRegP target)
16209 %{
16210 predicate(n->as_Call()->entry_point() == nullptr);
16211
16212 match(CallLeafNoFP target);
16213
16214 ins_cost(CALL_COST);
16215
16216 format %{ "CALL, runtime leaf nofp indirect $target" %}
16217
16218 ins_encode %{
16219 __ blr($target$$Register);
16220 %}
16221
16222 ins_pipe(pipe_class_call);
16223 %}
16224
16225 instruct CallLeafNoFPDirect(method meth)
16226 %{
16227 predicate(n->as_Call()->entry_point() != nullptr);
16228
16229 match(CallLeafNoFP);
16230
16231 effect(USE meth);
16232
16233 ins_cost(CALL_COST);
16234
16235 format %{ "CALL, runtime leaf nofp $meth" %}
16236
16237 ins_encode( aarch64_enc_java_to_runtime(meth) );
16238
16239 ins_pipe(pipe_class_call);
16240 %}
16241
16242 // Tail Call; Jump from runtime stub to Java code.
16243 // Also known as an 'interprocedural jump'.
16244 // Target of jump will eventually return to caller.
16245 // TailJump below removes the return address.
16246 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16247 // emitted just above the TailCall which has reset rfp to the caller state.
16248 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16249 %{
16250 match(TailCall jump_target method_ptr);
16251
16252 ins_cost(CALL_COST);
16253
16254 format %{ "br $jump_target\t# $method_ptr holds method" %}
16255
16256 ins_encode(aarch64_enc_tail_call(jump_target));
16257
16258 ins_pipe(pipe_class_call);
16259 %}
16260
16261 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16262 %{
16263 match(TailJump jump_target ex_oop);
16264
16265 ins_cost(CALL_COST);
16266
16267 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16268
16269 ins_encode(aarch64_enc_tail_jmp(jump_target));
16270
16271 ins_pipe(pipe_class_call);
16272 %}
16273
16274 // Forward exception.
16275 instruct ForwardExceptionjmp()
16276 %{
16277 match(ForwardException);
16278 ins_cost(CALL_COST);
16279
16280 format %{ "b forward_exception_stub" %}
16281 ins_encode %{
16282 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16283 %}
16284 ins_pipe(pipe_class_call);
16285 %}
16286
16287 // Create exception oop: created by stack-crawling runtime code.
16288 // Created exception is now available to this handler, and is setup
16289 // just prior to jumping to this handler. No code emitted.
16290 // TODO check
16291 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16292 instruct CreateException(iRegP_R0 ex_oop)
16293 %{
16294 match(Set ex_oop (CreateEx));
16295
16296 format %{ " -- \t// exception oop; no code emitted" %}
16297
16298 size(0);
16299
16300 ins_encode( /*empty*/ );
16301
16302 ins_pipe(pipe_class_empty);
16303 %}
16304
16305 // Rethrow exception: The exception oop will come in the first
16306 // argument position. Then JUMP (not call) to the rethrow stub code.
16307 instruct RethrowException() %{
16308 match(Rethrow);
16309 ins_cost(CALL_COST);
16310
16311 format %{ "b rethrow_stub" %}
16312
16313 ins_encode( aarch64_enc_rethrow() );
16314
16315 ins_pipe(pipe_class_call);
16316 %}
16317
16318
16319 // Return Instruction
16320 // epilog node loads ret address into lr as part of frame pop
16321 instruct Ret()
16322 %{
16323 match(Return);
16324
16325 format %{ "ret\t// return register" %}
16326
16327 ins_encode( aarch64_enc_ret() );
16328
16329 ins_pipe(pipe_branch);
16330 %}
16331
16332 // Die now.
16333 instruct ShouldNotReachHere() %{
16334 match(Halt);
16335
16336 ins_cost(CALL_COST);
16337 format %{ "ShouldNotReachHere" %}
16338
16339 ins_encode %{
16340 if (is_reachable()) {
16341 __ stop(_halt_reason);
16342 }
16343 %}
16344
16345 ins_pipe(pipe_class_default);
16346 %}
16347
16348 // ============================================================================
16349 // Partial Subtype Check
16350 //
16351 // superklass array for an instance of the superklass. Set a hidden
16352 // internal cache on a hit (cache is checked with exposed code in
16353 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16354 // encoding ALSO sets flags.
16355
16356 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16357 %{
16358 match(Set result (PartialSubtypeCheck sub super));
16359 predicate(!UseSecondarySupersTable);
16360 effect(KILL cr, KILL temp);
16361
16362 ins_cost(20 * INSN_COST); // slightly larger than the next version
16363 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16364
16365 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16366
16367 opcode(0x1); // Force zero of result reg on hit
16368
16369 ins_pipe(pipe_class_memory);
16370 %}
16371
16372 // Two versions of partialSubtypeCheck, both used when we need to
16373 // search for a super class in the secondary supers array. The first
16374 // is used when we don't know _a priori_ the class being searched
16375 // for. The second, far more common, is used when we do know: this is
16376 // used for instanceof, checkcast, and any case where C2 can determine
16377 // it by constant propagation.
16378
16379 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16380 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16381 rFlagsReg cr)
16382 %{
16383 match(Set result (PartialSubtypeCheck sub super));
16384 predicate(UseSecondarySupersTable);
16385 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16386
16387 ins_cost(10 * INSN_COST); // slightly larger than the next version
16388 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16389
16390 ins_encode %{
16391 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16392 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16393 $vtemp$$FloatRegister,
16394 $result$$Register, /*L_success*/nullptr);
16395 %}
16396
16397 ins_pipe(pipe_class_memory);
16398 %}
16399
16400 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16401 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16402 rFlagsReg cr)
16403 %{
16404 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16405 predicate(UseSecondarySupersTable);
16406 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16407
16408 ins_cost(5 * INSN_COST); // smaller than the next version
16409 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16410
16411 ins_encode %{
16412 bool success = false;
16413 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16414 if (InlineSecondarySupersTest) {
16415 success =
16416 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16417 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16418 $vtemp$$FloatRegister,
16419 $result$$Register,
16420 super_klass_slot);
16421 } else {
16422 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16423 success = (call != nullptr);
16424 }
16425 if (!success) {
16426 ciEnv::current()->record_failure("CodeCache is full");
16427 return;
16428 }
16429 %}
16430
16431 ins_pipe(pipe_class_memory);
16432 %}
16433
16434 // Intrisics for String.compareTo()
16435
16436 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16437 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16438 %{
16439 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16440 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16441 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16442
16443 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16444 ins_encode %{
16445 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16446 __ string_compare($str1$$Register, $str2$$Register,
16447 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16448 $tmp1$$Register, $tmp2$$Register,
16449 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16450 %}
16451 ins_pipe(pipe_class_memory);
16452 %}
16453
16454 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16455 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16456 %{
16457 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16458 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16459 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16460
16461 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16462 ins_encode %{
16463 __ string_compare($str1$$Register, $str2$$Register,
16464 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16465 $tmp1$$Register, $tmp2$$Register,
16466 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16467 %}
16468 ins_pipe(pipe_class_memory);
16469 %}
16470
16471 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16472 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16473 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16474 %{
16475 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16476 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16477 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16478 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16479
16480 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16481 ins_encode %{
16482 __ string_compare($str1$$Register, $str2$$Register,
16483 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16484 $tmp1$$Register, $tmp2$$Register,
16485 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16486 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16487 %}
16488 ins_pipe(pipe_class_memory);
16489 %}
16490
16491 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16492 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16493 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16494 %{
16495 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16496 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16497 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16498 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16499
16500 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16501 ins_encode %{
16502 __ string_compare($str1$$Register, $str2$$Register,
16503 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16504 $tmp1$$Register, $tmp2$$Register,
16505 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16506 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16507 %}
16508 ins_pipe(pipe_class_memory);
16509 %}
16510
16511 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16512 // these string_compare variants as NEON register type for convenience so that the prototype of
16513 // string_compare can be shared with all variants.
16514
16515 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16516 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16517 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16518 pRegGov_P1 pgtmp2, rFlagsReg cr)
16519 %{
16520 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16521 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16522 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16523 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16524
16525 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16526 ins_encode %{
16527 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16528 __ string_compare($str1$$Register, $str2$$Register,
16529 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16530 $tmp1$$Register, $tmp2$$Register,
16531 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16532 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16533 StrIntrinsicNode::LL);
16534 %}
16535 ins_pipe(pipe_class_memory);
16536 %}
16537
16538 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16539 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16540 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16541 pRegGov_P1 pgtmp2, rFlagsReg cr)
16542 %{
16543 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16544 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16545 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16546 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16547
16548 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16549 ins_encode %{
16550 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16551 __ string_compare($str1$$Register, $str2$$Register,
16552 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16553 $tmp1$$Register, $tmp2$$Register,
16554 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16555 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16556 StrIntrinsicNode::LU);
16557 %}
16558 ins_pipe(pipe_class_memory);
16559 %}
16560
16561 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16562 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16563 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16564 pRegGov_P1 pgtmp2, rFlagsReg cr)
16565 %{
16566 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16567 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16568 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16569 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16570
16571 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16572 ins_encode %{
16573 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16574 __ string_compare($str1$$Register, $str2$$Register,
16575 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16576 $tmp1$$Register, $tmp2$$Register,
16577 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16578 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16579 StrIntrinsicNode::UL);
16580 %}
16581 ins_pipe(pipe_class_memory);
16582 %}
16583
16584 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16585 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16586 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16587 pRegGov_P1 pgtmp2, rFlagsReg cr)
16588 %{
16589 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16590 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16591 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16592 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16593
16594 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16595 ins_encode %{
16596 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16597 __ string_compare($str1$$Register, $str2$$Register,
16598 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16599 $tmp1$$Register, $tmp2$$Register,
16600 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16601 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16602 StrIntrinsicNode::UU);
16603 %}
16604 ins_pipe(pipe_class_memory);
16605 %}
16606
16607 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16608 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16609 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16610 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16611 %{
16612 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16613 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16614 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16615 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16616 TEMP vtmp0, TEMP vtmp1, KILL cr);
16617 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16618 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16619
16620 ins_encode %{
16621 __ string_indexof($str1$$Register, $str2$$Register,
16622 $cnt1$$Register, $cnt2$$Register,
16623 $tmp1$$Register, $tmp2$$Register,
16624 $tmp3$$Register, $tmp4$$Register,
16625 $tmp5$$Register, $tmp6$$Register,
16626 -1, $result$$Register, StrIntrinsicNode::UU);
16627 %}
16628 ins_pipe(pipe_class_memory);
16629 %}
16630
16631 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16632 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16633 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16634 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16635 %{
16636 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16637 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16638 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16639 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16640 TEMP vtmp0, TEMP vtmp1, KILL cr);
16641 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16642 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16643
16644 ins_encode %{
16645 __ string_indexof($str1$$Register, $str2$$Register,
16646 $cnt1$$Register, $cnt2$$Register,
16647 $tmp1$$Register, $tmp2$$Register,
16648 $tmp3$$Register, $tmp4$$Register,
16649 $tmp5$$Register, $tmp6$$Register,
16650 -1, $result$$Register, StrIntrinsicNode::LL);
16651 %}
16652 ins_pipe(pipe_class_memory);
16653 %}
16654
16655 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16656 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16657 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16658 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16659 %{
16660 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16661 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16662 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16663 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16664 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16665 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16666 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16667
16668 ins_encode %{
16669 __ string_indexof($str1$$Register, $str2$$Register,
16670 $cnt1$$Register, $cnt2$$Register,
16671 $tmp1$$Register, $tmp2$$Register,
16672 $tmp3$$Register, $tmp4$$Register,
16673 $tmp5$$Register, $tmp6$$Register,
16674 -1, $result$$Register, StrIntrinsicNode::UL);
16675 %}
16676 ins_pipe(pipe_class_memory);
16677 %}
16678
16679 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16680 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16681 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16682 %{
16683 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16684 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16685 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16686 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16687 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16688 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16689
16690 ins_encode %{
16691 int icnt2 = (int)$int_cnt2$$constant;
16692 __ string_indexof($str1$$Register, $str2$$Register,
16693 $cnt1$$Register, zr,
16694 $tmp1$$Register, $tmp2$$Register,
16695 $tmp3$$Register, $tmp4$$Register, zr, zr,
16696 icnt2, $result$$Register, StrIntrinsicNode::UU);
16697 %}
16698 ins_pipe(pipe_class_memory);
16699 %}
16700
16701 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16702 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16703 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16704 %{
16705 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16706 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16707 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16708 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16709 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16710 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16711
16712 ins_encode %{
16713 int icnt2 = (int)$int_cnt2$$constant;
16714 __ string_indexof($str1$$Register, $str2$$Register,
16715 $cnt1$$Register, zr,
16716 $tmp1$$Register, $tmp2$$Register,
16717 $tmp3$$Register, $tmp4$$Register, zr, zr,
16718 icnt2, $result$$Register, StrIntrinsicNode::LL);
16719 %}
16720 ins_pipe(pipe_class_memory);
16721 %}
16722
16723 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16724 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16725 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16726 %{
16727 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16728 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16729 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16730 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16731 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16732 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16733
16734 ins_encode %{
16735 int icnt2 = (int)$int_cnt2$$constant;
16736 __ string_indexof($str1$$Register, $str2$$Register,
16737 $cnt1$$Register, zr,
16738 $tmp1$$Register, $tmp2$$Register,
16739 $tmp3$$Register, $tmp4$$Register, zr, zr,
16740 icnt2, $result$$Register, StrIntrinsicNode::UL);
16741 %}
16742 ins_pipe(pipe_class_memory);
16743 %}
16744
16745 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16746 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16747 iRegINoSp tmp3, rFlagsReg cr)
16748 %{
16749 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16750 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16751 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16752 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16753
16754 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16755
16756 ins_encode %{
16757 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16758 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16759 $tmp3$$Register);
16760 %}
16761 ins_pipe(pipe_class_memory);
16762 %}
16763
16764 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16765 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16766 iRegINoSp tmp3, rFlagsReg cr)
16767 %{
16768 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16769 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16770 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16771 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16772
16773 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16774
16775 ins_encode %{
16776 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16777 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16778 $tmp3$$Register);
16779 %}
16780 ins_pipe(pipe_class_memory);
16781 %}
16782
16783 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16784 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16785 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16786 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16787 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16788 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16789 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16790 ins_encode %{
16791 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16792 $result$$Register, $ztmp1$$FloatRegister,
16793 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16794 $ptmp$$PRegister, true /* isL */);
16795 %}
16796 ins_pipe(pipe_class_memory);
16797 %}
16798
16799 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16800 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16801 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16802 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16803 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16804 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16805 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16806 ins_encode %{
16807 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16808 $result$$Register, $ztmp1$$FloatRegister,
16809 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16810 $ptmp$$PRegister, false /* isL */);
16811 %}
16812 ins_pipe(pipe_class_memory);
16813 %}
16814
16815 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16816 iRegI_R0 result, rFlagsReg cr)
16817 %{
16818 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16819 match(Set result (StrEquals (Binary str1 str2) cnt));
16820 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16821
16822 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16823 ins_encode %{
16824 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16825 __ string_equals($str1$$Register, $str2$$Register,
16826 $result$$Register, $cnt$$Register);
16827 %}
16828 ins_pipe(pipe_class_memory);
16829 %}
16830
16831 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16832 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16833 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16834 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16835 iRegP_R10 tmp, rFlagsReg cr)
16836 %{
16837 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16838 match(Set result (AryEq ary1 ary2));
16839 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16840 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16841 TEMP vtmp6, TEMP vtmp7, KILL cr);
16842
16843 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16844 ins_encode %{
16845 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16846 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16847 $result$$Register, $tmp$$Register, 1);
16848 if (tpc == nullptr) {
16849 ciEnv::current()->record_failure("CodeCache is full");
16850 return;
16851 }
16852 %}
16853 ins_pipe(pipe_class_memory);
16854 %}
16855
16856 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16857 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16858 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16859 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16860 iRegP_R10 tmp, rFlagsReg cr)
16861 %{
16862 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16863 match(Set result (AryEq ary1 ary2));
16864 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16865 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16866 TEMP vtmp6, TEMP vtmp7, KILL cr);
16867
16868 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16869 ins_encode %{
16870 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16871 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16872 $result$$Register, $tmp$$Register, 2);
16873 if (tpc == nullptr) {
16874 ciEnv::current()->record_failure("CodeCache is full");
16875 return;
16876 }
16877 %}
16878 ins_pipe(pipe_class_memory);
16879 %}
16880
16881 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16882 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16883 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16884 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16885 %{
16886 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16887 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16888 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16889
16890 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16891 ins_encode %{
16892 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16893 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16894 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16895 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16896 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16897 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16898 (BasicType)$basic_type$$constant);
16899 if (tpc == nullptr) {
16900 ciEnv::current()->record_failure("CodeCache is full");
16901 return;
16902 }
16903 %}
16904 ins_pipe(pipe_class_memory);
16905 %}
16906
16907 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16908 %{
16909 match(Set result (CountPositives ary1 len));
16910 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16911 format %{ "count positives byte[] $ary1,$len -> $result" %}
16912 ins_encode %{
16913 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16914 if (tpc == nullptr) {
16915 ciEnv::current()->record_failure("CodeCache is full");
16916 return;
16917 }
16918 %}
16919 ins_pipe( pipe_slow );
16920 %}
16921
16922 // fast char[] to byte[] compression
16923 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16924 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16925 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16926 iRegI_R0 result, rFlagsReg cr)
16927 %{
16928 match(Set result (StrCompressedCopy src (Binary dst len)));
16929 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16930 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16931
16932 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16933 ins_encode %{
16934 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16935 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16936 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16937 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16938 %}
16939 ins_pipe(pipe_slow);
16940 %}
16941
16942 // fast byte[] to char[] inflation
16943 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16944 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16945 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16946 %{
16947 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16948 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16949 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16950 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16951
16952 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16953 ins_encode %{
16954 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16955 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16956 $vtmp2$$FloatRegister, $tmp$$Register);
16957 if (tpc == nullptr) {
16958 ciEnv::current()->record_failure("CodeCache is full");
16959 return;
16960 }
16961 %}
16962 ins_pipe(pipe_class_memory);
16963 %}
16964
16965 // encode char[] to byte[] in ISO_8859_1
16966 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16967 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16968 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16969 iRegI_R0 result, rFlagsReg cr)
16970 %{
16971 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16972 match(Set result (EncodeISOArray src (Binary dst len)));
16973 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16974 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16975
16976 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16977 ins_encode %{
16978 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16979 $result$$Register, false,
16980 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16981 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16982 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16983 %}
16984 ins_pipe(pipe_class_memory);
16985 %}
16986
16987 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16988 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16989 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16990 iRegI_R0 result, rFlagsReg cr)
16991 %{
16992 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16993 match(Set result (EncodeISOArray src (Binary dst len)));
16994 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16995 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16996
16997 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16998 ins_encode %{
16999 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17000 $result$$Register, true,
17001 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17002 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17003 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17004 %}
17005 ins_pipe(pipe_class_memory);
17006 %}
17007
17008 //----------------------------- CompressBits/ExpandBits ------------------------
17009
17010 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17011 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17012 match(Set dst (CompressBits src mask));
17013 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17014 format %{ "mov $tsrc, $src\n\t"
17015 "mov $tmask, $mask\n\t"
17016 "bext $tdst, $tsrc, $tmask\n\t"
17017 "mov $dst, $tdst"
17018 %}
17019 ins_encode %{
17020 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17021 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17022 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17023 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17024 %}
17025 ins_pipe(pipe_slow);
17026 %}
17027
17028 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17029 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17030 match(Set dst (CompressBits (LoadI mem) mask));
17031 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17032 format %{ "ldrs $tsrc, $mem\n\t"
17033 "ldrs $tmask, $mask\n\t"
17034 "bext $tdst, $tsrc, $tmask\n\t"
17035 "mov $dst, $tdst"
17036 %}
17037 ins_encode %{
17038 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17039 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17040 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17041 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17042 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17043 %}
17044 ins_pipe(pipe_slow);
17045 %}
17046
17047 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17048 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17049 match(Set dst (CompressBits src mask));
17050 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17051 format %{ "mov $tsrc, $src\n\t"
17052 "mov $tmask, $mask\n\t"
17053 "bext $tdst, $tsrc, $tmask\n\t"
17054 "mov $dst, $tdst"
17055 %}
17056 ins_encode %{
17057 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17058 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17059 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17060 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17061 %}
17062 ins_pipe(pipe_slow);
17063 %}
17064
17065 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17066 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17067 match(Set dst (CompressBits (LoadL mem) mask));
17068 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17069 format %{ "ldrd $tsrc, $mem\n\t"
17070 "ldrd $tmask, $mask\n\t"
17071 "bext $tdst, $tsrc, $tmask\n\t"
17072 "mov $dst, $tdst"
17073 %}
17074 ins_encode %{
17075 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17076 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17077 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17078 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17079 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17080 %}
17081 ins_pipe(pipe_slow);
17082 %}
17083
17084 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17085 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17086 match(Set dst (ExpandBits src mask));
17087 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17088 format %{ "mov $tsrc, $src\n\t"
17089 "mov $tmask, $mask\n\t"
17090 "bdep $tdst, $tsrc, $tmask\n\t"
17091 "mov $dst, $tdst"
17092 %}
17093 ins_encode %{
17094 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17095 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17096 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17097 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17098 %}
17099 ins_pipe(pipe_slow);
17100 %}
17101
17102 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17103 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17104 match(Set dst (ExpandBits (LoadI mem) mask));
17105 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17106 format %{ "ldrs $tsrc, $mem\n\t"
17107 "ldrs $tmask, $mask\n\t"
17108 "bdep $tdst, $tsrc, $tmask\n\t"
17109 "mov $dst, $tdst"
17110 %}
17111 ins_encode %{
17112 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17113 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17114 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17115 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17116 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17117 %}
17118 ins_pipe(pipe_slow);
17119 %}
17120
17121 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17122 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17123 match(Set dst (ExpandBits src mask));
17124 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17125 format %{ "mov $tsrc, $src\n\t"
17126 "mov $tmask, $mask\n\t"
17127 "bdep $tdst, $tsrc, $tmask\n\t"
17128 "mov $dst, $tdst"
17129 %}
17130 ins_encode %{
17131 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17132 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17133 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17134 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17135 %}
17136 ins_pipe(pipe_slow);
17137 %}
17138
17139
17140 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17141 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17142 match(Set dst (ExpandBits (LoadL mem) mask));
17143 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17144 format %{ "ldrd $tsrc, $mem\n\t"
17145 "ldrd $tmask, $mask\n\t"
17146 "bdep $tdst, $tsrc, $tmask\n\t"
17147 "mov $dst, $tdst"
17148 %}
17149 ins_encode %{
17150 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17151 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17152 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17153 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17154 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17155 %}
17156 ins_pipe(pipe_slow);
17157 %}
17158
17159 // ============================================================================
17160 // This name is KNOWN by the ADLC and cannot be changed.
17161 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17162 // for this guy.
17163 instruct tlsLoadP(thread_RegP dst)
17164 %{
17165 match(Set dst (ThreadLocal));
17166
17167 ins_cost(0);
17168
17169 format %{ " -- \t// $dst=Thread::current(), empty" %}
17170
17171 size(0);
17172
17173 ins_encode( /*empty*/ );
17174
17175 ins_pipe(pipe_class_empty);
17176 %}
17177
17178 //----------PEEPHOLE RULES-----------------------------------------------------
17179 // These must follow all instruction definitions as they use the names
17180 // defined in the instructions definitions.
17181 //
17182 // peepmatch ( root_instr_name [preceding_instruction]* );
17183 //
17184 // peepconstraint %{
17185 // (instruction_number.operand_name relational_op instruction_number.operand_name
17186 // [, ...] );
17187 // // instruction numbers are zero-based using left to right order in peepmatch
17188 //
17189 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17190 // // provide an instruction_number.operand_name for each operand that appears
17191 // // in the replacement instruction's match rule
17192 //
17193 // ---------VM FLAGS---------------------------------------------------------
17194 //
17195 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17196 //
17197 // Each peephole rule is given an identifying number starting with zero and
17198 // increasing by one in the order seen by the parser. An individual peephole
17199 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17200 // on the command-line.
17201 //
17202 // ---------CURRENT LIMITATIONS----------------------------------------------
17203 //
17204 // Only match adjacent instructions in same basic block
17205 // Only equality constraints
17206 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17207 // Only one replacement instruction
17208 //
17209 // ---------EXAMPLE----------------------------------------------------------
17210 //
17211 // // pertinent parts of existing instructions in architecture description
17212 // instruct movI(iRegINoSp dst, iRegI src)
17213 // %{
17214 // match(Set dst (CopyI src));
17215 // %}
17216 //
17217 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17218 // %{
17219 // match(Set dst (AddI dst src));
17220 // effect(KILL cr);
17221 // %}
17222 //
17223 // // Change (inc mov) to lea
17224 // peephole %{
17225 // // increment preceded by register-register move
17226 // peepmatch ( incI_iReg movI );
17227 // // require that the destination register of the increment
17228 // // match the destination register of the move
17229 // peepconstraint ( 0.dst == 1.dst );
17230 // // construct a replacement instruction that sets
17231 // // the destination to ( move's source register + one )
17232 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17233 // %}
17234 //
17235
17236 // Implementation no longer uses movX instructions since
17237 // machine-independent system no longer uses CopyX nodes.
17238 //
17239 // peephole
17240 // %{
17241 // peepmatch (incI_iReg movI);
17242 // peepconstraint (0.dst == 1.dst);
17243 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17244 // %}
17245
17246 // peephole
17247 // %{
17248 // peepmatch (decI_iReg movI);
17249 // peepconstraint (0.dst == 1.dst);
17250 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17251 // %}
17252
17253 // peephole
17254 // %{
17255 // peepmatch (addI_iReg_imm movI);
17256 // peepconstraint (0.dst == 1.dst);
17257 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17258 // %}
17259
17260 // peephole
17261 // %{
17262 // peepmatch (incL_iReg movL);
17263 // peepconstraint (0.dst == 1.dst);
17264 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17265 // %}
17266
17267 // peephole
17268 // %{
17269 // peepmatch (decL_iReg movL);
17270 // peepconstraint (0.dst == 1.dst);
17271 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17272 // %}
17273
17274 // peephole
17275 // %{
17276 // peepmatch (addL_iReg_imm movL);
17277 // peepconstraint (0.dst == 1.dst);
17278 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17279 // %}
17280
17281 // peephole
17282 // %{
17283 // peepmatch (addP_iReg_imm movP);
17284 // peepconstraint (0.dst == 1.dst);
17285 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17286 // %}
17287
17288 // // Change load of spilled value to only a spill
17289 // instruct storeI(memory mem, iRegI src)
17290 // %{
17291 // match(Set mem (StoreI mem src));
17292 // %}
17293 //
17294 // instruct loadI(iRegINoSp dst, memory mem)
17295 // %{
17296 // match(Set dst (LoadI mem));
17297 // %}
17298 //
17299
17300 //----------SMARTSPILL RULES---------------------------------------------------
17301 // These must follow all instruction definitions as they use the names
17302 // defined in the instructions definitions.
17303
17304 // Local Variables:
17305 // mode: c++
17306 // End: