1 //
2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return 0; // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1652 // lea(rscratch1, RuntimeAddress(addr)
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else {
1658 return 6 * NativeInstruction::instruction_size;
1659 }
1660 }
1661
1662 //=============================================================================
1663
1664 #ifndef PRODUCT
1665 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1666 st->print("BREAKPOINT");
1667 }
1668 #endif
1669
1670 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1671 __ brk(0);
1672 }
1673
1674 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1675 return MachNode::size(ra_);
1676 }
1677
1678 //=============================================================================
1679
1680 #ifndef PRODUCT
1681 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1682 st->print("nop \t# %d bytes pad for loops and calls", _count);
1683 }
1684 #endif
1685
1686 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1687 for (int i = 0; i < _count; i++) {
1688 __ nop();
1689 }
1690 }
1691
1692 uint MachNopNode::size(PhaseRegAlloc*) const {
1693 return _count * NativeInstruction::instruction_size;
1694 }
1695
1696 //=============================================================================
1697 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1698
1699 int ConstantTable::calculate_table_base_offset() const {
1700 return 0; // absolute addressing, no offset
1701 }
1702
1703 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1704 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1705 ShouldNotReachHere();
1706 }
1707
1708 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1709 // Empty encoding
1710 }
1711
1712 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1713 return 0;
1714 }
1715
1716 #ifndef PRODUCT
1717 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1718 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1719 }
1720 #endif
1721
1722 #ifndef PRODUCT
1723 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1724 Compile* C = ra_->C;
1725
1726 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1727
1728 if (C->output()->need_stack_bang(framesize))
1729 st->print("# stack bang size=%d\n\t", framesize);
1730
1731 if (VM_Version::use_rop_protection()) {
1732 st->print("ldr zr, [lr]\n\t");
1733 st->print("paciaz\n\t");
1734 }
1735 if (framesize < ((1 << 9) + 2 * wordSize)) {
1736 st->print("sub sp, sp, #%d\n\t", framesize);
1737 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1738 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1739 } else {
1740 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1741 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1742 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1743 st->print("sub sp, sp, rscratch1");
1744 }
1745 if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1746 st->print("\n\t");
1747 st->print("ldr rscratch1, [guard]\n\t");
1748 st->print("dmb ishld\n\t");
1749 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1750 st->print("cmp rscratch1, rscratch2\n\t");
1751 st->print("b.eq skip");
1752 st->print("\n\t");
1753 st->print("blr #nmethod_entry_barrier_stub\n\t");
1754 st->print("b skip\n\t");
1755 st->print("guard: int\n\t");
1756 st->print("\n\t");
1757 st->print("skip:\n\t");
1758 }
1759 }
1760 #endif
1761
1762 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1763 Compile* C = ra_->C;
1764
1765 // n.b. frame size includes space for return pc and rfp
1766 const int framesize = C->output()->frame_size_in_bytes();
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 if (C->clinit_barrier_on_entry()) {
1773 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1774
1775 Label L_skip_barrier;
1776
1777 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1778 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1779 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1780 __ bind(L_skip_barrier);
1781 }
1782
1783 if (C->max_vector_size() > 0) {
1784 __ reinitialize_ptrue();
1785 }
1786
1787 int bangsize = C->output()->bang_size_in_bytes();
1788 if (C->output()->need_stack_bang(bangsize))
1789 __ generate_stack_overflow_check(bangsize);
1790
1791 __ build_frame(framesize);
1792
1793 if (C->stub_function() == nullptr) {
1794 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1795 if (BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1796 // Dummy labels for just measuring the code size
1797 Label dummy_slow_path;
1798 Label dummy_continuation;
1799 Label dummy_guard;
1800 Label* slow_path = &dummy_slow_path;
1801 Label* continuation = &dummy_continuation;
1802 Label* guard = &dummy_guard;
1803 if (!Compile::current()->output()->in_scratch_emit_size()) {
1804 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1805 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1806 Compile::current()->output()->add_stub(stub);
1807 slow_path = &stub->entry();
1808 continuation = &stub->continuation();
1809 guard = &stub->guard();
1810 }
1811 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1812 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1813 }
1814 }
1815
1816 if (VerifyStackAtCalls) {
1817 Unimplemented();
1818 }
1819
1820 C->output()->set_frame_complete(__ offset());
1821
1822 if (C->has_mach_constant_base_node()) {
1823 // NOTE: We set the table base offset here because users might be
1824 // emitted before MachConstantBaseNode.
1825 ConstantTable& constant_table = C->output()->constant_table();
1826 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1827 }
1828 }
1829
1830 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1831 {
1832 return MachNode::size(ra_); // too many variables; just compute it
1833 // the hard way
1834 }
1835
1836 int MachPrologNode::reloc() const
1837 {
1838 return 0;
1839 }
1840
1841 //=============================================================================
1842
1843 #ifndef PRODUCT
1844 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1845 Compile* C = ra_->C;
1846 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1847
1848 st->print("# pop frame %d\n\t",framesize);
1849
1850 if (framesize == 0) {
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1853 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1854 st->print("add sp, sp, #%d\n\t", framesize);
1855 } else {
1856 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1857 st->print("add sp, sp, rscratch1\n\t");
1858 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1859 }
1860 if (VM_Version::use_rop_protection()) {
1861 st->print("autiaz\n\t");
1862 st->print("ldr zr, [lr]\n\t");
1863 }
1864
1865 if (do_polling() && C->is_method_compilation()) {
1866 st->print("# test polling word\n\t");
1867 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1868 st->print("cmp sp, rscratch1\n\t");
1869 st->print("bhi #slow_path");
1870 }
1871 }
1872 #endif
1873
1874 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 __ remove_frame(framesize);
1879
1880 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1881 __ reserved_stack_check();
1882 }
1883
1884 if (do_polling() && C->is_method_compilation()) {
1885 Label dummy_label;
1886 Label* code_stub = &dummy_label;
1887 if (!C->output()->in_scratch_emit_size()) {
1888 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1889 C->output()->add_stub(stub);
1890 code_stub = &stub->entry();
1891 }
1892 __ relocate(relocInfo::poll_return_type);
1893 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1894 }
1895 }
1896
1897 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1898 // Variable size. Determine dynamically.
1899 return MachNode::size(ra_);
1900 }
1901
1902 int MachEpilogNode::reloc() const {
1903 // Return number of relocatable values contained in this instruction.
1904 return 1; // 1 for polling page.
1905 }
1906
1907 const Pipeline * MachEpilogNode::pipeline() const {
1908 return MachNode::pipeline_class();
1909 }
1910
1911 //=============================================================================
1912
1913 static enum RC rc_class(OptoReg::Name reg) {
1914
1915 if (reg == OptoReg::Bad) {
1916 return rc_bad;
1917 }
1918
1919 // we have 32 int registers * 2 halves
1920 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1921
1922 if (reg < slots_of_int_registers) {
1923 return rc_int;
1924 }
1925
1926 // we have 32 float register * 8 halves
1927 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1928 if (reg < slots_of_int_registers + slots_of_float_registers) {
1929 return rc_float;
1930 }
1931
1932 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1933 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1934 return rc_predicate;
1935 }
1936
1937 // Between predicate regs & stack is the flags.
1938 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1939
1940 return rc_stack;
1941 }
1942
1943 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1944 Compile* C = ra_->C;
1945
1946 // Get registers to move.
1947 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1948 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1949 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1950 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1951
1952 enum RC src_hi_rc = rc_class(src_hi);
1953 enum RC src_lo_rc = rc_class(src_lo);
1954 enum RC dst_hi_rc = rc_class(dst_hi);
1955 enum RC dst_lo_rc = rc_class(dst_lo);
1956
1957 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1958
1959 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1960 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1961 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1962 "expected aligned-adjacent pairs");
1963 }
1964
1965 if (src_lo == dst_lo && src_hi == dst_hi) {
1966 return 0; // Self copy, no move.
1967 }
1968
1969 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1970 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1971 int src_offset = ra_->reg2offset(src_lo);
1972 int dst_offset = ra_->reg2offset(dst_lo);
1973
1974 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1975 uint ireg = ideal_reg();
1976 if (ireg == Op_VecA && masm) {
1977 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1978 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1979 // stack->stack
1980 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1981 sve_vector_reg_size_in_bytes);
1982 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1983 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1984 sve_vector_reg_size_in_bytes);
1985 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1986 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1987 sve_vector_reg_size_in_bytes);
1988 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1989 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1990 as_FloatRegister(Matcher::_regEncode[src_lo]),
1991 as_FloatRegister(Matcher::_regEncode[src_lo]));
1992 } else {
1993 ShouldNotReachHere();
1994 }
1995 } else if (masm) {
1996 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1997 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1998 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1999 // stack->stack
2000 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2001 if (ireg == Op_VecD) {
2002 __ unspill(rscratch1, true, src_offset);
2003 __ spill(rscratch1, true, dst_offset);
2004 } else {
2005 __ spill_copy128(src_offset, dst_offset);
2006 }
2007 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2008 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2009 ireg == Op_VecD ? __ T8B : __ T16B,
2010 as_FloatRegister(Matcher::_regEncode[src_lo]));
2011 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2012 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2013 ireg == Op_VecD ? __ D : __ Q,
2014 ra_->reg2offset(dst_lo));
2015 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2016 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 ireg == Op_VecD ? __ D : __ Q,
2018 ra_->reg2offset(src_lo));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 }
2023 } else if (masm) {
2024 switch (src_lo_rc) {
2025 case rc_int:
2026 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2027 if (is64) {
2028 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2029 as_Register(Matcher::_regEncode[src_lo]));
2030 } else {
2031 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2032 as_Register(Matcher::_regEncode[src_lo]));
2033 }
2034 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2035 if (is64) {
2036 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2037 as_Register(Matcher::_regEncode[src_lo]));
2038 } else {
2039 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2040 as_Register(Matcher::_regEncode[src_lo]));
2041 }
2042 } else { // gpr --> stack spill
2043 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2044 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2045 }
2046 break;
2047 case rc_float:
2048 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2049 if (is64) {
2050 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2051 as_FloatRegister(Matcher::_regEncode[src_lo]));
2052 } else {
2053 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2054 as_FloatRegister(Matcher::_regEncode[src_lo]));
2055 }
2056 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2057 if (is64) {
2058 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2059 as_FloatRegister(Matcher::_regEncode[src_lo]));
2060 } else {
2061 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2062 as_FloatRegister(Matcher::_regEncode[src_lo]));
2063 }
2064 } else { // fpr --> stack spill
2065 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2066 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2067 is64 ? __ D : __ S, dst_offset);
2068 }
2069 break;
2070 case rc_stack:
2071 if (dst_lo_rc == rc_int) { // stack --> gpr load
2072 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2073 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2074 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2075 is64 ? __ D : __ S, src_offset);
2076 } else if (dst_lo_rc == rc_predicate) {
2077 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2078 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2079 } else { // stack --> stack copy
2080 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2081 if (ideal_reg() == Op_RegVectMask) {
2082 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2083 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2084 } else {
2085 __ unspill(rscratch1, is64, src_offset);
2086 __ spill(rscratch1, is64, dst_offset);
2087 }
2088 }
2089 break;
2090 case rc_predicate:
2091 if (dst_lo_rc == rc_predicate) {
2092 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2093 } else if (dst_lo_rc == rc_stack) {
2094 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2095 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2096 } else {
2097 assert(false, "bad src and dst rc_class combination.");
2098 ShouldNotReachHere();
2099 }
2100 break;
2101 default:
2102 assert(false, "bad rc_class for spill");
2103 ShouldNotReachHere();
2104 }
2105 }
2106
2107 if (st) {
2108 st->print("spill ");
2109 if (src_lo_rc == rc_stack) {
2110 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2111 } else {
2112 st->print("%s -> ", Matcher::regName[src_lo]);
2113 }
2114 if (dst_lo_rc == rc_stack) {
2115 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2116 } else {
2117 st->print("%s", Matcher::regName[dst_lo]);
2118 }
2119 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2120 int vsize = 0;
2121 switch (ideal_reg()) {
2122 case Op_VecD:
2123 vsize = 64;
2124 break;
2125 case Op_VecX:
2126 vsize = 128;
2127 break;
2128 case Op_VecA:
2129 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2130 break;
2131 default:
2132 assert(false, "bad register type for spill");
2133 ShouldNotReachHere();
2134 }
2135 st->print("\t# vector spill size = %d", vsize);
2136 } else if (ideal_reg() == Op_RegVectMask) {
2137 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2138 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2139 st->print("\t# predicate spill size = %d", vsize);
2140 } else {
2141 st->print("\t# spill size = %d", is64 ? 64 : 32);
2142 }
2143 }
2144
2145 return 0;
2146
2147 }
2148
2149 #ifndef PRODUCT
2150 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2151 if (!ra_)
2152 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2153 else
2154 implementation(nullptr, ra_, false, st);
2155 }
2156 #endif
2157
2158 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2159 implementation(masm, ra_, false, nullptr);
2160 }
2161
2162 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2163 return MachNode::size(ra_);
2164 }
2165
2166 //=============================================================================
2167
2168 #ifndef PRODUCT
2169 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2170 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2171 int reg = ra_->get_reg_first(this);
2172 st->print("add %s, rsp, #%d]\t# box lock",
2173 Matcher::regName[reg], offset);
2174 }
2175 #endif
2176
2177 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2178 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2179 int reg = ra_->get_encode(this);
2180
2181 // This add will handle any 24-bit signed offset. 24 bits allows an
2182 // 8 megabyte stack frame.
2183 __ add(as_Register(reg), sp, offset);
2184 }
2185
2186 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2187 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2188 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2189
2190 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2191 return NativeInstruction::instruction_size;
2192 } else {
2193 return 2 * NativeInstruction::instruction_size;
2194 }
2195 }
2196
2197 //=============================================================================
2198
2199 #ifndef PRODUCT
2200 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2201 {
2202 st->print_cr("# MachUEPNode");
2203 if (UseCompressedClassPointers) {
2204 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2205 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2206 st->print_cr("\tcmpw rscratch1, r10");
2207 } else {
2208 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2209 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2210 st->print_cr("\tcmp rscratch1, r10");
2211 }
2212 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2213 }
2214 #endif
2215
2216 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2217 {
2218 __ ic_check(InteriorEntryAlignment);
2219 }
2220
2221 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2222 {
2223 return MachNode::size(ra_);
2224 }
2225
2226 // REQUIRED EMIT CODE
2227
2228 //=============================================================================
2229
2230 // Emit exception handler code.
2231 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2232 {
2233 // mov rscratch1 #exception_blob_entry_point
2234 // br rscratch1
2235 // Note that the code buffer's insts_mark is always relative to insts.
2236 // That's why we must use the macroassembler to generate a handler.
2237 address base = __ start_a_stub(size_exception_handler());
2238 if (base == nullptr) {
2239 ciEnv::current()->record_failure("CodeCache is full");
2240 return 0; // CodeBuffer::expand failed
2241 }
2242 int offset = __ offset();
2243 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2244 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2245 __ end_a_stub();
2246 return offset;
2247 }
2248
2249 // Emit deopt handler code.
2250 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2251 {
2252 // Note that the code buffer's insts_mark is always relative to insts.
2253 // That's why we must use the macroassembler to generate a handler.
2254 address base = __ start_a_stub(size_deopt_handler());
2255 if (base == nullptr) {
2256 ciEnv::current()->record_failure("CodeCache is full");
2257 return 0; // CodeBuffer::expand failed
2258 }
2259 int offset = __ offset();
2260
2261 __ adr(lr, __ pc());
2262 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2263
2264 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2265 __ end_a_stub();
2266 return offset;
2267 }
2268
2269 // REQUIRED MATCHER CODE
2270
2271 //=============================================================================
2272
2273 bool Matcher::match_rule_supported(int opcode) {
2274 if (!has_match_rule(opcode))
2275 return false;
2276
2277 switch (opcode) {
2278 case Op_OnSpinWait:
2279 return VM_Version::supports_on_spin_wait();
2280 case Op_CacheWB:
2281 case Op_CacheWBPreSync:
2282 case Op_CacheWBPostSync:
2283 if (!VM_Version::supports_data_cache_line_flush()) {
2284 return false;
2285 }
2286 break;
2287 case Op_ExpandBits:
2288 case Op_CompressBits:
2289 if (!VM_Version::supports_svebitperm()) {
2290 return false;
2291 }
2292 break;
2293 case Op_FmaF:
2294 case Op_FmaD:
2295 case Op_FmaVF:
2296 case Op_FmaVD:
2297 if (!UseFMA) {
2298 return false;
2299 }
2300 break;
2301 }
2302
2303 return true; // Per default match rules are supported.
2304 }
2305
2306 const RegMask* Matcher::predicate_reg_mask(void) {
2307 return &_PR_REG_mask;
2308 }
2309
2310 bool Matcher::supports_vector_calling_convention(void) {
2311 return EnableVectorSupport && UseVectorStubs;
2312 }
2313
2314 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2315 assert(EnableVectorSupport && UseVectorStubs, "sanity");
2316 int lo = V0_num;
2317 int hi = V0_H_num;
2318 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2319 hi = V0_K_num;
2320 }
2321 return OptoRegPair(hi, lo);
2322 }
2323
2324 // Is this branch offset short enough that a short branch can be used?
2325 //
2326 // NOTE: If the platform does not provide any short branch variants, then
2327 // this method should return false for offset 0.
2328 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2329 // The passed offset is relative to address of the branch.
2330
2331 return (-32768 <= offset && offset < 32768);
2332 }
2333
2334 // Vector width in bytes.
2335 int Matcher::vector_width_in_bytes(BasicType bt) {
2336 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2337 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2338 // Minimum 2 values in vector
2339 if (size < 2*type2aelembytes(bt)) size = 0;
2340 // But never < 4
2341 if (size < 4) size = 0;
2342 return size;
2343 }
2344
2345 // Limits on vector size (number of elements) loaded into vector.
2346 int Matcher::max_vector_size(const BasicType bt) {
2347 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2348 }
2349
2350 int Matcher::min_vector_size(const BasicType bt) {
2351 int max_size = max_vector_size(bt);
2352 // Limit the min vector size to 8 bytes.
2353 int size = 8 / type2aelembytes(bt);
2354 if (bt == T_BYTE) {
2355 // To support vector api shuffle/rearrange.
2356 size = 4;
2357 } else if (bt == T_BOOLEAN) {
2358 // To support vector api load/store mask.
2359 size = 2;
2360 }
2361 if (size < 2) size = 2;
2362 return MIN2(size, max_size);
2363 }
2364
2365 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2366 return Matcher::max_vector_size(bt);
2367 }
2368
2369 // Actual max scalable vector register length.
2370 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2371 return Matcher::max_vector_size(bt);
2372 }
2373
2374 // Vector ideal reg.
2375 uint Matcher::vector_ideal_reg(int len) {
2376 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2377 return Op_VecA;
2378 }
2379 switch(len) {
2380 // For 16-bit/32-bit mask vector, reuse VecD.
2381 case 2:
2382 case 4:
2383 case 8: return Op_VecD;
2384 case 16: return Op_VecX;
2385 }
2386 ShouldNotReachHere();
2387 return 0;
2388 }
2389
2390 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2391 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2392 switch (ideal_reg) {
2393 case Op_VecA: return new vecAOper();
2394 case Op_VecD: return new vecDOper();
2395 case Op_VecX: return new vecXOper();
2396 }
2397 ShouldNotReachHere();
2398 return nullptr;
2399 }
2400
2401 bool Matcher::is_reg2reg_move(MachNode* m) {
2402 return false;
2403 }
2404
2405 bool Matcher::is_generic_vector(MachOper* opnd) {
2406 return opnd->opcode() == VREG;
2407 }
2408
2409 // Return whether or not this register is ever used as an argument.
2410 // This function is used on startup to build the trampoline stubs in
2411 // generateOptoStub. Registers not mentioned will be killed by the VM
2412 // call in the trampoline, and arguments in those registers not be
2413 // available to the callee.
2414 bool Matcher::can_be_java_arg(int reg)
2415 {
2416 return
2417 reg == R0_num || reg == R0_H_num ||
2418 reg == R1_num || reg == R1_H_num ||
2419 reg == R2_num || reg == R2_H_num ||
2420 reg == R3_num || reg == R3_H_num ||
2421 reg == R4_num || reg == R4_H_num ||
2422 reg == R5_num || reg == R5_H_num ||
2423 reg == R6_num || reg == R6_H_num ||
2424 reg == R7_num || reg == R7_H_num ||
2425 reg == V0_num || reg == V0_H_num ||
2426 reg == V1_num || reg == V1_H_num ||
2427 reg == V2_num || reg == V2_H_num ||
2428 reg == V3_num || reg == V3_H_num ||
2429 reg == V4_num || reg == V4_H_num ||
2430 reg == V5_num || reg == V5_H_num ||
2431 reg == V6_num || reg == V6_H_num ||
2432 reg == V7_num || reg == V7_H_num;
2433 }
2434
2435 bool Matcher::is_spillable_arg(int reg)
2436 {
2437 return can_be_java_arg(reg);
2438 }
2439
2440 uint Matcher::int_pressure_limit()
2441 {
2442 // JDK-8183543: When taking the number of available registers as int
2443 // register pressure threshold, the jtreg test:
2444 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2445 // failed due to C2 compilation failure with
2446 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2447 //
2448 // A derived pointer is live at CallNode and then is flagged by RA
2449 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2450 // derived pointers and lastly fail to spill after reaching maximum
2451 // number of iterations. Lowering the default pressure threshold to
2452 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2453 // a high register pressure area of the code so that split_DEF can
2454 // generate DefinitionSpillCopy for the derived pointer.
2455 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2456 if (!PreserveFramePointer) {
2457 // When PreserveFramePointer is off, frame pointer is allocatable,
2458 // but different from other SOC registers, it is excluded from
2459 // fatproj's mask because its save type is No-Save. Decrease 1 to
2460 // ensure high pressure at fatproj when PreserveFramePointer is off.
2461 // See check_pressure_at_fatproj().
2462 default_int_pressure_threshold--;
2463 }
2464 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2465 }
2466
2467 uint Matcher::float_pressure_limit()
2468 {
2469 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2470 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2471 }
2472
2473 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2474 return false;
2475 }
2476
2477 RegMask Matcher::divI_proj_mask() {
2478 ShouldNotReachHere();
2479 return RegMask();
2480 }
2481
2482 // Register for MODI projection of divmodI.
2483 RegMask Matcher::modI_proj_mask() {
2484 ShouldNotReachHere();
2485 return RegMask();
2486 }
2487
2488 // Register for DIVL projection of divmodL.
2489 RegMask Matcher::divL_proj_mask() {
2490 ShouldNotReachHere();
2491 return RegMask();
2492 }
2493
2494 // Register for MODL projection of divmodL.
2495 RegMask Matcher::modL_proj_mask() {
2496 ShouldNotReachHere();
2497 return RegMask();
2498 }
2499
2500 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2501 return FP_REG_mask();
2502 }
2503
2504 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2505 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2506 Node* u = addp->fast_out(i);
2507 if (u->is_LoadStore()) {
2508 // On AArch64, LoadStoreNodes (i.e. compare and swap
2509 // instructions) only take register indirect as an operand, so
2510 // any attempt to use an AddPNode as an input to a LoadStoreNode
2511 // must fail.
2512 return false;
2513 }
2514 if (u->is_Mem()) {
2515 int opsize = u->as_Mem()->memory_size();
2516 assert(opsize > 0, "unexpected memory operand size");
2517 if (u->as_Mem()->memory_size() != (1<<shift)) {
2518 return false;
2519 }
2520 }
2521 }
2522 return true;
2523 }
2524
2525 // Convert BootTest condition to Assembler condition.
2526 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2527 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2528 Assembler::Condition result;
2529 switch(cond) {
2530 case BoolTest::eq:
2531 result = Assembler::EQ; break;
2532 case BoolTest::ne:
2533 result = Assembler::NE; break;
2534 case BoolTest::le:
2535 result = Assembler::LE; break;
2536 case BoolTest::ge:
2537 result = Assembler::GE; break;
2538 case BoolTest::lt:
2539 result = Assembler::LT; break;
2540 case BoolTest::gt:
2541 result = Assembler::GT; break;
2542 case BoolTest::ule:
2543 result = Assembler::LS; break;
2544 case BoolTest::uge:
2545 result = Assembler::HS; break;
2546 case BoolTest::ult:
2547 result = Assembler::LO; break;
2548 case BoolTest::ugt:
2549 result = Assembler::HI; break;
2550 case BoolTest::overflow:
2551 result = Assembler::VS; break;
2552 case BoolTest::no_overflow:
2553 result = Assembler::VC; break;
2554 default:
2555 ShouldNotReachHere();
2556 return Assembler::Condition(-1);
2557 }
2558
2559 // Check conversion
2560 if (cond & BoolTest::unsigned_compare) {
2561 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2562 } else {
2563 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2564 }
2565
2566 return result;
2567 }
2568
2569 // Binary src (Replicate con)
2570 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2571 if (n == nullptr || m == nullptr) {
2572 return false;
2573 }
2574
2575 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2576 return false;
2577 }
2578
2579 Node* imm_node = m->in(1);
2580 if (!imm_node->is_Con()) {
2581 return false;
2582 }
2583
2584 const Type* t = imm_node->bottom_type();
2585 if (!(t->isa_int() || t->isa_long())) {
2586 return false;
2587 }
2588
2589 switch (n->Opcode()) {
2590 case Op_AndV:
2591 case Op_OrV:
2592 case Op_XorV: {
2593 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2594 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2595 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2596 }
2597 case Op_AddVB:
2598 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2599 case Op_AddVS:
2600 case Op_AddVI:
2601 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2602 case Op_AddVL:
2603 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2604 default:
2605 return false;
2606 }
2607 }
2608
2609 // (XorV src (Replicate m1))
2610 // (XorVMask src (MaskAll m1))
2611 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2612 if (n != nullptr && m != nullptr) {
2613 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2614 VectorNode::is_all_ones_vector(m);
2615 }
2616 return false;
2617 }
2618
2619 // Should the matcher clone input 'm' of node 'n'?
2620 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2621 if (is_vshift_con_pattern(n, m) ||
2622 is_vector_bitwise_not_pattern(n, m) ||
2623 is_valid_sve_arith_imm_pattern(n, m) ||
2624 is_encode_and_store_pattern(n, m)) {
2625 mstack.push(m, Visit);
2626 return true;
2627 }
2628 return false;
2629 }
2630
2631 // Should the Matcher clone shifts on addressing modes, expecting them
2632 // to be subsumed into complex addressing expressions or compute them
2633 // into registers?
2634 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2635
2636 // Loads and stores with indirect memory input (e.g., volatile loads and
2637 // stores) do not subsume the input into complex addressing expressions. If
2638 // the addressing expression is input to at least one such load or store, do
2639 // not clone the addressing expression. Query needs_acquiring_load and
2640 // needs_releasing_store as a proxy for indirect memory input, as it is not
2641 // possible to directly query for indirect memory input at this stage.
2642 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2643 Node* n = m->fast_out(i);
2644 if (n->is_Load() && needs_acquiring_load(n)) {
2645 return false;
2646 }
2647 if (n->is_Store() && needs_releasing_store(n)) {
2648 return false;
2649 }
2650 }
2651
2652 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2653 return true;
2654 }
2655
2656 Node *off = m->in(AddPNode::Offset);
2657 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2658 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2659 // Are there other uses besides address expressions?
2660 !is_visited(off)) {
2661 address_visited.set(off->_idx); // Flag as address_visited
2662 mstack.push(off->in(2), Visit);
2663 Node *conv = off->in(1);
2664 if (conv->Opcode() == Op_ConvI2L &&
2665 // Are there other uses besides address expressions?
2666 !is_visited(conv)) {
2667 address_visited.set(conv->_idx); // Flag as address_visited
2668 mstack.push(conv->in(1), Pre_Visit);
2669 } else {
2670 mstack.push(conv, Pre_Visit);
2671 }
2672 address_visited.test_set(m->_idx); // Flag as address_visited
2673 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2674 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2675 return true;
2676 } else if (off->Opcode() == Op_ConvI2L &&
2677 // Are there other uses besides address expressions?
2678 !is_visited(off)) {
2679 address_visited.test_set(m->_idx); // Flag as address_visited
2680 address_visited.set(off->_idx); // Flag as address_visited
2681 mstack.push(off->in(1), Pre_Visit);
2682 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2683 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2684 return true;
2685 }
2686 return false;
2687 }
2688
2689 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2690 { \
2691 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2692 guarantee(DISP == 0, "mode not permitted for volatile"); \
2693 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2694 __ INSN(REG, as_Register(BASE)); \
2695 }
2696
2697
2698 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2699 {
2700 Address::extend scale;
2701
2702 // Hooboy, this is fugly. We need a way to communicate to the
2703 // encoder that the index needs to be sign extended, so we have to
2704 // enumerate all the cases.
2705 switch (opcode) {
2706 case INDINDEXSCALEDI2L:
2707 case INDINDEXSCALEDI2LN:
2708 case INDINDEXI2L:
2709 case INDINDEXI2LN:
2710 scale = Address::sxtw(size);
2711 break;
2712 default:
2713 scale = Address::lsl(size);
2714 }
2715
2716 if (index == -1) {
2717 return Address(base, disp);
2718 } else {
2719 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2720 return Address(base, as_Register(index), scale);
2721 }
2722 }
2723
2724
2725 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2726 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2727 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2728 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2729 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2730
2731 // Used for all non-volatile memory accesses. The use of
2732 // $mem->opcode() to discover whether this pattern uses sign-extended
2733 // offsets is something of a kludge.
2734 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2735 Register reg, int opcode,
2736 Register base, int index, int scale, int disp,
2737 int size_in_memory)
2738 {
2739 Address addr = mem2address(opcode, base, index, scale, disp);
2740 if (addr.getMode() == Address::base_plus_offset) {
2741 /* Fix up any out-of-range offsets. */
2742 assert_different_registers(rscratch1, base);
2743 assert_different_registers(rscratch1, reg);
2744 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2745 }
2746 (masm->*insn)(reg, addr);
2747 }
2748
2749 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2750 FloatRegister reg, int opcode,
2751 Register base, int index, int size, int disp,
2752 int size_in_memory)
2753 {
2754 Address::extend scale;
2755
2756 switch (opcode) {
2757 case INDINDEXSCALEDI2L:
2758 case INDINDEXSCALEDI2LN:
2759 scale = Address::sxtw(size);
2760 break;
2761 default:
2762 scale = Address::lsl(size);
2763 }
2764
2765 if (index == -1) {
2766 // Fix up any out-of-range offsets.
2767 assert_different_registers(rscratch1, base);
2768 Address addr = Address(base, disp);
2769 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2770 (masm->*insn)(reg, addr);
2771 } else {
2772 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2773 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2774 }
2775 }
2776
2777 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2778 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2779 int opcode, Register base, int index, int size, int disp)
2780 {
2781 if (index == -1) {
2782 (masm->*insn)(reg, T, Address(base, disp));
2783 } else {
2784 assert(disp == 0, "unsupported address mode");
2785 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2786 }
2787 }
2788
2789 %}
2790
2791
2792
2793 //----------ENCODING BLOCK-----------------------------------------------------
2794 // This block specifies the encoding classes used by the compiler to
2795 // output byte streams. Encoding classes are parameterized macros
2796 // used by Machine Instruction Nodes in order to generate the bit
2797 // encoding of the instruction. Operands specify their base encoding
2798 // interface with the interface keyword. There are currently
2799 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2800 // COND_INTER. REG_INTER causes an operand to generate a function
2801 // which returns its register number when queried. CONST_INTER causes
2802 // an operand to generate a function which returns the value of the
2803 // constant when queried. MEMORY_INTER causes an operand to generate
2804 // four functions which return the Base Register, the Index Register,
2805 // the Scale Value, and the Offset Value of the operand when queried.
2806 // COND_INTER causes an operand to generate six functions which return
2807 // the encoding code (ie - encoding bits for the instruction)
2808 // associated with each basic boolean condition for a conditional
2809 // instruction.
2810 //
2811 // Instructions specify two basic values for encoding. Again, a
2812 // function is available to check if the constant displacement is an
2813 // oop. They use the ins_encode keyword to specify their encoding
2814 // classes (which must be a sequence of enc_class names, and their
2815 // parameters, specified in the encoding block), and they use the
2816 // opcode keyword to specify, in order, their primary, secondary, and
2817 // tertiary opcode. Only the opcode sections which a particular
2818 // instruction needs for encoding need to be specified.
2819 encode %{
2820 // Build emit functions for each basic byte or larger field in the
2821 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2822 // from C++ code in the enc_class source block. Emit functions will
2823 // live in the main source block for now. In future, we can
2824 // generalize this by adding a syntax that specifies the sizes of
2825 // fields in an order, so that the adlc can build the emit functions
2826 // automagically
2827
2828 // catch all for unimplemented encodings
2829 enc_class enc_unimplemented %{
2830 __ unimplemented("C2 catch all");
2831 %}
2832
2833 // BEGIN Non-volatile memory access
2834
2835 // This encoding class is generated automatically from ad_encode.m4.
2836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2837 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2838 Register dst_reg = as_Register($dst$$reg);
2839 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2840 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2841 %}
2842
2843 // This encoding class is generated automatically from ad_encode.m4.
2844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2845 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2846 Register dst_reg = as_Register($dst$$reg);
2847 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2848 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2849 %}
2850
2851 // This encoding class is generated automatically from ad_encode.m4.
2852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2853 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2854 Register dst_reg = as_Register($dst$$reg);
2855 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2856 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2857 %}
2858
2859 // This encoding class is generated automatically from ad_encode.m4.
2860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2861 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2862 Register dst_reg = as_Register($dst$$reg);
2863 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2864 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2865 %}
2866
2867 // This encoding class is generated automatically from ad_encode.m4.
2868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2869 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2870 Register dst_reg = as_Register($dst$$reg);
2871 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2872 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2873 %}
2874
2875 // This encoding class is generated automatically from ad_encode.m4.
2876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2877 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2878 Register dst_reg = as_Register($dst$$reg);
2879 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2880 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2881 %}
2882
2883 // This encoding class is generated automatically from ad_encode.m4.
2884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2885 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2886 Register dst_reg = as_Register($dst$$reg);
2887 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2888 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2889 %}
2890
2891 // This encoding class is generated automatically from ad_encode.m4.
2892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2893 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2894 Register dst_reg = as_Register($dst$$reg);
2895 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2897 %}
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2926 Register dst_reg = as_Register($dst$$reg);
2927 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2934 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2935 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2942 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2943 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2950 Register src_reg = as_Register($src$$reg);
2951 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2952 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2953 %}
2954
2955 // This encoding class is generated automatically from ad_encode.m4.
2956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2957 enc_class aarch64_enc_strb0(memory1 mem) %{
2958 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2959 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2960 %}
2961
2962 // This encoding class is generated automatically from ad_encode.m4.
2963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2964 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2965 Register src_reg = as_Register($src$$reg);
2966 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2967 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2968 %}
2969
2970 // This encoding class is generated automatically from ad_encode.m4.
2971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2972 enc_class aarch64_enc_strh0(memory2 mem) %{
2973 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2974 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2975 %}
2976
2977 // This encoding class is generated automatically from ad_encode.m4.
2978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2979 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2980 Register src_reg = as_Register($src$$reg);
2981 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
2982 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2983 %}
2984
2985 // This encoding class is generated automatically from ad_encode.m4.
2986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2987 enc_class aarch64_enc_strw0(memory4 mem) %{
2988 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
2989 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2990 %}
2991
2992 // This encoding class is generated automatically from ad_encode.m4.
2993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2994 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2995 Register src_reg = as_Register($src$$reg);
2996 // we sometimes get asked to store the stack pointer into the
2997 // current thread -- we cannot do that directly on AArch64
2998 if (src_reg == r31_sp) {
2999 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3000 __ mov(rscratch2, sp);
3001 src_reg = rscratch2;
3002 }
3003 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3004 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3005 %}
3006
3007 // This encoding class is generated automatically from ad_encode.m4.
3008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3009 enc_class aarch64_enc_str0(memory8 mem) %{
3010 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3011 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3012 %}
3013
3014 // This encoding class is generated automatically from ad_encode.m4.
3015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3016 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3017 FloatRegister src_reg = as_FloatRegister($src$$reg);
3018 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3019 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3020 %}
3021
3022 // This encoding class is generated automatically from ad_encode.m4.
3023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3024 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3025 FloatRegister src_reg = as_FloatRegister($src$$reg);
3026 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3027 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3028 %}
3029
3030 // This encoding class is generated automatically from ad_encode.m4.
3031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3032 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3033 __ membar(Assembler::StoreStore);
3034 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3035 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3036 %}
3037
3038 // END Non-volatile memory access
3039
3040 // Vector loads and stores
3041 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3042 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3043 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3044 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3045 %}
3046
3047 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3048 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3049 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3050 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3051 %}
3052
3053 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3054 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3055 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3056 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3057 %}
3058
3059 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3060 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3061 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3062 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3063 %}
3064
3065 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3066 FloatRegister src_reg = as_FloatRegister($src$$reg);
3067 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3068 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3069 %}
3070
3071 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3072 FloatRegister src_reg = as_FloatRegister($src$$reg);
3073 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3074 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3075 %}
3076
3077 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3078 FloatRegister src_reg = as_FloatRegister($src$$reg);
3079 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3080 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3081 %}
3082
3083 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3084 FloatRegister src_reg = as_FloatRegister($src$$reg);
3085 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3086 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3087 %}
3088
3089 // volatile loads and stores
3090
3091 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3092 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3093 rscratch1, stlrb);
3094 %}
3095
3096 enc_class aarch64_enc_stlrb0(memory mem) %{
3097 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3098 rscratch1, stlrb);
3099 %}
3100
3101 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3102 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3103 rscratch1, stlrh);
3104 %}
3105
3106 enc_class aarch64_enc_stlrh0(memory mem) %{
3107 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3108 rscratch1, stlrh);
3109 %}
3110
3111 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3112 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3113 rscratch1, stlrw);
3114 %}
3115
3116 enc_class aarch64_enc_stlrw0(memory mem) %{
3117 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3118 rscratch1, stlrw);
3119 %}
3120
3121 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3122 Register dst_reg = as_Register($dst$$reg);
3123 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3124 rscratch1, ldarb);
3125 __ sxtbw(dst_reg, dst_reg);
3126 %}
3127
3128 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3129 Register dst_reg = as_Register($dst$$reg);
3130 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3131 rscratch1, ldarb);
3132 __ sxtb(dst_reg, dst_reg);
3133 %}
3134
3135 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3136 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3137 rscratch1, ldarb);
3138 %}
3139
3140 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3141 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3142 rscratch1, ldarb);
3143 %}
3144
3145 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3146 Register dst_reg = as_Register($dst$$reg);
3147 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3148 rscratch1, ldarh);
3149 __ sxthw(dst_reg, dst_reg);
3150 %}
3151
3152 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3153 Register dst_reg = as_Register($dst$$reg);
3154 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3155 rscratch1, ldarh);
3156 __ sxth(dst_reg, dst_reg);
3157 %}
3158
3159 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3160 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3161 rscratch1, ldarh);
3162 %}
3163
3164 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3165 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3166 rscratch1, ldarh);
3167 %}
3168
3169 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3170 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, ldarw);
3172 %}
3173
3174 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3175 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3176 rscratch1, ldarw);
3177 %}
3178
3179 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3180 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3181 rscratch1, ldar);
3182 %}
3183
3184 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3185 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3186 rscratch1, ldarw);
3187 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3188 %}
3189
3190 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3191 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3192 rscratch1, ldar);
3193 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3194 %}
3195
3196 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3197 Register src_reg = as_Register($src$$reg);
3198 // we sometimes get asked to store the stack pointer into the
3199 // current thread -- we cannot do that directly on AArch64
3200 if (src_reg == r31_sp) {
3201 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3202 __ mov(rscratch2, sp);
3203 src_reg = rscratch2;
3204 }
3205 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3206 rscratch1, stlr);
3207 %}
3208
3209 enc_class aarch64_enc_stlr0(memory mem) %{
3210 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3211 rscratch1, stlr);
3212 %}
3213
3214 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3215 {
3216 FloatRegister src_reg = as_FloatRegister($src$$reg);
3217 __ fmovs(rscratch2, src_reg);
3218 }
3219 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3220 rscratch1, stlrw);
3221 %}
3222
3223 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3224 {
3225 FloatRegister src_reg = as_FloatRegister($src$$reg);
3226 __ fmovd(rscratch2, src_reg);
3227 }
3228 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3229 rscratch1, stlr);
3230 %}
3231
3232 // synchronized read/update encodings
3233
3234 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3235 Register dst_reg = as_Register($dst$$reg);
3236 Register base = as_Register($mem$$base);
3237 int index = $mem$$index;
3238 int scale = $mem$$scale;
3239 int disp = $mem$$disp;
3240 if (index == -1) {
3241 if (disp != 0) {
3242 __ lea(rscratch1, Address(base, disp));
3243 __ ldaxr(dst_reg, rscratch1);
3244 } else {
3245 // TODO
3246 // should we ever get anything other than this case?
3247 __ ldaxr(dst_reg, base);
3248 }
3249 } else {
3250 Register index_reg = as_Register(index);
3251 if (disp == 0) {
3252 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3253 __ ldaxr(dst_reg, rscratch1);
3254 } else {
3255 __ lea(rscratch1, Address(base, disp));
3256 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3257 __ ldaxr(dst_reg, rscratch1);
3258 }
3259 }
3260 %}
3261
3262 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3263 Register src_reg = as_Register($src$$reg);
3264 Register base = as_Register($mem$$base);
3265 int index = $mem$$index;
3266 int scale = $mem$$scale;
3267 int disp = $mem$$disp;
3268 if (index == -1) {
3269 if (disp != 0) {
3270 __ lea(rscratch2, Address(base, disp));
3271 __ stlxr(rscratch1, src_reg, rscratch2);
3272 } else {
3273 // TODO
3274 // should we ever get anything other than this case?
3275 __ stlxr(rscratch1, src_reg, base);
3276 }
3277 } else {
3278 Register index_reg = as_Register(index);
3279 if (disp == 0) {
3280 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3281 __ stlxr(rscratch1, src_reg, rscratch2);
3282 } else {
3283 __ lea(rscratch2, Address(base, disp));
3284 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3285 __ stlxr(rscratch1, src_reg, rscratch2);
3286 }
3287 }
3288 __ cmpw(rscratch1, zr);
3289 %}
3290
3291 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3292 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3293 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3294 Assembler::xword, /*acquire*/ false, /*release*/ true,
3295 /*weak*/ false, noreg);
3296 %}
3297
3298 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3299 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3300 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3301 Assembler::word, /*acquire*/ false, /*release*/ true,
3302 /*weak*/ false, noreg);
3303 %}
3304
3305 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3306 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3307 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3308 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3309 /*weak*/ false, noreg);
3310 %}
3311
3312 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3313 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3314 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3315 Assembler::byte, /*acquire*/ false, /*release*/ true,
3316 /*weak*/ false, noreg);
3317 %}
3318
3319
3320 // The only difference between aarch64_enc_cmpxchg and
3321 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3322 // CompareAndSwap sequence to serve as a barrier on acquiring a
3323 // lock.
3324 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3325 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3326 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3327 Assembler::xword, /*acquire*/ true, /*release*/ true,
3328 /*weak*/ false, noreg);
3329 %}
3330
3331 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3332 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3333 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3334 Assembler::word, /*acquire*/ true, /*release*/ true,
3335 /*weak*/ false, noreg);
3336 %}
3337
3338 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3339 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3340 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3341 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3342 /*weak*/ false, noreg);
3343 %}
3344
3345 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3346 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3347 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3348 Assembler::byte, /*acquire*/ true, /*release*/ true,
3349 /*weak*/ false, noreg);
3350 %}
3351
3352 // auxiliary used for CompareAndSwapX to set result register
3353 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3354 Register res_reg = as_Register($res$$reg);
3355 __ cset(res_reg, Assembler::EQ);
3356 %}
3357
3358 // prefetch encodings
3359
3360 enc_class aarch64_enc_prefetchw(memory mem) %{
3361 Register base = as_Register($mem$$base);
3362 int index = $mem$$index;
3363 int scale = $mem$$scale;
3364 int disp = $mem$$disp;
3365 if (index == -1) {
3366 // Fix up any out-of-range offsets.
3367 assert_different_registers(rscratch1, base);
3368 Address addr = Address(base, disp);
3369 addr = __ legitimize_address(addr, 8, rscratch1);
3370 __ prfm(addr, PSTL1KEEP);
3371 } else {
3372 Register index_reg = as_Register(index);
3373 if (disp == 0) {
3374 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3375 } else {
3376 __ lea(rscratch1, Address(base, disp));
3377 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3378 }
3379 }
3380 %}
3381
3382 // mov encodings
3383
3384 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3385 uint32_t con = (uint32_t)$src$$constant;
3386 Register dst_reg = as_Register($dst$$reg);
3387 if (con == 0) {
3388 __ movw(dst_reg, zr);
3389 } else {
3390 __ movw(dst_reg, con);
3391 }
3392 %}
3393
3394 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3395 Register dst_reg = as_Register($dst$$reg);
3396 uint64_t con = (uint64_t)$src$$constant;
3397 if (con == 0) {
3398 __ mov(dst_reg, zr);
3399 } else {
3400 __ mov(dst_reg, con);
3401 }
3402 %}
3403
3404 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3405 Register dst_reg = as_Register($dst$$reg);
3406 address con = (address)$src$$constant;
3407 if (con == nullptr || con == (address)1) {
3408 ShouldNotReachHere();
3409 } else {
3410 relocInfo::relocType rtype = $src->constant_reloc();
3411 if (rtype == relocInfo::oop_type) {
3412 __ movoop(dst_reg, (jobject)con);
3413 } else if (rtype == relocInfo::metadata_type) {
3414 __ mov_metadata(dst_reg, (Metadata*)con);
3415 } else {
3416 assert(rtype == relocInfo::none, "unexpected reloc type");
3417 if (! __ is_valid_AArch64_address(con) ||
3418 con < (address)(uintptr_t)os::vm_page_size()) {
3419 __ mov(dst_reg, con);
3420 } else {
3421 uint64_t offset;
3422 __ adrp(dst_reg, con, offset);
3423 __ add(dst_reg, dst_reg, offset);
3424 }
3425 }
3426 }
3427 %}
3428
3429 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3430 Register dst_reg = as_Register($dst$$reg);
3431 __ mov(dst_reg, zr);
3432 %}
3433
3434 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3435 Register dst_reg = as_Register($dst$$reg);
3436 __ mov(dst_reg, (uint64_t)1);
3437 %}
3438
3439 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3440 __ load_byte_map_base($dst$$Register);
3441 %}
3442
3443 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3444 Register dst_reg = as_Register($dst$$reg);
3445 address con = (address)$src$$constant;
3446 if (con == nullptr) {
3447 ShouldNotReachHere();
3448 } else {
3449 relocInfo::relocType rtype = $src->constant_reloc();
3450 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3451 __ set_narrow_oop(dst_reg, (jobject)con);
3452 }
3453 %}
3454
3455 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3456 Register dst_reg = as_Register($dst$$reg);
3457 __ mov(dst_reg, zr);
3458 %}
3459
3460 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3461 Register dst_reg = as_Register($dst$$reg);
3462 address con = (address)$src$$constant;
3463 if (con == nullptr) {
3464 ShouldNotReachHere();
3465 } else {
3466 relocInfo::relocType rtype = $src->constant_reloc();
3467 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3468 __ set_narrow_klass(dst_reg, (Klass *)con);
3469 }
3470 %}
3471
3472 // arithmetic encodings
3473
3474 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3475 Register dst_reg = as_Register($dst$$reg);
3476 Register src_reg = as_Register($src1$$reg);
3477 int32_t con = (int32_t)$src2$$constant;
3478 // add has primary == 0, subtract has primary == 1
3479 if ($primary) { con = -con; }
3480 if (con < 0) {
3481 __ subw(dst_reg, src_reg, -con);
3482 } else {
3483 __ addw(dst_reg, src_reg, con);
3484 }
3485 %}
3486
3487 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3488 Register dst_reg = as_Register($dst$$reg);
3489 Register src_reg = as_Register($src1$$reg);
3490 int32_t con = (int32_t)$src2$$constant;
3491 // add has primary == 0, subtract has primary == 1
3492 if ($primary) { con = -con; }
3493 if (con < 0) {
3494 __ sub(dst_reg, src_reg, -con);
3495 } else {
3496 __ add(dst_reg, src_reg, con);
3497 }
3498 %}
3499
3500 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3501 Register dst_reg = as_Register($dst$$reg);
3502 Register src1_reg = as_Register($src1$$reg);
3503 Register src2_reg = as_Register($src2$$reg);
3504 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3505 %}
3506
3507 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3508 Register dst_reg = as_Register($dst$$reg);
3509 Register src1_reg = as_Register($src1$$reg);
3510 Register src2_reg = as_Register($src2$$reg);
3511 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3512 %}
3513
3514 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3515 Register dst_reg = as_Register($dst$$reg);
3516 Register src1_reg = as_Register($src1$$reg);
3517 Register src2_reg = as_Register($src2$$reg);
3518 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3519 %}
3520
3521 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3522 Register dst_reg = as_Register($dst$$reg);
3523 Register src1_reg = as_Register($src1$$reg);
3524 Register src2_reg = as_Register($src2$$reg);
3525 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3526 %}
3527
3528 // compare instruction encodings
3529
3530 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3531 Register reg1 = as_Register($src1$$reg);
3532 Register reg2 = as_Register($src2$$reg);
3533 __ cmpw(reg1, reg2);
3534 %}
3535
3536 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3537 Register reg = as_Register($src1$$reg);
3538 int32_t val = $src2$$constant;
3539 if (val >= 0) {
3540 __ subsw(zr, reg, val);
3541 } else {
3542 __ addsw(zr, reg, -val);
3543 }
3544 %}
3545
3546 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3547 Register reg1 = as_Register($src1$$reg);
3548 uint32_t val = (uint32_t)$src2$$constant;
3549 __ movw(rscratch1, val);
3550 __ cmpw(reg1, rscratch1);
3551 %}
3552
3553 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3554 Register reg1 = as_Register($src1$$reg);
3555 Register reg2 = as_Register($src2$$reg);
3556 __ cmp(reg1, reg2);
3557 %}
3558
3559 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3560 Register reg = as_Register($src1$$reg);
3561 int64_t val = $src2$$constant;
3562 if (val >= 0) {
3563 __ subs(zr, reg, val);
3564 } else if (val != -val) {
3565 __ adds(zr, reg, -val);
3566 } else {
3567 // aargh, Long.MIN_VALUE is a special case
3568 __ orr(rscratch1, zr, (uint64_t)val);
3569 __ subs(zr, reg, rscratch1);
3570 }
3571 %}
3572
3573 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3574 Register reg1 = as_Register($src1$$reg);
3575 uint64_t val = (uint64_t)$src2$$constant;
3576 __ mov(rscratch1, val);
3577 __ cmp(reg1, rscratch1);
3578 %}
3579
3580 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3581 Register reg1 = as_Register($src1$$reg);
3582 Register reg2 = as_Register($src2$$reg);
3583 __ cmp(reg1, reg2);
3584 %}
3585
3586 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3587 Register reg1 = as_Register($src1$$reg);
3588 Register reg2 = as_Register($src2$$reg);
3589 __ cmpw(reg1, reg2);
3590 %}
3591
3592 enc_class aarch64_enc_testp(iRegP src) %{
3593 Register reg = as_Register($src$$reg);
3594 __ cmp(reg, zr);
3595 %}
3596
3597 enc_class aarch64_enc_testn(iRegN src) %{
3598 Register reg = as_Register($src$$reg);
3599 __ cmpw(reg, zr);
3600 %}
3601
3602 enc_class aarch64_enc_b(label lbl) %{
3603 Label *L = $lbl$$label;
3604 __ b(*L);
3605 %}
3606
3607 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3608 Label *L = $lbl$$label;
3609 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3610 %}
3611
3612 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3613 Label *L = $lbl$$label;
3614 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3615 %}
3616
3617 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3618 %{
3619 Register sub_reg = as_Register($sub$$reg);
3620 Register super_reg = as_Register($super$$reg);
3621 Register temp_reg = as_Register($temp$$reg);
3622 Register result_reg = as_Register($result$$reg);
3623
3624 Label miss;
3625 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3626 nullptr, &miss,
3627 /*set_cond_codes:*/ true);
3628 if ($primary) {
3629 __ mov(result_reg, zr);
3630 }
3631 __ bind(miss);
3632 %}
3633
3634 enc_class aarch64_enc_java_static_call(method meth) %{
3635 address addr = (address)$meth$$method;
3636 address call;
3637 if (!_method) {
3638 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3639 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3640 if (call == nullptr) {
3641 ciEnv::current()->record_failure("CodeCache is full");
3642 return;
3643 }
3644 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3645 // The NOP here is purely to ensure that eliding a call to
3646 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3647 __ nop();
3648 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3649 } else {
3650 int method_index = resolved_method_index(masm);
3651 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3652 : static_call_Relocation::spec(method_index);
3653 call = __ trampoline_call(Address(addr, rspec));
3654 if (call == nullptr) {
3655 ciEnv::current()->record_failure("CodeCache is full");
3656 return;
3657 }
3658 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3659 // Calls of the same statically bound method can share
3660 // a stub to the interpreter.
3661 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3662 } else {
3663 // Emit stub for static call
3664 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3665 if (stub == nullptr) {
3666 ciEnv::current()->record_failure("CodeCache is full");
3667 return;
3668 }
3669 }
3670 }
3671
3672 __ post_call_nop();
3673
3674 // Only non uncommon_trap calls need to reinitialize ptrue.
3675 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3676 __ reinitialize_ptrue();
3677 }
3678 %}
3679
3680 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3681 int method_index = resolved_method_index(masm);
3682 address call = __ ic_call((address)$meth$$method, method_index);
3683 if (call == nullptr) {
3684 ciEnv::current()->record_failure("CodeCache is full");
3685 return;
3686 }
3687 __ post_call_nop();
3688 if (Compile::current()->max_vector_size() > 0) {
3689 __ reinitialize_ptrue();
3690 }
3691 %}
3692
3693 enc_class aarch64_enc_call_epilog() %{
3694 if (VerifyStackAtCalls) {
3695 // Check that stack depth is unchanged: find majik cookie on stack
3696 __ call_Unimplemented();
3697 }
3698 %}
3699
3700 enc_class aarch64_enc_java_to_runtime(method meth) %{
3701 // some calls to generated routines (arraycopy code) are scheduled
3702 // by C2 as runtime calls. if so we can call them using a br (they
3703 // will be in a reachable segment) otherwise we have to use a blr
3704 // which loads the absolute address into a register.
3705 address entry = (address)$meth$$method;
3706 CodeBlob *cb = CodeCache::find_blob(entry);
3707 if (cb) {
3708 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3709 if (call == nullptr) {
3710 ciEnv::current()->record_failure("CodeCache is full");
3711 return;
3712 }
3713 __ post_call_nop();
3714 } else {
3715 Label retaddr;
3716 // Make the anchor frame walkable
3717 __ adr(rscratch2, retaddr);
3718 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3719 __ lea(rscratch1, RuntimeAddress(entry));
3720 __ blr(rscratch1);
3721 __ bind(retaddr);
3722 __ post_call_nop();
3723 }
3724 if (Compile::current()->max_vector_size() > 0) {
3725 __ reinitialize_ptrue();
3726 }
3727 %}
3728
3729 enc_class aarch64_enc_rethrow() %{
3730 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3731 %}
3732
3733 enc_class aarch64_enc_ret() %{
3734 #ifdef ASSERT
3735 if (Compile::current()->max_vector_size() > 0) {
3736 __ verify_ptrue();
3737 }
3738 #endif
3739 __ ret(lr);
3740 %}
3741
3742 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3743 Register target_reg = as_Register($jump_target$$reg);
3744 __ br(target_reg);
3745 %}
3746
3747 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3748 Register target_reg = as_Register($jump_target$$reg);
3749 // exception oop should be in r0
3750 // ret addr has been popped into lr
3751 // callee expects it in r3
3752 __ mov(r3, lr);
3753 __ br(target_reg);
3754 %}
3755
3756 %}
3757
3758 //----------FRAME--------------------------------------------------------------
3759 // Definition of frame structure and management information.
3760 //
3761 // S T A C K L A Y O U T Allocators stack-slot number
3762 // | (to get allocators register number
3763 // G Owned by | | v add OptoReg::stack0())
3764 // r CALLER | |
3765 // o | +--------+ pad to even-align allocators stack-slot
3766 // w V | pad0 | numbers; owned by CALLER
3767 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3768 // h ^ | in | 5
3769 // | | args | 4 Holes in incoming args owned by SELF
3770 // | | | | 3
3771 // | | +--------+
3772 // V | | old out| Empty on Intel, window on Sparc
3773 // | old |preserve| Must be even aligned.
3774 // | SP-+--------+----> Matcher::_old_SP, even aligned
3775 // | | in | 3 area for Intel ret address
3776 // Owned by |preserve| Empty on Sparc.
3777 // SELF +--------+
3778 // | | pad2 | 2 pad to align old SP
3779 // | +--------+ 1
3780 // | | locks | 0
3781 // | +--------+----> OptoReg::stack0(), even aligned
3782 // | | pad1 | 11 pad to align new SP
3783 // | +--------+
3784 // | | | 10
3785 // | | spills | 9 spills
3786 // V | | 8 (pad0 slot for callee)
3787 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3788 // ^ | out | 7
3789 // | | args | 6 Holes in outgoing args owned by CALLEE
3790 // Owned by +--------+
3791 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3792 // | new |preserve| Must be even-aligned.
3793 // | SP-+--------+----> Matcher::_new_SP, even aligned
3794 // | | |
3795 //
3796 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3797 // known from SELF's arguments and the Java calling convention.
3798 // Region 6-7 is determined per call site.
3799 // Note 2: If the calling convention leaves holes in the incoming argument
3800 // area, those holes are owned by SELF. Holes in the outgoing area
3801 // are owned by the CALLEE. Holes should not be necessary in the
3802 // incoming area, as the Java calling convention is completely under
3803 // the control of the AD file. Doubles can be sorted and packed to
3804 // avoid holes. Holes in the outgoing arguments may be necessary for
3805 // varargs C calling conventions.
3806 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3807 // even aligned with pad0 as needed.
3808 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3809 // (the latter is true on Intel but is it false on AArch64?)
3810 // region 6-11 is even aligned; it may be padded out more so that
3811 // the region from SP to FP meets the minimum stack alignment.
3812 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3813 // alignment. Region 11, pad1, may be dynamically extended so that
3814 // SP meets the minimum alignment.
3815
3816 frame %{
3817 // These three registers define part of the calling convention
3818 // between compiled code and the interpreter.
3819
3820 // Inline Cache Register or Method for I2C.
3821 inline_cache_reg(R12);
3822
3823 // Number of stack slots consumed by locking an object
3824 sync_stack_slots(2);
3825
3826 // Compiled code's Frame Pointer
3827 frame_pointer(R31);
3828
3829 // Interpreter stores its frame pointer in a register which is
3830 // stored to the stack by I2CAdaptors.
3831 // I2CAdaptors convert from interpreted java to compiled java.
3832 interpreter_frame_pointer(R29);
3833
3834 // Stack alignment requirement
3835 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3836
3837 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3838 // for calls to C. Supports the var-args backing area for register parms.
3839 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3840
3841 // The after-PROLOG location of the return address. Location of
3842 // return address specifies a type (REG or STACK) and a number
3843 // representing the register number (i.e. - use a register name) or
3844 // stack slot.
3845 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3846 // Otherwise, it is above the locks and verification slot and alignment word
3847 // TODO this may well be correct but need to check why that - 2 is there
3848 // ppc port uses 0 but we definitely need to allow for fixed_slots
3849 // which folds in the space used for monitors
3850 return_addr(STACK - 2 +
3851 align_up((Compile::current()->in_preserve_stack_slots() +
3852 Compile::current()->fixed_slots()),
3853 stack_alignment_in_slots()));
3854
3855 // Location of compiled Java return values. Same as C for now.
3856 return_value
3857 %{
3858 // TODO do we allow ideal_reg == Op_RegN???
3859 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3860 "only return normal values");
3861
3862 static const int lo[Op_RegL + 1] = { // enum name
3863 0, // Op_Node
3864 0, // Op_Set
3865 R0_num, // Op_RegN
3866 R0_num, // Op_RegI
3867 R0_num, // Op_RegP
3868 V0_num, // Op_RegF
3869 V0_num, // Op_RegD
3870 R0_num // Op_RegL
3871 };
3872
3873 static const int hi[Op_RegL + 1] = { // enum name
3874 0, // Op_Node
3875 0, // Op_Set
3876 OptoReg::Bad, // Op_RegN
3877 OptoReg::Bad, // Op_RegI
3878 R0_H_num, // Op_RegP
3879 OptoReg::Bad, // Op_RegF
3880 V0_H_num, // Op_RegD
3881 R0_H_num // Op_RegL
3882 };
3883
3884 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3885 %}
3886 %}
3887
3888 //----------ATTRIBUTES---------------------------------------------------------
3889 //----------Operand Attributes-------------------------------------------------
3890 op_attrib op_cost(1); // Required cost attribute
3891
3892 //----------Instruction Attributes---------------------------------------------
3893 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3894 ins_attrib ins_size(32); // Required size attribute (in bits)
3895 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3896 // a non-matching short branch variant
3897 // of some long branch?
3898 ins_attrib ins_alignment(4); // Required alignment attribute (must
3899 // be a power of 2) specifies the
3900 // alignment that some part of the
3901 // instruction (not necessarily the
3902 // start) requires. If > 1, a
3903 // compute_padding() function must be
3904 // provided for the instruction
3905
3906 //----------OPERANDS-----------------------------------------------------------
3907 // Operand definitions must precede instruction definitions for correct parsing
3908 // in the ADLC because operands constitute user defined types which are used in
3909 // instruction definitions.
3910
3911 //----------Simple Operands----------------------------------------------------
3912
3913 // Integer operands 32 bit
3914 // 32 bit immediate
3915 operand immI()
3916 %{
3917 match(ConI);
3918
3919 op_cost(0);
3920 format %{ %}
3921 interface(CONST_INTER);
3922 %}
3923
3924 // 32 bit zero
3925 operand immI0()
3926 %{
3927 predicate(n->get_int() == 0);
3928 match(ConI);
3929
3930 op_cost(0);
3931 format %{ %}
3932 interface(CONST_INTER);
3933 %}
3934
3935 // 32 bit unit increment
3936 operand immI_1()
3937 %{
3938 predicate(n->get_int() == 1);
3939 match(ConI);
3940
3941 op_cost(0);
3942 format %{ %}
3943 interface(CONST_INTER);
3944 %}
3945
3946 // 32 bit unit decrement
3947 operand immI_M1()
3948 %{
3949 predicate(n->get_int() == -1);
3950 match(ConI);
3951
3952 op_cost(0);
3953 format %{ %}
3954 interface(CONST_INTER);
3955 %}
3956
3957 // Shift values for add/sub extension shift
3958 operand immIExt()
3959 %{
3960 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3961 match(ConI);
3962
3963 op_cost(0);
3964 format %{ %}
3965 interface(CONST_INTER);
3966 %}
3967
3968 operand immI_gt_1()
3969 %{
3970 predicate(n->get_int() > 1);
3971 match(ConI);
3972
3973 op_cost(0);
3974 format %{ %}
3975 interface(CONST_INTER);
3976 %}
3977
3978 operand immI_le_4()
3979 %{
3980 predicate(n->get_int() <= 4);
3981 match(ConI);
3982
3983 op_cost(0);
3984 format %{ %}
3985 interface(CONST_INTER);
3986 %}
3987
3988 operand immI_16()
3989 %{
3990 predicate(n->get_int() == 16);
3991 match(ConI);
3992
3993 op_cost(0);
3994 format %{ %}
3995 interface(CONST_INTER);
3996 %}
3997
3998 operand immI_24()
3999 %{
4000 predicate(n->get_int() == 24);
4001 match(ConI);
4002
4003 op_cost(0);
4004 format %{ %}
4005 interface(CONST_INTER);
4006 %}
4007
4008 operand immI_32()
4009 %{
4010 predicate(n->get_int() == 32);
4011 match(ConI);
4012
4013 op_cost(0);
4014 format %{ %}
4015 interface(CONST_INTER);
4016 %}
4017
4018 operand immI_48()
4019 %{
4020 predicate(n->get_int() == 48);
4021 match(ConI);
4022
4023 op_cost(0);
4024 format %{ %}
4025 interface(CONST_INTER);
4026 %}
4027
4028 operand immI_56()
4029 %{
4030 predicate(n->get_int() == 56);
4031 match(ConI);
4032
4033 op_cost(0);
4034 format %{ %}
4035 interface(CONST_INTER);
4036 %}
4037
4038 operand immI_255()
4039 %{
4040 predicate(n->get_int() == 255);
4041 match(ConI);
4042
4043 op_cost(0);
4044 format %{ %}
4045 interface(CONST_INTER);
4046 %}
4047
4048 operand immI_65535()
4049 %{
4050 predicate(n->get_int() == 65535);
4051 match(ConI);
4052
4053 op_cost(0);
4054 format %{ %}
4055 interface(CONST_INTER);
4056 %}
4057
4058 operand immI_positive()
4059 %{
4060 predicate(n->get_int() > 0);
4061 match(ConI);
4062
4063 op_cost(0);
4064 format %{ %}
4065 interface(CONST_INTER);
4066 %}
4067
4068 // BoolTest condition for signed compare
4069 operand immI_cmp_cond()
4070 %{
4071 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4072 match(ConI);
4073
4074 op_cost(0);
4075 format %{ %}
4076 interface(CONST_INTER);
4077 %}
4078
4079 // BoolTest condition for unsigned compare
4080 operand immI_cmpU_cond()
4081 %{
4082 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4083 match(ConI);
4084
4085 op_cost(0);
4086 format %{ %}
4087 interface(CONST_INTER);
4088 %}
4089
4090 operand immL_255()
4091 %{
4092 predicate(n->get_long() == 255L);
4093 match(ConL);
4094
4095 op_cost(0);
4096 format %{ %}
4097 interface(CONST_INTER);
4098 %}
4099
4100 operand immL_65535()
4101 %{
4102 predicate(n->get_long() == 65535L);
4103 match(ConL);
4104
4105 op_cost(0);
4106 format %{ %}
4107 interface(CONST_INTER);
4108 %}
4109
4110 operand immL_4294967295()
4111 %{
4112 predicate(n->get_long() == 4294967295L);
4113 match(ConL);
4114
4115 op_cost(0);
4116 format %{ %}
4117 interface(CONST_INTER);
4118 %}
4119
4120 operand immL_bitmask()
4121 %{
4122 predicate((n->get_long() != 0)
4123 && ((n->get_long() & 0xc000000000000000l) == 0)
4124 && is_power_of_2(n->get_long() + 1));
4125 match(ConL);
4126
4127 op_cost(0);
4128 format %{ %}
4129 interface(CONST_INTER);
4130 %}
4131
4132 operand immI_bitmask()
4133 %{
4134 predicate((n->get_int() != 0)
4135 && ((n->get_int() & 0xc0000000) == 0)
4136 && is_power_of_2(n->get_int() + 1));
4137 match(ConI);
4138
4139 op_cost(0);
4140 format %{ %}
4141 interface(CONST_INTER);
4142 %}
4143
4144 operand immL_positive_bitmaskI()
4145 %{
4146 predicate((n->get_long() != 0)
4147 && ((julong)n->get_long() < 0x80000000ULL)
4148 && is_power_of_2(n->get_long() + 1));
4149 match(ConL);
4150
4151 op_cost(0);
4152 format %{ %}
4153 interface(CONST_INTER);
4154 %}
4155
4156 // Scale values for scaled offset addressing modes (up to long but not quad)
4157 operand immIScale()
4158 %{
4159 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4160 match(ConI);
4161
4162 op_cost(0);
4163 format %{ %}
4164 interface(CONST_INTER);
4165 %}
4166
4167 // 5 bit signed integer
4168 operand immI5()
4169 %{
4170 predicate(Assembler::is_simm(n->get_int(), 5));
4171 match(ConI);
4172
4173 op_cost(0);
4174 format %{ %}
4175 interface(CONST_INTER);
4176 %}
4177
4178 // 7 bit unsigned integer
4179 operand immIU7()
4180 %{
4181 predicate(Assembler::is_uimm(n->get_int(), 7));
4182 match(ConI);
4183
4184 op_cost(0);
4185 format %{ %}
4186 interface(CONST_INTER);
4187 %}
4188
4189 // Offset for scaled or unscaled immediate loads and stores
4190 operand immIOffset()
4191 %{
4192 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4193 match(ConI);
4194
4195 op_cost(0);
4196 format %{ %}
4197 interface(CONST_INTER);
4198 %}
4199
4200 operand immIOffset1()
4201 %{
4202 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4203 match(ConI);
4204
4205 op_cost(0);
4206 format %{ %}
4207 interface(CONST_INTER);
4208 %}
4209
4210 operand immIOffset2()
4211 %{
4212 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4213 match(ConI);
4214
4215 op_cost(0);
4216 format %{ %}
4217 interface(CONST_INTER);
4218 %}
4219
4220 operand immIOffset4()
4221 %{
4222 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4223 match(ConI);
4224
4225 op_cost(0);
4226 format %{ %}
4227 interface(CONST_INTER);
4228 %}
4229
4230 operand immIOffset8()
4231 %{
4232 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4233 match(ConI);
4234
4235 op_cost(0);
4236 format %{ %}
4237 interface(CONST_INTER);
4238 %}
4239
4240 operand immIOffset16()
4241 %{
4242 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4243 match(ConI);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immLOffset()
4251 %{
4252 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4253 match(ConL);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immLoffset1()
4261 %{
4262 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4263 match(ConL);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immLoffset2()
4271 %{
4272 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4273 match(ConL);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 operand immLoffset4()
4281 %{
4282 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4283 match(ConL);
4284
4285 op_cost(0);
4286 format %{ %}
4287 interface(CONST_INTER);
4288 %}
4289
4290 operand immLoffset8()
4291 %{
4292 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4293 match(ConL);
4294
4295 op_cost(0);
4296 format %{ %}
4297 interface(CONST_INTER);
4298 %}
4299
4300 operand immLoffset16()
4301 %{
4302 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4303 match(ConL);
4304
4305 op_cost(0);
4306 format %{ %}
4307 interface(CONST_INTER);
4308 %}
4309
4310 // 5 bit signed long integer
4311 operand immL5()
4312 %{
4313 predicate(Assembler::is_simm(n->get_long(), 5));
4314 match(ConL);
4315
4316 op_cost(0);
4317 format %{ %}
4318 interface(CONST_INTER);
4319 %}
4320
4321 // 7 bit unsigned long integer
4322 operand immLU7()
4323 %{
4324 predicate(Assembler::is_uimm(n->get_long(), 7));
4325 match(ConL);
4326
4327 op_cost(0);
4328 format %{ %}
4329 interface(CONST_INTER);
4330 %}
4331
4332 // 8 bit signed value.
4333 operand immI8()
4334 %{
4335 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4336 match(ConI);
4337
4338 op_cost(0);
4339 format %{ %}
4340 interface(CONST_INTER);
4341 %}
4342
4343 // 8 bit signed value (simm8), or #simm8 LSL 8.
4344 operand immI8_shift8()
4345 %{
4346 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4347 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4348 match(ConI);
4349
4350 op_cost(0);
4351 format %{ %}
4352 interface(CONST_INTER);
4353 %}
4354
4355 // 8 bit signed value (simm8), or #simm8 LSL 8.
4356 operand immL8_shift8()
4357 %{
4358 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4359 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4360 match(ConL);
4361
4362 op_cost(0);
4363 format %{ %}
4364 interface(CONST_INTER);
4365 %}
4366
4367 // 8 bit integer valid for vector add sub immediate
4368 operand immBAddSubV()
4369 %{
4370 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4371 match(ConI);
4372
4373 op_cost(0);
4374 format %{ %}
4375 interface(CONST_INTER);
4376 %}
4377
4378 // 32 bit integer valid for add sub immediate
4379 operand immIAddSub()
4380 %{
4381 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4382 match(ConI);
4383 op_cost(0);
4384 format %{ %}
4385 interface(CONST_INTER);
4386 %}
4387
4388 // 32 bit integer valid for vector add sub immediate
4389 operand immIAddSubV()
4390 %{
4391 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4392 match(ConI);
4393
4394 op_cost(0);
4395 format %{ %}
4396 interface(CONST_INTER);
4397 %}
4398
4399 // 32 bit unsigned integer valid for logical immediate
4400
4401 operand immBLog()
4402 %{
4403 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4404 match(ConI);
4405
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 operand immSLog()
4412 %{
4413 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4414 match(ConI);
4415
4416 op_cost(0);
4417 format %{ %}
4418 interface(CONST_INTER);
4419 %}
4420
4421 operand immILog()
4422 %{
4423 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4424 match(ConI);
4425
4426 op_cost(0);
4427 format %{ %}
4428 interface(CONST_INTER);
4429 %}
4430
4431 // Integer operands 64 bit
4432 // 64 bit immediate
4433 operand immL()
4434 %{
4435 match(ConL);
4436
4437 op_cost(0);
4438 format %{ %}
4439 interface(CONST_INTER);
4440 %}
4441
4442 // 64 bit zero
4443 operand immL0()
4444 %{
4445 predicate(n->get_long() == 0);
4446 match(ConL);
4447
4448 op_cost(0);
4449 format %{ %}
4450 interface(CONST_INTER);
4451 %}
4452
4453 // 64 bit unit decrement
4454 operand immL_M1()
4455 %{
4456 predicate(n->get_long() == -1);
4457 match(ConL);
4458
4459 op_cost(0);
4460 format %{ %}
4461 interface(CONST_INTER);
4462 %}
4463
4464 // 64 bit integer valid for add sub immediate
4465 operand immLAddSub()
4466 %{
4467 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4468 match(ConL);
4469 op_cost(0);
4470 format %{ %}
4471 interface(CONST_INTER);
4472 %}
4473
4474 // 64 bit integer valid for addv subv immediate
4475 operand immLAddSubV()
4476 %{
4477 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4478 match(ConL);
4479
4480 op_cost(0);
4481 format %{ %}
4482 interface(CONST_INTER);
4483 %}
4484
4485 // 64 bit integer valid for logical immediate
4486 operand immLLog()
4487 %{
4488 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4489 match(ConL);
4490 op_cost(0);
4491 format %{ %}
4492 interface(CONST_INTER);
4493 %}
4494
4495 // Long Immediate: low 32-bit mask
4496 operand immL_32bits()
4497 %{
4498 predicate(n->get_long() == 0xFFFFFFFFL);
4499 match(ConL);
4500 op_cost(0);
4501 format %{ %}
4502 interface(CONST_INTER);
4503 %}
4504
4505 // Pointer operands
4506 // Pointer Immediate
4507 operand immP()
4508 %{
4509 match(ConP);
4510
4511 op_cost(0);
4512 format %{ %}
4513 interface(CONST_INTER);
4514 %}
4515
4516 // nullptr Pointer Immediate
4517 operand immP0()
4518 %{
4519 predicate(n->get_ptr() == 0);
4520 match(ConP);
4521
4522 op_cost(0);
4523 format %{ %}
4524 interface(CONST_INTER);
4525 %}
4526
4527 // Pointer Immediate One
4528 // this is used in object initialization (initial object header)
4529 operand immP_1()
4530 %{
4531 predicate(n->get_ptr() == 1);
4532 match(ConP);
4533
4534 op_cost(0);
4535 format %{ %}
4536 interface(CONST_INTER);
4537 %}
4538
4539 // Card Table Byte Map Base
4540 operand immByteMapBase()
4541 %{
4542 // Get base of card map
4543 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4544 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4545 match(ConP);
4546
4547 op_cost(0);
4548 format %{ %}
4549 interface(CONST_INTER);
4550 %}
4551
4552 // Float and Double operands
4553 // Double Immediate
4554 operand immD()
4555 %{
4556 match(ConD);
4557 op_cost(0);
4558 format %{ %}
4559 interface(CONST_INTER);
4560 %}
4561
4562 // Double Immediate: +0.0d
4563 operand immD0()
4564 %{
4565 predicate(jlong_cast(n->getd()) == 0);
4566 match(ConD);
4567
4568 op_cost(0);
4569 format %{ %}
4570 interface(CONST_INTER);
4571 %}
4572
4573 // constant 'double +0.0'.
4574 operand immDPacked()
4575 %{
4576 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4577 match(ConD);
4578 op_cost(0);
4579 format %{ %}
4580 interface(CONST_INTER);
4581 %}
4582
4583 // Float Immediate
4584 operand immF()
4585 %{
4586 match(ConF);
4587 op_cost(0);
4588 format %{ %}
4589 interface(CONST_INTER);
4590 %}
4591
4592 // Float Immediate: +0.0f.
4593 operand immF0()
4594 %{
4595 predicate(jint_cast(n->getf()) == 0);
4596 match(ConF);
4597
4598 op_cost(0);
4599 format %{ %}
4600 interface(CONST_INTER);
4601 %}
4602
4603 //
4604 operand immFPacked()
4605 %{
4606 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4607 match(ConF);
4608 op_cost(0);
4609 format %{ %}
4610 interface(CONST_INTER);
4611 %}
4612
4613 // Narrow pointer operands
4614 // Narrow Pointer Immediate
4615 operand immN()
4616 %{
4617 match(ConN);
4618
4619 op_cost(0);
4620 format %{ %}
4621 interface(CONST_INTER);
4622 %}
4623
4624 // Narrow nullptr Pointer Immediate
4625 operand immN0()
4626 %{
4627 predicate(n->get_narrowcon() == 0);
4628 match(ConN);
4629
4630 op_cost(0);
4631 format %{ %}
4632 interface(CONST_INTER);
4633 %}
4634
4635 operand immNKlass()
4636 %{
4637 match(ConNKlass);
4638
4639 op_cost(0);
4640 format %{ %}
4641 interface(CONST_INTER);
4642 %}
4643
4644 // Integer 32 bit Register Operands
4645 // Integer 32 bitRegister (excludes SP)
4646 operand iRegI()
4647 %{
4648 constraint(ALLOC_IN_RC(any_reg32));
4649 match(RegI);
4650 match(iRegINoSp);
4651 op_cost(0);
4652 format %{ %}
4653 interface(REG_INTER);
4654 %}
4655
4656 // Integer 32 bit Register not Special
4657 operand iRegINoSp()
4658 %{
4659 constraint(ALLOC_IN_RC(no_special_reg32));
4660 match(RegI);
4661 op_cost(0);
4662 format %{ %}
4663 interface(REG_INTER);
4664 %}
4665
4666 // Integer 64 bit Register Operands
4667 // Integer 64 bit Register (includes SP)
4668 operand iRegL()
4669 %{
4670 constraint(ALLOC_IN_RC(any_reg));
4671 match(RegL);
4672 match(iRegLNoSp);
4673 op_cost(0);
4674 format %{ %}
4675 interface(REG_INTER);
4676 %}
4677
4678 // Integer 64 bit Register not Special
4679 operand iRegLNoSp()
4680 %{
4681 constraint(ALLOC_IN_RC(no_special_reg));
4682 match(RegL);
4683 match(iRegL_R0);
4684 format %{ %}
4685 interface(REG_INTER);
4686 %}
4687
4688 // Pointer Register Operands
4689 // Pointer Register
4690 operand iRegP()
4691 %{
4692 constraint(ALLOC_IN_RC(ptr_reg));
4693 match(RegP);
4694 match(iRegPNoSp);
4695 match(iRegP_R0);
4696 //match(iRegP_R2);
4697 //match(iRegP_R4);
4698 match(iRegP_R5);
4699 match(thread_RegP);
4700 op_cost(0);
4701 format %{ %}
4702 interface(REG_INTER);
4703 %}
4704
4705 // Pointer 64 bit Register not Special
4706 operand iRegPNoSp()
4707 %{
4708 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4709 match(RegP);
4710 // match(iRegP);
4711 // match(iRegP_R0);
4712 // match(iRegP_R2);
4713 // match(iRegP_R4);
4714 // match(iRegP_R5);
4715 // match(thread_RegP);
4716 op_cost(0);
4717 format %{ %}
4718 interface(REG_INTER);
4719 %}
4720
4721 // This operand is not allowed to use rfp even if
4722 // rfp is not used to hold the frame pointer.
4723 operand iRegPNoSpNoRfp()
4724 %{
4725 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4726 match(RegP);
4727 match(iRegPNoSp);
4728 op_cost(0);
4729 format %{ %}
4730 interface(REG_INTER);
4731 %}
4732
4733 // Pointer 64 bit Register R0 only
4734 operand iRegP_R0()
4735 %{
4736 constraint(ALLOC_IN_RC(r0_reg));
4737 match(RegP);
4738 // match(iRegP);
4739 match(iRegPNoSp);
4740 op_cost(0);
4741 format %{ %}
4742 interface(REG_INTER);
4743 %}
4744
4745 // Pointer 64 bit Register R1 only
4746 operand iRegP_R1()
4747 %{
4748 constraint(ALLOC_IN_RC(r1_reg));
4749 match(RegP);
4750 // match(iRegP);
4751 match(iRegPNoSp);
4752 op_cost(0);
4753 format %{ %}
4754 interface(REG_INTER);
4755 %}
4756
4757 // Pointer 64 bit Register R2 only
4758 operand iRegP_R2()
4759 %{
4760 constraint(ALLOC_IN_RC(r2_reg));
4761 match(RegP);
4762 // match(iRegP);
4763 match(iRegPNoSp);
4764 op_cost(0);
4765 format %{ %}
4766 interface(REG_INTER);
4767 %}
4768
4769 // Pointer 64 bit Register R3 only
4770 operand iRegP_R3()
4771 %{
4772 constraint(ALLOC_IN_RC(r3_reg));
4773 match(RegP);
4774 // match(iRegP);
4775 match(iRegPNoSp);
4776 op_cost(0);
4777 format %{ %}
4778 interface(REG_INTER);
4779 %}
4780
4781 // Pointer 64 bit Register R4 only
4782 operand iRegP_R4()
4783 %{
4784 constraint(ALLOC_IN_RC(r4_reg));
4785 match(RegP);
4786 // match(iRegP);
4787 match(iRegPNoSp);
4788 op_cost(0);
4789 format %{ %}
4790 interface(REG_INTER);
4791 %}
4792
4793 // Pointer 64 bit Register R5 only
4794 operand iRegP_R5()
4795 %{
4796 constraint(ALLOC_IN_RC(r5_reg));
4797 match(RegP);
4798 // match(iRegP);
4799 match(iRegPNoSp);
4800 op_cost(0);
4801 format %{ %}
4802 interface(REG_INTER);
4803 %}
4804
4805 // Pointer 64 bit Register R10 only
4806 operand iRegP_R10()
4807 %{
4808 constraint(ALLOC_IN_RC(r10_reg));
4809 match(RegP);
4810 // match(iRegP);
4811 match(iRegPNoSp);
4812 op_cost(0);
4813 format %{ %}
4814 interface(REG_INTER);
4815 %}
4816
4817 // Long 64 bit Register R0 only
4818 operand iRegL_R0()
4819 %{
4820 constraint(ALLOC_IN_RC(r0_reg));
4821 match(RegL);
4822 match(iRegLNoSp);
4823 op_cost(0);
4824 format %{ %}
4825 interface(REG_INTER);
4826 %}
4827
4828 // Long 64 bit Register R11 only
4829 operand iRegL_R11()
4830 %{
4831 constraint(ALLOC_IN_RC(r11_reg));
4832 match(RegL);
4833 match(iRegLNoSp);
4834 op_cost(0);
4835 format %{ %}
4836 interface(REG_INTER);
4837 %}
4838
4839 // Register R0 only
4840 operand iRegI_R0()
4841 %{
4842 constraint(ALLOC_IN_RC(int_r0_reg));
4843 match(RegI);
4844 match(iRegINoSp);
4845 op_cost(0);
4846 format %{ %}
4847 interface(REG_INTER);
4848 %}
4849
4850 // Register R2 only
4851 operand iRegI_R2()
4852 %{
4853 constraint(ALLOC_IN_RC(int_r2_reg));
4854 match(RegI);
4855 match(iRegINoSp);
4856 op_cost(0);
4857 format %{ %}
4858 interface(REG_INTER);
4859 %}
4860
4861 // Register R3 only
4862 operand iRegI_R3()
4863 %{
4864 constraint(ALLOC_IN_RC(int_r3_reg));
4865 match(RegI);
4866 match(iRegINoSp);
4867 op_cost(0);
4868 format %{ %}
4869 interface(REG_INTER);
4870 %}
4871
4872
4873 // Register R4 only
4874 operand iRegI_R4()
4875 %{
4876 constraint(ALLOC_IN_RC(int_r4_reg));
4877 match(RegI);
4878 match(iRegINoSp);
4879 op_cost(0);
4880 format %{ %}
4881 interface(REG_INTER);
4882 %}
4883
4884
4885 // Pointer Register Operands
4886 // Narrow Pointer Register
4887 operand iRegN()
4888 %{
4889 constraint(ALLOC_IN_RC(any_reg32));
4890 match(RegN);
4891 match(iRegNNoSp);
4892 op_cost(0);
4893 format %{ %}
4894 interface(REG_INTER);
4895 %}
4896
4897 // Integer 64 bit Register not Special
4898 operand iRegNNoSp()
4899 %{
4900 constraint(ALLOC_IN_RC(no_special_reg32));
4901 match(RegN);
4902 op_cost(0);
4903 format %{ %}
4904 interface(REG_INTER);
4905 %}
4906
4907 // Float Register
4908 // Float register operands
4909 operand vRegF()
4910 %{
4911 constraint(ALLOC_IN_RC(float_reg));
4912 match(RegF);
4913
4914 op_cost(0);
4915 format %{ %}
4916 interface(REG_INTER);
4917 %}
4918
4919 // Double Register
4920 // Double register operands
4921 operand vRegD()
4922 %{
4923 constraint(ALLOC_IN_RC(double_reg));
4924 match(RegD);
4925
4926 op_cost(0);
4927 format %{ %}
4928 interface(REG_INTER);
4929 %}
4930
4931 // Generic vector class. This will be used for
4932 // all vector operands, including NEON and SVE.
4933 operand vReg()
4934 %{
4935 constraint(ALLOC_IN_RC(dynamic));
4936 match(VecA);
4937 match(VecD);
4938 match(VecX);
4939
4940 op_cost(0);
4941 format %{ %}
4942 interface(REG_INTER);
4943 %}
4944
4945 operand vecA()
4946 %{
4947 constraint(ALLOC_IN_RC(vectora_reg));
4948 match(VecA);
4949
4950 op_cost(0);
4951 format %{ %}
4952 interface(REG_INTER);
4953 %}
4954
4955 operand vecD()
4956 %{
4957 constraint(ALLOC_IN_RC(vectord_reg));
4958 match(VecD);
4959
4960 op_cost(0);
4961 format %{ %}
4962 interface(REG_INTER);
4963 %}
4964
4965 operand vecX()
4966 %{
4967 constraint(ALLOC_IN_RC(vectorx_reg));
4968 match(VecX);
4969
4970 op_cost(0);
4971 format %{ %}
4972 interface(REG_INTER);
4973 %}
4974
4975 operand vRegD_V0()
4976 %{
4977 constraint(ALLOC_IN_RC(v0_reg));
4978 match(RegD);
4979 op_cost(0);
4980 format %{ %}
4981 interface(REG_INTER);
4982 %}
4983
4984 operand vRegD_V1()
4985 %{
4986 constraint(ALLOC_IN_RC(v1_reg));
4987 match(RegD);
4988 op_cost(0);
4989 format %{ %}
4990 interface(REG_INTER);
4991 %}
4992
4993 operand vRegD_V2()
4994 %{
4995 constraint(ALLOC_IN_RC(v2_reg));
4996 match(RegD);
4997 op_cost(0);
4998 format %{ %}
4999 interface(REG_INTER);
5000 %}
5001
5002 operand vRegD_V3()
5003 %{
5004 constraint(ALLOC_IN_RC(v3_reg));
5005 match(RegD);
5006 op_cost(0);
5007 format %{ %}
5008 interface(REG_INTER);
5009 %}
5010
5011 operand vRegD_V4()
5012 %{
5013 constraint(ALLOC_IN_RC(v4_reg));
5014 match(RegD);
5015 op_cost(0);
5016 format %{ %}
5017 interface(REG_INTER);
5018 %}
5019
5020 operand vRegD_V5()
5021 %{
5022 constraint(ALLOC_IN_RC(v5_reg));
5023 match(RegD);
5024 op_cost(0);
5025 format %{ %}
5026 interface(REG_INTER);
5027 %}
5028
5029 operand vRegD_V6()
5030 %{
5031 constraint(ALLOC_IN_RC(v6_reg));
5032 match(RegD);
5033 op_cost(0);
5034 format %{ %}
5035 interface(REG_INTER);
5036 %}
5037
5038 operand vRegD_V7()
5039 %{
5040 constraint(ALLOC_IN_RC(v7_reg));
5041 match(RegD);
5042 op_cost(0);
5043 format %{ %}
5044 interface(REG_INTER);
5045 %}
5046
5047 operand vRegD_V12()
5048 %{
5049 constraint(ALLOC_IN_RC(v12_reg));
5050 match(RegD);
5051 op_cost(0);
5052 format %{ %}
5053 interface(REG_INTER);
5054 %}
5055
5056 operand vRegD_V13()
5057 %{
5058 constraint(ALLOC_IN_RC(v13_reg));
5059 match(RegD);
5060 op_cost(0);
5061 format %{ %}
5062 interface(REG_INTER);
5063 %}
5064
5065 operand pReg()
5066 %{
5067 constraint(ALLOC_IN_RC(pr_reg));
5068 match(RegVectMask);
5069 match(pRegGov);
5070 op_cost(0);
5071 format %{ %}
5072 interface(REG_INTER);
5073 %}
5074
5075 operand pRegGov()
5076 %{
5077 constraint(ALLOC_IN_RC(gov_pr));
5078 match(RegVectMask);
5079 match(pReg);
5080 op_cost(0);
5081 format %{ %}
5082 interface(REG_INTER);
5083 %}
5084
5085 operand pRegGov_P0()
5086 %{
5087 constraint(ALLOC_IN_RC(p0_reg));
5088 match(RegVectMask);
5089 op_cost(0);
5090 format %{ %}
5091 interface(REG_INTER);
5092 %}
5093
5094 operand pRegGov_P1()
5095 %{
5096 constraint(ALLOC_IN_RC(p1_reg));
5097 match(RegVectMask);
5098 op_cost(0);
5099 format %{ %}
5100 interface(REG_INTER);
5101 %}
5102
5103 // Flags register, used as output of signed compare instructions
5104
5105 // note that on AArch64 we also use this register as the output for
5106 // for floating point compare instructions (CmpF CmpD). this ensures
5107 // that ordered inequality tests use GT, GE, LT or LE none of which
5108 // pass through cases where the result is unordered i.e. one or both
5109 // inputs to the compare is a NaN. this means that the ideal code can
5110 // replace e.g. a GT with an LE and not end up capturing the NaN case
5111 // (where the comparison should always fail). EQ and NE tests are
5112 // always generated in ideal code so that unordered folds into the NE
5113 // case, matching the behaviour of AArch64 NE.
5114 //
5115 // This differs from x86 where the outputs of FP compares use a
5116 // special FP flags registers and where compares based on this
5117 // register are distinguished into ordered inequalities (cmpOpUCF) and
5118 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5119 // to explicitly handle the unordered case in branches. x86 also has
5120 // to include extra CMoveX rules to accept a cmpOpUCF input.
5121
5122 operand rFlagsReg()
5123 %{
5124 constraint(ALLOC_IN_RC(int_flags));
5125 match(RegFlags);
5126
5127 op_cost(0);
5128 format %{ "RFLAGS" %}
5129 interface(REG_INTER);
5130 %}
5131
5132 // Flags register, used as output of unsigned compare instructions
5133 operand rFlagsRegU()
5134 %{
5135 constraint(ALLOC_IN_RC(int_flags));
5136 match(RegFlags);
5137
5138 op_cost(0);
5139 format %{ "RFLAGSU" %}
5140 interface(REG_INTER);
5141 %}
5142
5143 // Special Registers
5144
5145 // Method Register
5146 operand inline_cache_RegP(iRegP reg)
5147 %{
5148 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5149 match(reg);
5150 match(iRegPNoSp);
5151 op_cost(0);
5152 format %{ %}
5153 interface(REG_INTER);
5154 %}
5155
5156 // Thread Register
5157 operand thread_RegP(iRegP reg)
5158 %{
5159 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5160 match(reg);
5161 op_cost(0);
5162 format %{ %}
5163 interface(REG_INTER);
5164 %}
5165
5166 //----------Memory Operands----------------------------------------------------
5167
5168 operand indirect(iRegP reg)
5169 %{
5170 constraint(ALLOC_IN_RC(ptr_reg));
5171 match(reg);
5172 op_cost(0);
5173 format %{ "[$reg]" %}
5174 interface(MEMORY_INTER) %{
5175 base($reg);
5176 index(0xffffffff);
5177 scale(0x0);
5178 disp(0x0);
5179 %}
5180 %}
5181
5182 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5183 %{
5184 constraint(ALLOC_IN_RC(ptr_reg));
5185 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5186 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5187 op_cost(0);
5188 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5189 interface(MEMORY_INTER) %{
5190 base($reg);
5191 index($ireg);
5192 scale($scale);
5193 disp(0x0);
5194 %}
5195 %}
5196
5197 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5198 %{
5199 constraint(ALLOC_IN_RC(ptr_reg));
5200 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5201 match(AddP reg (LShiftL lreg scale));
5202 op_cost(0);
5203 format %{ "$reg, $lreg lsl($scale)" %}
5204 interface(MEMORY_INTER) %{
5205 base($reg);
5206 index($lreg);
5207 scale($scale);
5208 disp(0x0);
5209 %}
5210 %}
5211
5212 operand indIndexI2L(iRegP reg, iRegI ireg)
5213 %{
5214 constraint(ALLOC_IN_RC(ptr_reg));
5215 match(AddP reg (ConvI2L ireg));
5216 op_cost(0);
5217 format %{ "$reg, $ireg, 0, I2L" %}
5218 interface(MEMORY_INTER) %{
5219 base($reg);
5220 index($ireg);
5221 scale(0x0);
5222 disp(0x0);
5223 %}
5224 %}
5225
5226 operand indIndex(iRegP reg, iRegL lreg)
5227 %{
5228 constraint(ALLOC_IN_RC(ptr_reg));
5229 match(AddP reg lreg);
5230 op_cost(0);
5231 format %{ "$reg, $lreg" %}
5232 interface(MEMORY_INTER) %{
5233 base($reg);
5234 index($lreg);
5235 scale(0x0);
5236 disp(0x0);
5237 %}
5238 %}
5239
5240 operand indOffI1(iRegP reg, immIOffset1 off)
5241 %{
5242 constraint(ALLOC_IN_RC(ptr_reg));
5243 match(AddP reg off);
5244 op_cost(0);
5245 format %{ "[$reg, $off]" %}
5246 interface(MEMORY_INTER) %{
5247 base($reg);
5248 index(0xffffffff);
5249 scale(0x0);
5250 disp($off);
5251 %}
5252 %}
5253
5254 operand indOffI2(iRegP reg, immIOffset2 off)
5255 %{
5256 constraint(ALLOC_IN_RC(ptr_reg));
5257 match(AddP reg off);
5258 op_cost(0);
5259 format %{ "[$reg, $off]" %}
5260 interface(MEMORY_INTER) %{
5261 base($reg);
5262 index(0xffffffff);
5263 scale(0x0);
5264 disp($off);
5265 %}
5266 %}
5267
5268 operand indOffI4(iRegP reg, immIOffset4 off)
5269 %{
5270 constraint(ALLOC_IN_RC(ptr_reg));
5271 match(AddP reg off);
5272 op_cost(0);
5273 format %{ "[$reg, $off]" %}
5274 interface(MEMORY_INTER) %{
5275 base($reg);
5276 index(0xffffffff);
5277 scale(0x0);
5278 disp($off);
5279 %}
5280 %}
5281
5282 operand indOffI8(iRegP reg, immIOffset8 off)
5283 %{
5284 constraint(ALLOC_IN_RC(ptr_reg));
5285 match(AddP reg off);
5286 op_cost(0);
5287 format %{ "[$reg, $off]" %}
5288 interface(MEMORY_INTER) %{
5289 base($reg);
5290 index(0xffffffff);
5291 scale(0x0);
5292 disp($off);
5293 %}
5294 %}
5295
5296 operand indOffI16(iRegP reg, immIOffset16 off)
5297 %{
5298 constraint(ALLOC_IN_RC(ptr_reg));
5299 match(AddP reg off);
5300 op_cost(0);
5301 format %{ "[$reg, $off]" %}
5302 interface(MEMORY_INTER) %{
5303 base($reg);
5304 index(0xffffffff);
5305 scale(0x0);
5306 disp($off);
5307 %}
5308 %}
5309
5310 operand indOffL1(iRegP reg, immLoffset1 off)
5311 %{
5312 constraint(ALLOC_IN_RC(ptr_reg));
5313 match(AddP reg off);
5314 op_cost(0);
5315 format %{ "[$reg, $off]" %}
5316 interface(MEMORY_INTER) %{
5317 base($reg);
5318 index(0xffffffff);
5319 scale(0x0);
5320 disp($off);
5321 %}
5322 %}
5323
5324 operand indOffL2(iRegP reg, immLoffset2 off)
5325 %{
5326 constraint(ALLOC_IN_RC(ptr_reg));
5327 match(AddP reg off);
5328 op_cost(0);
5329 format %{ "[$reg, $off]" %}
5330 interface(MEMORY_INTER) %{
5331 base($reg);
5332 index(0xffffffff);
5333 scale(0x0);
5334 disp($off);
5335 %}
5336 %}
5337
5338 operand indOffL4(iRegP reg, immLoffset4 off)
5339 %{
5340 constraint(ALLOC_IN_RC(ptr_reg));
5341 match(AddP reg off);
5342 op_cost(0);
5343 format %{ "[$reg, $off]" %}
5344 interface(MEMORY_INTER) %{
5345 base($reg);
5346 index(0xffffffff);
5347 scale(0x0);
5348 disp($off);
5349 %}
5350 %}
5351
5352 operand indOffL8(iRegP reg, immLoffset8 off)
5353 %{
5354 constraint(ALLOC_IN_RC(ptr_reg));
5355 match(AddP reg off);
5356 op_cost(0);
5357 format %{ "[$reg, $off]" %}
5358 interface(MEMORY_INTER) %{
5359 base($reg);
5360 index(0xffffffff);
5361 scale(0x0);
5362 disp($off);
5363 %}
5364 %}
5365
5366 operand indOffL16(iRegP reg, immLoffset16 off)
5367 %{
5368 constraint(ALLOC_IN_RC(ptr_reg));
5369 match(AddP reg off);
5370 op_cost(0);
5371 format %{ "[$reg, $off]" %}
5372 interface(MEMORY_INTER) %{
5373 base($reg);
5374 index(0xffffffff);
5375 scale(0x0);
5376 disp($off);
5377 %}
5378 %}
5379
5380 operand indirectX2P(iRegL reg)
5381 %{
5382 constraint(ALLOC_IN_RC(ptr_reg));
5383 match(CastX2P reg);
5384 op_cost(0);
5385 format %{ "[$reg]\t# long -> ptr" %}
5386 interface(MEMORY_INTER) %{
5387 base($reg);
5388 index(0xffffffff);
5389 scale(0x0);
5390 disp(0x0);
5391 %}
5392 %}
5393
5394 operand indOffX2P(iRegL reg, immLOffset off)
5395 %{
5396 constraint(ALLOC_IN_RC(ptr_reg));
5397 match(AddP (CastX2P reg) off);
5398 op_cost(0);
5399 format %{ "[$reg, $off]\t# long -> ptr" %}
5400 interface(MEMORY_INTER) %{
5401 base($reg);
5402 index(0xffffffff);
5403 scale(0x0);
5404 disp($off);
5405 %}
5406 %}
5407
5408 operand indirectN(iRegN reg)
5409 %{
5410 predicate(CompressedOops::shift() == 0);
5411 constraint(ALLOC_IN_RC(ptr_reg));
5412 match(DecodeN reg);
5413 op_cost(0);
5414 format %{ "[$reg]\t# narrow" %}
5415 interface(MEMORY_INTER) %{
5416 base($reg);
5417 index(0xffffffff);
5418 scale(0x0);
5419 disp(0x0);
5420 %}
5421 %}
5422
5423 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5424 %{
5425 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5426 constraint(ALLOC_IN_RC(ptr_reg));
5427 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5428 op_cost(0);
5429 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5430 interface(MEMORY_INTER) %{
5431 base($reg);
5432 index($ireg);
5433 scale($scale);
5434 disp(0x0);
5435 %}
5436 %}
5437
5438 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5439 %{
5440 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5441 constraint(ALLOC_IN_RC(ptr_reg));
5442 match(AddP (DecodeN reg) (LShiftL lreg scale));
5443 op_cost(0);
5444 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5445 interface(MEMORY_INTER) %{
5446 base($reg);
5447 index($lreg);
5448 scale($scale);
5449 disp(0x0);
5450 %}
5451 %}
5452
5453 operand indIndexI2LN(iRegN reg, iRegI ireg)
5454 %{
5455 predicate(CompressedOops::shift() == 0);
5456 constraint(ALLOC_IN_RC(ptr_reg));
5457 match(AddP (DecodeN reg) (ConvI2L ireg));
5458 op_cost(0);
5459 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5460 interface(MEMORY_INTER) %{
5461 base($reg);
5462 index($ireg);
5463 scale(0x0);
5464 disp(0x0);
5465 %}
5466 %}
5467
5468 operand indIndexN(iRegN reg, iRegL lreg)
5469 %{
5470 predicate(CompressedOops::shift() == 0);
5471 constraint(ALLOC_IN_RC(ptr_reg));
5472 match(AddP (DecodeN reg) lreg);
5473 op_cost(0);
5474 format %{ "$reg, $lreg\t# narrow" %}
5475 interface(MEMORY_INTER) %{
5476 base($reg);
5477 index($lreg);
5478 scale(0x0);
5479 disp(0x0);
5480 %}
5481 %}
5482
5483 operand indOffIN(iRegN reg, immIOffset off)
5484 %{
5485 predicate(CompressedOops::shift() == 0);
5486 constraint(ALLOC_IN_RC(ptr_reg));
5487 match(AddP (DecodeN reg) off);
5488 op_cost(0);
5489 format %{ "[$reg, $off]\t# narrow" %}
5490 interface(MEMORY_INTER) %{
5491 base($reg);
5492 index(0xffffffff);
5493 scale(0x0);
5494 disp($off);
5495 %}
5496 %}
5497
5498 operand indOffLN(iRegN reg, immLOffset off)
5499 %{
5500 predicate(CompressedOops::shift() == 0);
5501 constraint(ALLOC_IN_RC(ptr_reg));
5502 match(AddP (DecodeN reg) off);
5503 op_cost(0);
5504 format %{ "[$reg, $off]\t# narrow" %}
5505 interface(MEMORY_INTER) %{
5506 base($reg);
5507 index(0xffffffff);
5508 scale(0x0);
5509 disp($off);
5510 %}
5511 %}
5512
5513
5514 //----------Special Memory Operands--------------------------------------------
5515 // Stack Slot Operand - This operand is used for loading and storing temporary
5516 // values on the stack where a match requires a value to
5517 // flow through memory.
5518 operand stackSlotP(sRegP reg)
5519 %{
5520 constraint(ALLOC_IN_RC(stack_slots));
5521 op_cost(100);
5522 // No match rule because this operand is only generated in matching
5523 // match(RegP);
5524 format %{ "[$reg]" %}
5525 interface(MEMORY_INTER) %{
5526 base(0x1e); // RSP
5527 index(0x0); // No Index
5528 scale(0x0); // No Scale
5529 disp($reg); // Stack Offset
5530 %}
5531 %}
5532
5533 operand stackSlotI(sRegI reg)
5534 %{
5535 constraint(ALLOC_IN_RC(stack_slots));
5536 // No match rule because this operand is only generated in matching
5537 // match(RegI);
5538 format %{ "[$reg]" %}
5539 interface(MEMORY_INTER) %{
5540 base(0x1e); // RSP
5541 index(0x0); // No Index
5542 scale(0x0); // No Scale
5543 disp($reg); // Stack Offset
5544 %}
5545 %}
5546
5547 operand stackSlotF(sRegF reg)
5548 %{
5549 constraint(ALLOC_IN_RC(stack_slots));
5550 // No match rule because this operand is only generated in matching
5551 // match(RegF);
5552 format %{ "[$reg]" %}
5553 interface(MEMORY_INTER) %{
5554 base(0x1e); // RSP
5555 index(0x0); // No Index
5556 scale(0x0); // No Scale
5557 disp($reg); // Stack Offset
5558 %}
5559 %}
5560
5561 operand stackSlotD(sRegD reg)
5562 %{
5563 constraint(ALLOC_IN_RC(stack_slots));
5564 // No match rule because this operand is only generated in matching
5565 // match(RegD);
5566 format %{ "[$reg]" %}
5567 interface(MEMORY_INTER) %{
5568 base(0x1e); // RSP
5569 index(0x0); // No Index
5570 scale(0x0); // No Scale
5571 disp($reg); // Stack Offset
5572 %}
5573 %}
5574
5575 operand stackSlotL(sRegL reg)
5576 %{
5577 constraint(ALLOC_IN_RC(stack_slots));
5578 // No match rule because this operand is only generated in matching
5579 // match(RegL);
5580 format %{ "[$reg]" %}
5581 interface(MEMORY_INTER) %{
5582 base(0x1e); // RSP
5583 index(0x0); // No Index
5584 scale(0x0); // No Scale
5585 disp($reg); // Stack Offset
5586 %}
5587 %}
5588
5589 // Operands for expressing Control Flow
5590 // NOTE: Label is a predefined operand which should not be redefined in
5591 // the AD file. It is generically handled within the ADLC.
5592
5593 //----------Conditional Branch Operands----------------------------------------
5594 // Comparison Op - This is the operation of the comparison, and is limited to
5595 // the following set of codes:
5596 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5597 //
5598 // Other attributes of the comparison, such as unsignedness, are specified
5599 // by the comparison instruction that sets a condition code flags register.
5600 // That result is represented by a flags operand whose subtype is appropriate
5601 // to the unsignedness (etc.) of the comparison.
5602 //
5603 // Later, the instruction which matches both the Comparison Op (a Bool) and
5604 // the flags (produced by the Cmp) specifies the coding of the comparison op
5605 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5606
5607 // used for signed integral comparisons and fp comparisons
5608
5609 operand cmpOp()
5610 %{
5611 match(Bool);
5612
5613 format %{ "" %}
5614 interface(COND_INTER) %{
5615 equal(0x0, "eq");
5616 not_equal(0x1, "ne");
5617 less(0xb, "lt");
5618 greater_equal(0xa, "ge");
5619 less_equal(0xd, "le");
5620 greater(0xc, "gt");
5621 overflow(0x6, "vs");
5622 no_overflow(0x7, "vc");
5623 %}
5624 %}
5625
5626 // used for unsigned integral comparisons
5627
5628 operand cmpOpU()
5629 %{
5630 match(Bool);
5631
5632 format %{ "" %}
5633 interface(COND_INTER) %{
5634 equal(0x0, "eq");
5635 not_equal(0x1, "ne");
5636 less(0x3, "lo");
5637 greater_equal(0x2, "hs");
5638 less_equal(0x9, "ls");
5639 greater(0x8, "hi");
5640 overflow(0x6, "vs");
5641 no_overflow(0x7, "vc");
5642 %}
5643 %}
5644
5645 // used for certain integral comparisons which can be
5646 // converted to cbxx or tbxx instructions
5647
5648 operand cmpOpEqNe()
5649 %{
5650 match(Bool);
5651 op_cost(0);
5652 predicate(n->as_Bool()->_test._test == BoolTest::ne
5653 || n->as_Bool()->_test._test == BoolTest::eq);
5654
5655 format %{ "" %}
5656 interface(COND_INTER) %{
5657 equal(0x0, "eq");
5658 not_equal(0x1, "ne");
5659 less(0xb, "lt");
5660 greater_equal(0xa, "ge");
5661 less_equal(0xd, "le");
5662 greater(0xc, "gt");
5663 overflow(0x6, "vs");
5664 no_overflow(0x7, "vc");
5665 %}
5666 %}
5667
5668 // used for certain integral comparisons which can be
5669 // converted to cbxx or tbxx instructions
5670
5671 operand cmpOpLtGe()
5672 %{
5673 match(Bool);
5674 op_cost(0);
5675
5676 predicate(n->as_Bool()->_test._test == BoolTest::lt
5677 || n->as_Bool()->_test._test == BoolTest::ge);
5678
5679 format %{ "" %}
5680 interface(COND_INTER) %{
5681 equal(0x0, "eq");
5682 not_equal(0x1, "ne");
5683 less(0xb, "lt");
5684 greater_equal(0xa, "ge");
5685 less_equal(0xd, "le");
5686 greater(0xc, "gt");
5687 overflow(0x6, "vs");
5688 no_overflow(0x7, "vc");
5689 %}
5690 %}
5691
5692 // used for certain unsigned integral comparisons which can be
5693 // converted to cbxx or tbxx instructions
5694
5695 operand cmpOpUEqNeLeGt()
5696 %{
5697 match(Bool);
5698 op_cost(0);
5699
5700 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5701 n->as_Bool()->_test._test == BoolTest::ne ||
5702 n->as_Bool()->_test._test == BoolTest::le ||
5703 n->as_Bool()->_test._test == BoolTest::gt);
5704
5705 format %{ "" %}
5706 interface(COND_INTER) %{
5707 equal(0x0, "eq");
5708 not_equal(0x1, "ne");
5709 less(0x3, "lo");
5710 greater_equal(0x2, "hs");
5711 less_equal(0x9, "ls");
5712 greater(0x8, "hi");
5713 overflow(0x6, "vs");
5714 no_overflow(0x7, "vc");
5715 %}
5716 %}
5717
5718 // Special operand allowing long args to int ops to be truncated for free
5719
5720 operand iRegL2I(iRegL reg) %{
5721
5722 op_cost(0);
5723
5724 match(ConvL2I reg);
5725
5726 format %{ "l2i($reg)" %}
5727
5728 interface(REG_INTER)
5729 %}
5730
5731 operand iRegL2P(iRegL reg) %{
5732
5733 op_cost(0);
5734
5735 match(CastX2P reg);
5736
5737 format %{ "l2p($reg)" %}
5738
5739 interface(REG_INTER)
5740 %}
5741
5742 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5743 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5744 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5745 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5746
5747 //----------OPERAND CLASSES----------------------------------------------------
5748 // Operand Classes are groups of operands that are used as to simplify
5749 // instruction definitions by not requiring the AD writer to specify
5750 // separate instructions for every form of operand when the
5751 // instruction accepts multiple operand types with the same basic
5752 // encoding and format. The classic case of this is memory operands.
5753
5754 // memory is used to define read/write location for load/store
5755 // instruction defs. we can turn a memory op into an Address
5756
5757 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5758 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5759
5760 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5761 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5762
5763 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5764 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5765
5766 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5767 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5768
5769 // All of the memory operands. For the pipeline description.
5770 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5771 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5772 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5773
5774 opclass memory_noindex(indirect,
5775 indOffI1, indOffL1,indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5776 indirectN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5777
5778 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5779 // operations. it allows the src to be either an iRegI or a (ConvL2I
5780 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5781 // can be elided because the 32-bit instruction will just employ the
5782 // lower 32 bits anyway.
5783 //
5784 // n.b. this does not elide all L2I conversions. if the truncated
5785 // value is consumed by more than one operation then the ConvL2I
5786 // cannot be bundled into the consuming nodes so an l2i gets planted
5787 // (actually a movw $dst $src) and the downstream instructions consume
5788 // the result of the l2i as an iRegI input. That's a shame since the
5789 // movw is actually redundant but its not too costly.
5790
5791 opclass iRegIorL2I(iRegI, iRegL2I);
5792 opclass iRegPorL2P(iRegP, iRegL2P);
5793
5794 //----------PIPELINE-----------------------------------------------------------
5795 // Rules which define the behavior of the target architectures pipeline.
5796
5797 // For specific pipelines, eg A53, define the stages of that pipeline
5798 //pipe_desc(ISS, EX1, EX2, WR);
5799 #define ISS S0
5800 #define EX1 S1
5801 #define EX2 S2
5802 #define WR S3
5803
5804 // Integer ALU reg operation
5805 pipeline %{
5806
5807 attributes %{
5808 // ARM instructions are of fixed length
5809 fixed_size_instructions; // Fixed size instructions TODO does
5810 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5811 // ARM instructions come in 32-bit word units
5812 instruction_unit_size = 4; // An instruction is 4 bytes long
5813 instruction_fetch_unit_size = 64; // The processor fetches one line
5814 instruction_fetch_units = 1; // of 64 bytes
5815
5816 // List of nop instructions
5817 nops( MachNop );
5818 %}
5819
5820 // We don't use an actual pipeline model so don't care about resources
5821 // or description. we do use pipeline classes to introduce fixed
5822 // latencies
5823
5824 //----------RESOURCES----------------------------------------------------------
5825 // Resources are the functional units available to the machine
5826
5827 resources( INS0, INS1, INS01 = INS0 | INS1,
5828 ALU0, ALU1, ALU = ALU0 | ALU1,
5829 MAC,
5830 DIV,
5831 BRANCH,
5832 LDST,
5833 NEON_FP);
5834
5835 //----------PIPELINE DESCRIPTION-----------------------------------------------
5836 // Pipeline Description specifies the stages in the machine's pipeline
5837
5838 // Define the pipeline as a generic 6 stage pipeline
5839 pipe_desc(S0, S1, S2, S3, S4, S5);
5840
5841 //----------PIPELINE CLASSES---------------------------------------------------
5842 // Pipeline Classes describe the stages in which input and output are
5843 // referenced by the hardware pipeline.
5844
5845 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5846 %{
5847 single_instruction;
5848 src1 : S1(read);
5849 src2 : S2(read);
5850 dst : S5(write);
5851 INS01 : ISS;
5852 NEON_FP : S5;
5853 %}
5854
5855 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5856 %{
5857 single_instruction;
5858 src1 : S1(read);
5859 src2 : S2(read);
5860 dst : S5(write);
5861 INS01 : ISS;
5862 NEON_FP : S5;
5863 %}
5864
5865 pipe_class fp_uop_s(vRegF dst, vRegF src)
5866 %{
5867 single_instruction;
5868 src : S1(read);
5869 dst : S5(write);
5870 INS01 : ISS;
5871 NEON_FP : S5;
5872 %}
5873
5874 pipe_class fp_uop_d(vRegD dst, vRegD src)
5875 %{
5876 single_instruction;
5877 src : S1(read);
5878 dst : S5(write);
5879 INS01 : ISS;
5880 NEON_FP : S5;
5881 %}
5882
5883 pipe_class fp_d2f(vRegF dst, vRegD src)
5884 %{
5885 single_instruction;
5886 src : S1(read);
5887 dst : S5(write);
5888 INS01 : ISS;
5889 NEON_FP : S5;
5890 %}
5891
5892 pipe_class fp_f2d(vRegD dst, vRegF src)
5893 %{
5894 single_instruction;
5895 src : S1(read);
5896 dst : S5(write);
5897 INS01 : ISS;
5898 NEON_FP : S5;
5899 %}
5900
5901 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5902 %{
5903 single_instruction;
5904 src : S1(read);
5905 dst : S5(write);
5906 INS01 : ISS;
5907 NEON_FP : S5;
5908 %}
5909
5910 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5911 %{
5912 single_instruction;
5913 src : S1(read);
5914 dst : S5(write);
5915 INS01 : ISS;
5916 NEON_FP : S5;
5917 %}
5918
5919 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5920 %{
5921 single_instruction;
5922 src : S1(read);
5923 dst : S5(write);
5924 INS01 : ISS;
5925 NEON_FP : S5;
5926 %}
5927
5928 pipe_class fp_l2f(vRegF dst, iRegL src)
5929 %{
5930 single_instruction;
5931 src : S1(read);
5932 dst : S5(write);
5933 INS01 : ISS;
5934 NEON_FP : S5;
5935 %}
5936
5937 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5938 %{
5939 single_instruction;
5940 src : S1(read);
5941 dst : S5(write);
5942 INS01 : ISS;
5943 NEON_FP : S5;
5944 %}
5945
5946 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5947 %{
5948 single_instruction;
5949 src : S1(read);
5950 dst : S5(write);
5951 INS01 : ISS;
5952 NEON_FP : S5;
5953 %}
5954
5955 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5956 %{
5957 single_instruction;
5958 src : S1(read);
5959 dst : S5(write);
5960 INS01 : ISS;
5961 NEON_FP : S5;
5962 %}
5963
5964 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5965 %{
5966 single_instruction;
5967 src : S1(read);
5968 dst : S5(write);
5969 INS01 : ISS;
5970 NEON_FP : S5;
5971 %}
5972
5973 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5974 %{
5975 single_instruction;
5976 src1 : S1(read);
5977 src2 : S2(read);
5978 dst : S5(write);
5979 INS0 : ISS;
5980 NEON_FP : S5;
5981 %}
5982
5983 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
5984 %{
5985 single_instruction;
5986 src1 : S1(read);
5987 src2 : S2(read);
5988 dst : S5(write);
5989 INS0 : ISS;
5990 NEON_FP : S5;
5991 %}
5992
5993 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
5994 %{
5995 single_instruction;
5996 cr : S1(read);
5997 src1 : S1(read);
5998 src2 : S1(read);
5999 dst : S3(write);
6000 INS01 : ISS;
6001 NEON_FP : S3;
6002 %}
6003
6004 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6005 %{
6006 single_instruction;
6007 cr : S1(read);
6008 src1 : S1(read);
6009 src2 : S1(read);
6010 dst : S3(write);
6011 INS01 : ISS;
6012 NEON_FP : S3;
6013 %}
6014
6015 pipe_class fp_imm_s(vRegF dst)
6016 %{
6017 single_instruction;
6018 dst : S3(write);
6019 INS01 : ISS;
6020 NEON_FP : S3;
6021 %}
6022
6023 pipe_class fp_imm_d(vRegD dst)
6024 %{
6025 single_instruction;
6026 dst : S3(write);
6027 INS01 : ISS;
6028 NEON_FP : S3;
6029 %}
6030
6031 pipe_class fp_load_constant_s(vRegF dst)
6032 %{
6033 single_instruction;
6034 dst : S4(write);
6035 INS01 : ISS;
6036 NEON_FP : S4;
6037 %}
6038
6039 pipe_class fp_load_constant_d(vRegD dst)
6040 %{
6041 single_instruction;
6042 dst : S4(write);
6043 INS01 : ISS;
6044 NEON_FP : S4;
6045 %}
6046
6047 //------- Integer ALU operations --------------------------
6048
6049 // Integer ALU reg-reg operation
6050 // Operands needed in EX1, result generated in EX2
6051 // Eg. ADD x0, x1, x2
6052 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6053 %{
6054 single_instruction;
6055 dst : EX2(write);
6056 src1 : EX1(read);
6057 src2 : EX1(read);
6058 INS01 : ISS; // Dual issue as instruction 0 or 1
6059 ALU : EX2;
6060 %}
6061
6062 // Integer ALU reg-reg operation with constant shift
6063 // Shifted register must be available in LATE_ISS instead of EX1
6064 // Eg. ADD x0, x1, x2, LSL #2
6065 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6066 %{
6067 single_instruction;
6068 dst : EX2(write);
6069 src1 : EX1(read);
6070 src2 : ISS(read);
6071 INS01 : ISS;
6072 ALU : EX2;
6073 %}
6074
6075 // Integer ALU reg operation with constant shift
6076 // Eg. LSL x0, x1, #shift
6077 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6078 %{
6079 single_instruction;
6080 dst : EX2(write);
6081 src1 : ISS(read);
6082 INS01 : ISS;
6083 ALU : EX2;
6084 %}
6085
6086 // Integer ALU reg-reg operation with variable shift
6087 // Both operands must be available in LATE_ISS instead of EX1
6088 // Result is available in EX1 instead of EX2
6089 // Eg. LSLV x0, x1, x2
6090 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6091 %{
6092 single_instruction;
6093 dst : EX1(write);
6094 src1 : ISS(read);
6095 src2 : ISS(read);
6096 INS01 : ISS;
6097 ALU : EX1;
6098 %}
6099
6100 // Integer ALU reg-reg operation with extract
6101 // As for _vshift above, but result generated in EX2
6102 // Eg. EXTR x0, x1, x2, #N
6103 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6104 %{
6105 single_instruction;
6106 dst : EX2(write);
6107 src1 : ISS(read);
6108 src2 : ISS(read);
6109 INS1 : ISS; // Can only dual issue as Instruction 1
6110 ALU : EX1;
6111 %}
6112
6113 // Integer ALU reg operation
6114 // Eg. NEG x0, x1
6115 pipe_class ialu_reg(iRegI dst, iRegI src)
6116 %{
6117 single_instruction;
6118 dst : EX2(write);
6119 src : EX1(read);
6120 INS01 : ISS;
6121 ALU : EX2;
6122 %}
6123
6124 // Integer ALU reg mmediate operation
6125 // Eg. ADD x0, x1, #N
6126 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6127 %{
6128 single_instruction;
6129 dst : EX2(write);
6130 src1 : EX1(read);
6131 INS01 : ISS;
6132 ALU : EX2;
6133 %}
6134
6135 // Integer ALU immediate operation (no source operands)
6136 // Eg. MOV x0, #N
6137 pipe_class ialu_imm(iRegI dst)
6138 %{
6139 single_instruction;
6140 dst : EX1(write);
6141 INS01 : ISS;
6142 ALU : EX1;
6143 %}
6144
6145 //------- Compare operation -------------------------------
6146
6147 // Compare reg-reg
6148 // Eg. CMP x0, x1
6149 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6150 %{
6151 single_instruction;
6152 // fixed_latency(16);
6153 cr : EX2(write);
6154 op1 : EX1(read);
6155 op2 : EX1(read);
6156 INS01 : ISS;
6157 ALU : EX2;
6158 %}
6159
6160 // Compare reg-reg
6161 // Eg. CMP x0, #N
6162 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6163 %{
6164 single_instruction;
6165 // fixed_latency(16);
6166 cr : EX2(write);
6167 op1 : EX1(read);
6168 INS01 : ISS;
6169 ALU : EX2;
6170 %}
6171
6172 //------- Conditional instructions ------------------------
6173
6174 // Conditional no operands
6175 // Eg. CSINC x0, zr, zr, <cond>
6176 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6177 %{
6178 single_instruction;
6179 cr : EX1(read);
6180 dst : EX2(write);
6181 INS01 : ISS;
6182 ALU : EX2;
6183 %}
6184
6185 // Conditional 2 operand
6186 // EG. CSEL X0, X1, X2, <cond>
6187 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6188 %{
6189 single_instruction;
6190 cr : EX1(read);
6191 src1 : EX1(read);
6192 src2 : EX1(read);
6193 dst : EX2(write);
6194 INS01 : ISS;
6195 ALU : EX2;
6196 %}
6197
6198 // Conditional 2 operand
6199 // EG. CSEL X0, X1, X2, <cond>
6200 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6201 %{
6202 single_instruction;
6203 cr : EX1(read);
6204 src : EX1(read);
6205 dst : EX2(write);
6206 INS01 : ISS;
6207 ALU : EX2;
6208 %}
6209
6210 //------- Multiply pipeline operations --------------------
6211
6212 // Multiply reg-reg
6213 // Eg. MUL w0, w1, w2
6214 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6215 %{
6216 single_instruction;
6217 dst : WR(write);
6218 src1 : ISS(read);
6219 src2 : ISS(read);
6220 INS01 : ISS;
6221 MAC : WR;
6222 %}
6223
6224 // Multiply accumulate
6225 // Eg. MADD w0, w1, w2, w3
6226 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6227 %{
6228 single_instruction;
6229 dst : WR(write);
6230 src1 : ISS(read);
6231 src2 : ISS(read);
6232 src3 : ISS(read);
6233 INS01 : ISS;
6234 MAC : WR;
6235 %}
6236
6237 // Eg. MUL w0, w1, w2
6238 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6239 %{
6240 single_instruction;
6241 fixed_latency(3); // Maximum latency for 64 bit mul
6242 dst : WR(write);
6243 src1 : ISS(read);
6244 src2 : ISS(read);
6245 INS01 : ISS;
6246 MAC : WR;
6247 %}
6248
6249 // Multiply accumulate
6250 // Eg. MADD w0, w1, w2, w3
6251 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6252 %{
6253 single_instruction;
6254 fixed_latency(3); // Maximum latency for 64 bit mul
6255 dst : WR(write);
6256 src1 : ISS(read);
6257 src2 : ISS(read);
6258 src3 : ISS(read);
6259 INS01 : ISS;
6260 MAC : WR;
6261 %}
6262
6263 //------- Divide pipeline operations --------------------
6264
6265 // Eg. SDIV w0, w1, w2
6266 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6267 %{
6268 single_instruction;
6269 fixed_latency(8); // Maximum latency for 32 bit divide
6270 dst : WR(write);
6271 src1 : ISS(read);
6272 src2 : ISS(read);
6273 INS0 : ISS; // Can only dual issue as instruction 0
6274 DIV : WR;
6275 %}
6276
6277 // Eg. SDIV x0, x1, x2
6278 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6279 %{
6280 single_instruction;
6281 fixed_latency(16); // Maximum latency for 64 bit divide
6282 dst : WR(write);
6283 src1 : ISS(read);
6284 src2 : ISS(read);
6285 INS0 : ISS; // Can only dual issue as instruction 0
6286 DIV : WR;
6287 %}
6288
6289 //------- Load pipeline operations ------------------------
6290
6291 // Load - prefetch
6292 // Eg. PFRM <mem>
6293 pipe_class iload_prefetch(memory mem)
6294 %{
6295 single_instruction;
6296 mem : ISS(read);
6297 INS01 : ISS;
6298 LDST : WR;
6299 %}
6300
6301 // Load - reg, mem
6302 // Eg. LDR x0, <mem>
6303 pipe_class iload_reg_mem(iRegI dst, memory mem)
6304 %{
6305 single_instruction;
6306 dst : WR(write);
6307 mem : ISS(read);
6308 INS01 : ISS;
6309 LDST : WR;
6310 %}
6311
6312 // Load - reg, reg
6313 // Eg. LDR x0, [sp, x1]
6314 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6315 %{
6316 single_instruction;
6317 dst : WR(write);
6318 src : ISS(read);
6319 INS01 : ISS;
6320 LDST : WR;
6321 %}
6322
6323 //------- Store pipeline operations -----------------------
6324
6325 // Store - zr, mem
6326 // Eg. STR zr, <mem>
6327 pipe_class istore_mem(memory mem)
6328 %{
6329 single_instruction;
6330 mem : ISS(read);
6331 INS01 : ISS;
6332 LDST : WR;
6333 %}
6334
6335 // Store - reg, mem
6336 // Eg. STR x0, <mem>
6337 pipe_class istore_reg_mem(iRegI src, memory mem)
6338 %{
6339 single_instruction;
6340 mem : ISS(read);
6341 src : EX2(read);
6342 INS01 : ISS;
6343 LDST : WR;
6344 %}
6345
6346 // Store - reg, reg
6347 // Eg. STR x0, [sp, x1]
6348 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6349 %{
6350 single_instruction;
6351 dst : ISS(read);
6352 src : EX2(read);
6353 INS01 : ISS;
6354 LDST : WR;
6355 %}
6356
6357 //------- Store pipeline operations -----------------------
6358
6359 // Branch
6360 pipe_class pipe_branch()
6361 %{
6362 single_instruction;
6363 INS01 : ISS;
6364 BRANCH : EX1;
6365 %}
6366
6367 // Conditional branch
6368 pipe_class pipe_branch_cond(rFlagsReg cr)
6369 %{
6370 single_instruction;
6371 cr : EX1(read);
6372 INS01 : ISS;
6373 BRANCH : EX1;
6374 %}
6375
6376 // Compare & Branch
6377 // EG. CBZ/CBNZ
6378 pipe_class pipe_cmp_branch(iRegI op1)
6379 %{
6380 single_instruction;
6381 op1 : EX1(read);
6382 INS01 : ISS;
6383 BRANCH : EX1;
6384 %}
6385
6386 //------- Synchronisation operations ----------------------
6387
6388 // Any operation requiring serialization.
6389 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6390 pipe_class pipe_serial()
6391 %{
6392 single_instruction;
6393 force_serialization;
6394 fixed_latency(16);
6395 INS01 : ISS(2); // Cannot dual issue with any other instruction
6396 LDST : WR;
6397 %}
6398
6399 // Generic big/slow expanded idiom - also serialized
6400 pipe_class pipe_slow()
6401 %{
6402 instruction_count(10);
6403 multiple_bundles;
6404 force_serialization;
6405 fixed_latency(16);
6406 INS01 : ISS(2); // Cannot dual issue with any other instruction
6407 LDST : WR;
6408 %}
6409
6410 // Empty pipeline class
6411 pipe_class pipe_class_empty()
6412 %{
6413 single_instruction;
6414 fixed_latency(0);
6415 %}
6416
6417 // Default pipeline class.
6418 pipe_class pipe_class_default()
6419 %{
6420 single_instruction;
6421 fixed_latency(2);
6422 %}
6423
6424 // Pipeline class for compares.
6425 pipe_class pipe_class_compare()
6426 %{
6427 single_instruction;
6428 fixed_latency(16);
6429 %}
6430
6431 // Pipeline class for memory operations.
6432 pipe_class pipe_class_memory()
6433 %{
6434 single_instruction;
6435 fixed_latency(16);
6436 %}
6437
6438 // Pipeline class for call.
6439 pipe_class pipe_class_call()
6440 %{
6441 single_instruction;
6442 fixed_latency(100);
6443 %}
6444
6445 // Define the class for the Nop node.
6446 define %{
6447 MachNop = pipe_class_empty;
6448 %}
6449
6450 %}
6451 //----------INSTRUCTIONS-------------------------------------------------------
6452 //
6453 // match -- States which machine-independent subtree may be replaced
6454 // by this instruction.
6455 // ins_cost -- The estimated cost of this instruction is used by instruction
6456 // selection to identify a minimum cost tree of machine
6457 // instructions that matches a tree of machine-independent
6458 // instructions.
6459 // format -- A string providing the disassembly for this instruction.
6460 // The value of an instruction's operand may be inserted
6461 // by referring to it with a '$' prefix.
6462 // opcode -- Three instruction opcodes may be provided. These are referred
6463 // to within an encode class as $primary, $secondary, and $tertiary
6464 // rrspectively. The primary opcode is commonly used to
6465 // indicate the type of machine instruction, while secondary
6466 // and tertiary are often used for prefix options or addressing
6467 // modes.
6468 // ins_encode -- A list of encode classes with parameters. The encode class
6469 // name must have been defined in an 'enc_class' specification
6470 // in the encode section of the architecture description.
6471
6472 // ============================================================================
6473 // Memory (Load/Store) Instructions
6474
6475 // Load Instructions
6476
6477 // Load Byte (8 bit signed)
6478 instruct loadB(iRegINoSp dst, memory1 mem)
6479 %{
6480 match(Set dst (LoadB mem));
6481 predicate(!needs_acquiring_load(n));
6482
6483 ins_cost(4 * INSN_COST);
6484 format %{ "ldrsbw $dst, $mem\t# byte" %}
6485
6486 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6487
6488 ins_pipe(iload_reg_mem);
6489 %}
6490
6491 // Load Byte (8 bit signed) into long
6492 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6493 %{
6494 match(Set dst (ConvI2L (LoadB mem)));
6495 predicate(!needs_acquiring_load(n->in(1)));
6496
6497 ins_cost(4 * INSN_COST);
6498 format %{ "ldrsb $dst, $mem\t# byte" %}
6499
6500 ins_encode(aarch64_enc_ldrsb(dst, mem));
6501
6502 ins_pipe(iload_reg_mem);
6503 %}
6504
6505 // Load Byte (8 bit unsigned)
6506 instruct loadUB(iRegINoSp dst, memory1 mem)
6507 %{
6508 match(Set dst (LoadUB mem));
6509 predicate(!needs_acquiring_load(n));
6510
6511 ins_cost(4 * INSN_COST);
6512 format %{ "ldrbw $dst, $mem\t# byte" %}
6513
6514 ins_encode(aarch64_enc_ldrb(dst, mem));
6515
6516 ins_pipe(iload_reg_mem);
6517 %}
6518
6519 // Load Byte (8 bit unsigned) into long
6520 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6521 %{
6522 match(Set dst (ConvI2L (LoadUB mem)));
6523 predicate(!needs_acquiring_load(n->in(1)));
6524
6525 ins_cost(4 * INSN_COST);
6526 format %{ "ldrb $dst, $mem\t# byte" %}
6527
6528 ins_encode(aarch64_enc_ldrb(dst, mem));
6529
6530 ins_pipe(iload_reg_mem);
6531 %}
6532
6533 // Load Short (16 bit signed)
6534 instruct loadS(iRegINoSp dst, memory2 mem)
6535 %{
6536 match(Set dst (LoadS mem));
6537 predicate(!needs_acquiring_load(n));
6538
6539 ins_cost(4 * INSN_COST);
6540 format %{ "ldrshw $dst, $mem\t# short" %}
6541
6542 ins_encode(aarch64_enc_ldrshw(dst, mem));
6543
6544 ins_pipe(iload_reg_mem);
6545 %}
6546
6547 // Load Short (16 bit signed) into long
6548 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6549 %{
6550 match(Set dst (ConvI2L (LoadS mem)));
6551 predicate(!needs_acquiring_load(n->in(1)));
6552
6553 ins_cost(4 * INSN_COST);
6554 format %{ "ldrsh $dst, $mem\t# short" %}
6555
6556 ins_encode(aarch64_enc_ldrsh(dst, mem));
6557
6558 ins_pipe(iload_reg_mem);
6559 %}
6560
6561 // Load Char (16 bit unsigned)
6562 instruct loadUS(iRegINoSp dst, memory2 mem)
6563 %{
6564 match(Set dst (LoadUS mem));
6565 predicate(!needs_acquiring_load(n));
6566
6567 ins_cost(4 * INSN_COST);
6568 format %{ "ldrh $dst, $mem\t# short" %}
6569
6570 ins_encode(aarch64_enc_ldrh(dst, mem));
6571
6572 ins_pipe(iload_reg_mem);
6573 %}
6574
6575 // Load Short/Char (16 bit unsigned) into long
6576 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6577 %{
6578 match(Set dst (ConvI2L (LoadUS mem)));
6579 predicate(!needs_acquiring_load(n->in(1)));
6580
6581 ins_cost(4 * INSN_COST);
6582 format %{ "ldrh $dst, $mem\t# short" %}
6583
6584 ins_encode(aarch64_enc_ldrh(dst, mem));
6585
6586 ins_pipe(iload_reg_mem);
6587 %}
6588
6589 // Load Integer (32 bit signed)
6590 instruct loadI(iRegINoSp dst, memory4 mem)
6591 %{
6592 match(Set dst (LoadI mem));
6593 predicate(!needs_acquiring_load(n));
6594
6595 ins_cost(4 * INSN_COST);
6596 format %{ "ldrw $dst, $mem\t# int" %}
6597
6598 ins_encode(aarch64_enc_ldrw(dst, mem));
6599
6600 ins_pipe(iload_reg_mem);
6601 %}
6602
6603 // Load Integer (32 bit signed) into long
6604 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6605 %{
6606 match(Set dst (ConvI2L (LoadI mem)));
6607 predicate(!needs_acquiring_load(n->in(1)));
6608
6609 ins_cost(4 * INSN_COST);
6610 format %{ "ldrsw $dst, $mem\t# int" %}
6611
6612 ins_encode(aarch64_enc_ldrsw(dst, mem));
6613
6614 ins_pipe(iload_reg_mem);
6615 %}
6616
6617 // Load Integer (32 bit unsigned) into long
6618 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6619 %{
6620 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6621 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6622
6623 ins_cost(4 * INSN_COST);
6624 format %{ "ldrw $dst, $mem\t# int" %}
6625
6626 ins_encode(aarch64_enc_ldrw(dst, mem));
6627
6628 ins_pipe(iload_reg_mem);
6629 %}
6630
6631 // Load Long (64 bit signed)
6632 instruct loadL(iRegLNoSp dst, memory8 mem)
6633 %{
6634 match(Set dst (LoadL mem));
6635 predicate(!needs_acquiring_load(n));
6636
6637 ins_cost(4 * INSN_COST);
6638 format %{ "ldr $dst, $mem\t# int" %}
6639
6640 ins_encode(aarch64_enc_ldr(dst, mem));
6641
6642 ins_pipe(iload_reg_mem);
6643 %}
6644
6645 // Load Range
6646 instruct loadRange(iRegINoSp dst, memory4 mem)
6647 %{
6648 match(Set dst (LoadRange mem));
6649
6650 ins_cost(4 * INSN_COST);
6651 format %{ "ldrw $dst, $mem\t# range" %}
6652
6653 ins_encode(aarch64_enc_ldrw(dst, mem));
6654
6655 ins_pipe(iload_reg_mem);
6656 %}
6657
6658 // Load Pointer
6659 instruct loadP(iRegPNoSp dst, memory8 mem)
6660 %{
6661 match(Set dst (LoadP mem));
6662 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6663
6664 ins_cost(4 * INSN_COST);
6665 format %{ "ldr $dst, $mem\t# ptr" %}
6666
6667 ins_encode(aarch64_enc_ldr(dst, mem));
6668
6669 ins_pipe(iload_reg_mem);
6670 %}
6671
6672 // Load Compressed Pointer
6673 instruct loadN(iRegNNoSp dst, memory4 mem)
6674 %{
6675 match(Set dst (LoadN mem));
6676 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6677
6678 ins_cost(4 * INSN_COST);
6679 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6680
6681 ins_encode(aarch64_enc_ldrw(dst, mem));
6682
6683 ins_pipe(iload_reg_mem);
6684 %}
6685
6686 // Load Klass Pointer
6687 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6688 %{
6689 match(Set dst (LoadKlass mem));
6690 predicate(!needs_acquiring_load(n));
6691
6692 ins_cost(4 * INSN_COST);
6693 format %{ "ldr $dst, $mem\t# class" %}
6694
6695 ins_encode(aarch64_enc_ldr(dst, mem));
6696
6697 ins_pipe(iload_reg_mem);
6698 %}
6699
6700 // Load Narrow Klass Pointer
6701 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6702 %{
6703 match(Set dst (LoadNKlass mem));
6704 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6705
6706 ins_cost(4 * INSN_COST);
6707 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6708
6709 ins_encode(aarch64_enc_ldrw(dst, mem));
6710
6711 ins_pipe(iload_reg_mem);
6712 %}
6713
6714 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory_noindex mem)
6715 %{
6716 match(Set dst (LoadNKlass mem));
6717 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6718
6719 ins_cost(4 * INSN_COST);
6720 format %{
6721 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6722 "lsrw $dst, $dst, markWord::klass_shift"
6723 %}
6724 ins_encode %{
6725 assert($mem$$index$$Register == noreg, "must not have indexed address");
6726 // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
6727 // obj-start, so that we can load from the object's mark-word instead.
6728 __ ldrw($dst$$Register, Address($mem$$base$$Register, $mem$$disp - Type::klass_offset()));
6729 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift);
6730 %}
6731 ins_pipe(iload_reg_mem);
6732 %}
6733
6734 // Load Float
6735 instruct loadF(vRegF dst, memory4 mem)
6736 %{
6737 match(Set dst (LoadF mem));
6738 predicate(!needs_acquiring_load(n));
6739
6740 ins_cost(4 * INSN_COST);
6741 format %{ "ldrs $dst, $mem\t# float" %}
6742
6743 ins_encode( aarch64_enc_ldrs(dst, mem) );
6744
6745 ins_pipe(pipe_class_memory);
6746 %}
6747
6748 // Load Double
6749 instruct loadD(vRegD dst, memory8 mem)
6750 %{
6751 match(Set dst (LoadD mem));
6752 predicate(!needs_acquiring_load(n));
6753
6754 ins_cost(4 * INSN_COST);
6755 format %{ "ldrd $dst, $mem\t# double" %}
6756
6757 ins_encode( aarch64_enc_ldrd(dst, mem) );
6758
6759 ins_pipe(pipe_class_memory);
6760 %}
6761
6762
6763 // Load Int Constant
6764 instruct loadConI(iRegINoSp dst, immI src)
6765 %{
6766 match(Set dst src);
6767
6768 ins_cost(INSN_COST);
6769 format %{ "mov $dst, $src\t# int" %}
6770
6771 ins_encode( aarch64_enc_movw_imm(dst, src) );
6772
6773 ins_pipe(ialu_imm);
6774 %}
6775
6776 // Load Long Constant
6777 instruct loadConL(iRegLNoSp dst, immL src)
6778 %{
6779 match(Set dst src);
6780
6781 ins_cost(INSN_COST);
6782 format %{ "mov $dst, $src\t# long" %}
6783
6784 ins_encode( aarch64_enc_mov_imm(dst, src) );
6785
6786 ins_pipe(ialu_imm);
6787 %}
6788
6789 // Load Pointer Constant
6790
6791 instruct loadConP(iRegPNoSp dst, immP con)
6792 %{
6793 match(Set dst con);
6794
6795 ins_cost(INSN_COST * 4);
6796 format %{
6797 "mov $dst, $con\t# ptr\n\t"
6798 %}
6799
6800 ins_encode(aarch64_enc_mov_p(dst, con));
6801
6802 ins_pipe(ialu_imm);
6803 %}
6804
6805 // Load Null Pointer Constant
6806
6807 instruct loadConP0(iRegPNoSp dst, immP0 con)
6808 %{
6809 match(Set dst con);
6810
6811 ins_cost(INSN_COST);
6812 format %{ "mov $dst, $con\t# nullptr ptr" %}
6813
6814 ins_encode(aarch64_enc_mov_p0(dst, con));
6815
6816 ins_pipe(ialu_imm);
6817 %}
6818
6819 // Load Pointer Constant One
6820
6821 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6822 %{
6823 match(Set dst con);
6824
6825 ins_cost(INSN_COST);
6826 format %{ "mov $dst, $con\t# nullptr ptr" %}
6827
6828 ins_encode(aarch64_enc_mov_p1(dst, con));
6829
6830 ins_pipe(ialu_imm);
6831 %}
6832
6833 // Load Byte Map Base Constant
6834
6835 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6836 %{
6837 match(Set dst con);
6838
6839 ins_cost(INSN_COST);
6840 format %{ "adr $dst, $con\t# Byte Map Base" %}
6841
6842 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6843
6844 ins_pipe(ialu_imm);
6845 %}
6846
6847 // Load Narrow Pointer Constant
6848
6849 instruct loadConN(iRegNNoSp dst, immN con)
6850 %{
6851 match(Set dst con);
6852
6853 ins_cost(INSN_COST * 4);
6854 format %{ "mov $dst, $con\t# compressed ptr" %}
6855
6856 ins_encode(aarch64_enc_mov_n(dst, con));
6857
6858 ins_pipe(ialu_imm);
6859 %}
6860
6861 // Load Narrow Null Pointer Constant
6862
6863 instruct loadConN0(iRegNNoSp dst, immN0 con)
6864 %{
6865 match(Set dst con);
6866
6867 ins_cost(INSN_COST);
6868 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6869
6870 ins_encode(aarch64_enc_mov_n0(dst, con));
6871
6872 ins_pipe(ialu_imm);
6873 %}
6874
6875 // Load Narrow Klass Constant
6876
6877 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6878 %{
6879 match(Set dst con);
6880
6881 ins_cost(INSN_COST);
6882 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6883
6884 ins_encode(aarch64_enc_mov_nk(dst, con));
6885
6886 ins_pipe(ialu_imm);
6887 %}
6888
6889 // Load Packed Float Constant
6890
6891 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6892 match(Set dst con);
6893 ins_cost(INSN_COST * 4);
6894 format %{ "fmovs $dst, $con"%}
6895 ins_encode %{
6896 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6897 %}
6898
6899 ins_pipe(fp_imm_s);
6900 %}
6901
6902 // Load Float Constant
6903
6904 instruct loadConF(vRegF dst, immF con) %{
6905 match(Set dst con);
6906
6907 ins_cost(INSN_COST * 4);
6908
6909 format %{
6910 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6911 %}
6912
6913 ins_encode %{
6914 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6915 %}
6916
6917 ins_pipe(fp_load_constant_s);
6918 %}
6919
6920 // Load Packed Double Constant
6921
6922 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6923 match(Set dst con);
6924 ins_cost(INSN_COST);
6925 format %{ "fmovd $dst, $con"%}
6926 ins_encode %{
6927 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6928 %}
6929
6930 ins_pipe(fp_imm_d);
6931 %}
6932
6933 // Load Double Constant
6934
6935 instruct loadConD(vRegD dst, immD con) %{
6936 match(Set dst con);
6937
6938 ins_cost(INSN_COST * 5);
6939 format %{
6940 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6941 %}
6942
6943 ins_encode %{
6944 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6945 %}
6946
6947 ins_pipe(fp_load_constant_d);
6948 %}
6949
6950 // Store Instructions
6951
6952 // Store Byte
6953 instruct storeB(iRegIorL2I src, memory1 mem)
6954 %{
6955 match(Set mem (StoreB mem src));
6956 predicate(!needs_releasing_store(n));
6957
6958 ins_cost(INSN_COST);
6959 format %{ "strb $src, $mem\t# byte" %}
6960
6961 ins_encode(aarch64_enc_strb(src, mem));
6962
6963 ins_pipe(istore_reg_mem);
6964 %}
6965
6966
6967 instruct storeimmB0(immI0 zero, memory1 mem)
6968 %{
6969 match(Set mem (StoreB mem zero));
6970 predicate(!needs_releasing_store(n));
6971
6972 ins_cost(INSN_COST);
6973 format %{ "strb rscractch2, $mem\t# byte" %}
6974
6975 ins_encode(aarch64_enc_strb0(mem));
6976
6977 ins_pipe(istore_mem);
6978 %}
6979
6980 // Store Char/Short
6981 instruct storeC(iRegIorL2I src, memory2 mem)
6982 %{
6983 match(Set mem (StoreC mem src));
6984 predicate(!needs_releasing_store(n));
6985
6986 ins_cost(INSN_COST);
6987 format %{ "strh $src, $mem\t# short" %}
6988
6989 ins_encode(aarch64_enc_strh(src, mem));
6990
6991 ins_pipe(istore_reg_mem);
6992 %}
6993
6994 instruct storeimmC0(immI0 zero, memory2 mem)
6995 %{
6996 match(Set mem (StoreC mem zero));
6997 predicate(!needs_releasing_store(n));
6998
6999 ins_cost(INSN_COST);
7000 format %{ "strh zr, $mem\t# short" %}
7001
7002 ins_encode(aarch64_enc_strh0(mem));
7003
7004 ins_pipe(istore_mem);
7005 %}
7006
7007 // Store Integer
7008
7009 instruct storeI(iRegIorL2I src, memory4 mem)
7010 %{
7011 match(Set mem(StoreI mem src));
7012 predicate(!needs_releasing_store(n));
7013
7014 ins_cost(INSN_COST);
7015 format %{ "strw $src, $mem\t# int" %}
7016
7017 ins_encode(aarch64_enc_strw(src, mem));
7018
7019 ins_pipe(istore_reg_mem);
7020 %}
7021
7022 instruct storeimmI0(immI0 zero, memory4 mem)
7023 %{
7024 match(Set mem(StoreI mem zero));
7025 predicate(!needs_releasing_store(n));
7026
7027 ins_cost(INSN_COST);
7028 format %{ "strw zr, $mem\t# int" %}
7029
7030 ins_encode(aarch64_enc_strw0(mem));
7031
7032 ins_pipe(istore_mem);
7033 %}
7034
7035 // Store Long (64 bit signed)
7036 instruct storeL(iRegL src, memory8 mem)
7037 %{
7038 match(Set mem (StoreL mem src));
7039 predicate(!needs_releasing_store(n));
7040
7041 ins_cost(INSN_COST);
7042 format %{ "str $src, $mem\t# int" %}
7043
7044 ins_encode(aarch64_enc_str(src, mem));
7045
7046 ins_pipe(istore_reg_mem);
7047 %}
7048
7049 // Store Long (64 bit signed)
7050 instruct storeimmL0(immL0 zero, memory8 mem)
7051 %{
7052 match(Set mem (StoreL mem zero));
7053 predicate(!needs_releasing_store(n));
7054
7055 ins_cost(INSN_COST);
7056 format %{ "str zr, $mem\t# int" %}
7057
7058 ins_encode(aarch64_enc_str0(mem));
7059
7060 ins_pipe(istore_mem);
7061 %}
7062
7063 // Store Pointer
7064 instruct storeP(iRegP src, memory8 mem)
7065 %{
7066 match(Set mem (StoreP mem src));
7067 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7068
7069 ins_cost(INSN_COST);
7070 format %{ "str $src, $mem\t# ptr" %}
7071
7072 ins_encode(aarch64_enc_str(src, mem));
7073
7074 ins_pipe(istore_reg_mem);
7075 %}
7076
7077 // Store Pointer
7078 instruct storeimmP0(immP0 zero, memory8 mem)
7079 %{
7080 match(Set mem (StoreP mem zero));
7081 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7082
7083 ins_cost(INSN_COST);
7084 format %{ "str zr, $mem\t# ptr" %}
7085
7086 ins_encode(aarch64_enc_str0(mem));
7087
7088 ins_pipe(istore_mem);
7089 %}
7090
7091 // Store Compressed Pointer
7092 instruct storeN(iRegN src, memory4 mem)
7093 %{
7094 match(Set mem (StoreN mem src));
7095 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7096
7097 ins_cost(INSN_COST);
7098 format %{ "strw $src, $mem\t# compressed ptr" %}
7099
7100 ins_encode(aarch64_enc_strw(src, mem));
7101
7102 ins_pipe(istore_reg_mem);
7103 %}
7104
7105 instruct storeImmN0(immN0 zero, memory4 mem)
7106 %{
7107 match(Set mem (StoreN mem zero));
7108 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7109
7110 ins_cost(INSN_COST);
7111 format %{ "strw zr, $mem\t# compressed ptr" %}
7112
7113 ins_encode(aarch64_enc_strw0(mem));
7114
7115 ins_pipe(istore_mem);
7116 %}
7117
7118 // Store Float
7119 instruct storeF(vRegF src, memory4 mem)
7120 %{
7121 match(Set mem (StoreF mem src));
7122 predicate(!needs_releasing_store(n));
7123
7124 ins_cost(INSN_COST);
7125 format %{ "strs $src, $mem\t# float" %}
7126
7127 ins_encode( aarch64_enc_strs(src, mem) );
7128
7129 ins_pipe(pipe_class_memory);
7130 %}
7131
7132 // TODO
7133 // implement storeImmF0 and storeFImmPacked
7134
7135 // Store Double
7136 instruct storeD(vRegD src, memory8 mem)
7137 %{
7138 match(Set mem (StoreD mem src));
7139 predicate(!needs_releasing_store(n));
7140
7141 ins_cost(INSN_COST);
7142 format %{ "strd $src, $mem\t# double" %}
7143
7144 ins_encode( aarch64_enc_strd(src, mem) );
7145
7146 ins_pipe(pipe_class_memory);
7147 %}
7148
7149 // Store Compressed Klass Pointer
7150 instruct storeNKlass(iRegN src, memory4 mem)
7151 %{
7152 predicate(!needs_releasing_store(n));
7153 match(Set mem (StoreNKlass mem src));
7154
7155 ins_cost(INSN_COST);
7156 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7157
7158 ins_encode(aarch64_enc_strw(src, mem));
7159
7160 ins_pipe(istore_reg_mem);
7161 %}
7162
7163 // TODO
7164 // implement storeImmD0 and storeDImmPacked
7165
7166 // prefetch instructions
7167 // Must be safe to execute with invalid address (cannot fault).
7168
7169 instruct prefetchalloc( memory8 mem ) %{
7170 match(PrefetchAllocation mem);
7171
7172 ins_cost(INSN_COST);
7173 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7174
7175 ins_encode( aarch64_enc_prefetchw(mem) );
7176
7177 ins_pipe(iload_prefetch);
7178 %}
7179
7180 // ---------------- volatile loads and stores ----------------
7181
7182 // Load Byte (8 bit signed)
7183 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7184 %{
7185 match(Set dst (LoadB mem));
7186
7187 ins_cost(VOLATILE_REF_COST);
7188 format %{ "ldarsb $dst, $mem\t# byte" %}
7189
7190 ins_encode(aarch64_enc_ldarsb(dst, mem));
7191
7192 ins_pipe(pipe_serial);
7193 %}
7194
7195 // Load Byte (8 bit signed) into long
7196 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7197 %{
7198 match(Set dst (ConvI2L (LoadB mem)));
7199
7200 ins_cost(VOLATILE_REF_COST);
7201 format %{ "ldarsb $dst, $mem\t# byte" %}
7202
7203 ins_encode(aarch64_enc_ldarsb(dst, mem));
7204
7205 ins_pipe(pipe_serial);
7206 %}
7207
7208 // Load Byte (8 bit unsigned)
7209 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7210 %{
7211 match(Set dst (LoadUB mem));
7212
7213 ins_cost(VOLATILE_REF_COST);
7214 format %{ "ldarb $dst, $mem\t# byte" %}
7215
7216 ins_encode(aarch64_enc_ldarb(dst, mem));
7217
7218 ins_pipe(pipe_serial);
7219 %}
7220
7221 // Load Byte (8 bit unsigned) into long
7222 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7223 %{
7224 match(Set dst (ConvI2L (LoadUB mem)));
7225
7226 ins_cost(VOLATILE_REF_COST);
7227 format %{ "ldarb $dst, $mem\t# byte" %}
7228
7229 ins_encode(aarch64_enc_ldarb(dst, mem));
7230
7231 ins_pipe(pipe_serial);
7232 %}
7233
7234 // Load Short (16 bit signed)
7235 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7236 %{
7237 match(Set dst (LoadS mem));
7238
7239 ins_cost(VOLATILE_REF_COST);
7240 format %{ "ldarshw $dst, $mem\t# short" %}
7241
7242 ins_encode(aarch64_enc_ldarshw(dst, mem));
7243
7244 ins_pipe(pipe_serial);
7245 %}
7246
7247 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7248 %{
7249 match(Set dst (LoadUS mem));
7250
7251 ins_cost(VOLATILE_REF_COST);
7252 format %{ "ldarhw $dst, $mem\t# short" %}
7253
7254 ins_encode(aarch64_enc_ldarhw(dst, mem));
7255
7256 ins_pipe(pipe_serial);
7257 %}
7258
7259 // Load Short/Char (16 bit unsigned) into long
7260 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7261 %{
7262 match(Set dst (ConvI2L (LoadUS mem)));
7263
7264 ins_cost(VOLATILE_REF_COST);
7265 format %{ "ldarh $dst, $mem\t# short" %}
7266
7267 ins_encode(aarch64_enc_ldarh(dst, mem));
7268
7269 ins_pipe(pipe_serial);
7270 %}
7271
7272 // Load Short/Char (16 bit signed) into long
7273 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7274 %{
7275 match(Set dst (ConvI2L (LoadS mem)));
7276
7277 ins_cost(VOLATILE_REF_COST);
7278 format %{ "ldarh $dst, $mem\t# short" %}
7279
7280 ins_encode(aarch64_enc_ldarsh(dst, mem));
7281
7282 ins_pipe(pipe_serial);
7283 %}
7284
7285 // Load Integer (32 bit signed)
7286 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7287 %{
7288 match(Set dst (LoadI mem));
7289
7290 ins_cost(VOLATILE_REF_COST);
7291 format %{ "ldarw $dst, $mem\t# int" %}
7292
7293 ins_encode(aarch64_enc_ldarw(dst, mem));
7294
7295 ins_pipe(pipe_serial);
7296 %}
7297
7298 // Load Integer (32 bit unsigned) into long
7299 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7300 %{
7301 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7302
7303 ins_cost(VOLATILE_REF_COST);
7304 format %{ "ldarw $dst, $mem\t# int" %}
7305
7306 ins_encode(aarch64_enc_ldarw(dst, mem));
7307
7308 ins_pipe(pipe_serial);
7309 %}
7310
7311 // Load Long (64 bit signed)
7312 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7313 %{
7314 match(Set dst (LoadL mem));
7315
7316 ins_cost(VOLATILE_REF_COST);
7317 format %{ "ldar $dst, $mem\t# int" %}
7318
7319 ins_encode(aarch64_enc_ldar(dst, mem));
7320
7321 ins_pipe(pipe_serial);
7322 %}
7323
7324 // Load Pointer
7325 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7326 %{
7327 match(Set dst (LoadP mem));
7328 predicate(n->as_Load()->barrier_data() == 0);
7329
7330 ins_cost(VOLATILE_REF_COST);
7331 format %{ "ldar $dst, $mem\t# ptr" %}
7332
7333 ins_encode(aarch64_enc_ldar(dst, mem));
7334
7335 ins_pipe(pipe_serial);
7336 %}
7337
7338 // Load Compressed Pointer
7339 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7340 %{
7341 match(Set dst (LoadN mem));
7342 predicate(n->as_Load()->barrier_data() == 0);
7343
7344 ins_cost(VOLATILE_REF_COST);
7345 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7346
7347 ins_encode(aarch64_enc_ldarw(dst, mem));
7348
7349 ins_pipe(pipe_serial);
7350 %}
7351
7352 // Load Float
7353 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7354 %{
7355 match(Set dst (LoadF mem));
7356
7357 ins_cost(VOLATILE_REF_COST);
7358 format %{ "ldars $dst, $mem\t# float" %}
7359
7360 ins_encode( aarch64_enc_fldars(dst, mem) );
7361
7362 ins_pipe(pipe_serial);
7363 %}
7364
7365 // Load Double
7366 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7367 %{
7368 match(Set dst (LoadD mem));
7369
7370 ins_cost(VOLATILE_REF_COST);
7371 format %{ "ldard $dst, $mem\t# double" %}
7372
7373 ins_encode( aarch64_enc_fldard(dst, mem) );
7374
7375 ins_pipe(pipe_serial);
7376 %}
7377
7378 // Store Byte
7379 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7380 %{
7381 match(Set mem (StoreB mem src));
7382
7383 ins_cost(VOLATILE_REF_COST);
7384 format %{ "stlrb $src, $mem\t# byte" %}
7385
7386 ins_encode(aarch64_enc_stlrb(src, mem));
7387
7388 ins_pipe(pipe_class_memory);
7389 %}
7390
7391 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7392 %{
7393 match(Set mem (StoreB mem zero));
7394
7395 ins_cost(VOLATILE_REF_COST);
7396 format %{ "stlrb zr, $mem\t# byte" %}
7397
7398 ins_encode(aarch64_enc_stlrb0(mem));
7399
7400 ins_pipe(pipe_class_memory);
7401 %}
7402
7403 // Store Char/Short
7404 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7405 %{
7406 match(Set mem (StoreC mem src));
7407
7408 ins_cost(VOLATILE_REF_COST);
7409 format %{ "stlrh $src, $mem\t# short" %}
7410
7411 ins_encode(aarch64_enc_stlrh(src, mem));
7412
7413 ins_pipe(pipe_class_memory);
7414 %}
7415
7416 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7417 %{
7418 match(Set mem (StoreC mem zero));
7419
7420 ins_cost(VOLATILE_REF_COST);
7421 format %{ "stlrh zr, $mem\t# short" %}
7422
7423 ins_encode(aarch64_enc_stlrh0(mem));
7424
7425 ins_pipe(pipe_class_memory);
7426 %}
7427
7428 // Store Integer
7429
7430 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7431 %{
7432 match(Set mem(StoreI mem src));
7433
7434 ins_cost(VOLATILE_REF_COST);
7435 format %{ "stlrw $src, $mem\t# int" %}
7436
7437 ins_encode(aarch64_enc_stlrw(src, mem));
7438
7439 ins_pipe(pipe_class_memory);
7440 %}
7441
7442 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7443 %{
7444 match(Set mem(StoreI mem zero));
7445
7446 ins_cost(VOLATILE_REF_COST);
7447 format %{ "stlrw zr, $mem\t# int" %}
7448
7449 ins_encode(aarch64_enc_stlrw0(mem));
7450
7451 ins_pipe(pipe_class_memory);
7452 %}
7453
7454 // Store Long (64 bit signed)
7455 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7456 %{
7457 match(Set mem (StoreL mem src));
7458
7459 ins_cost(VOLATILE_REF_COST);
7460 format %{ "stlr $src, $mem\t# int" %}
7461
7462 ins_encode(aarch64_enc_stlr(src, mem));
7463
7464 ins_pipe(pipe_class_memory);
7465 %}
7466
7467 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7468 %{
7469 match(Set mem (StoreL mem zero));
7470
7471 ins_cost(VOLATILE_REF_COST);
7472 format %{ "stlr zr, $mem\t# int" %}
7473
7474 ins_encode(aarch64_enc_stlr0(mem));
7475
7476 ins_pipe(pipe_class_memory);
7477 %}
7478
7479 // Store Pointer
7480 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7481 %{
7482 match(Set mem (StoreP mem src));
7483 predicate(n->as_Store()->barrier_data() == 0);
7484
7485 ins_cost(VOLATILE_REF_COST);
7486 format %{ "stlr $src, $mem\t# ptr" %}
7487
7488 ins_encode(aarch64_enc_stlr(src, mem));
7489
7490 ins_pipe(pipe_class_memory);
7491 %}
7492
7493 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7494 %{
7495 match(Set mem (StoreP mem zero));
7496 predicate(n->as_Store()->barrier_data() == 0);
7497
7498 ins_cost(VOLATILE_REF_COST);
7499 format %{ "stlr zr, $mem\t# ptr" %}
7500
7501 ins_encode(aarch64_enc_stlr0(mem));
7502
7503 ins_pipe(pipe_class_memory);
7504 %}
7505
7506 // Store Compressed Pointer
7507 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7508 %{
7509 match(Set mem (StoreN mem src));
7510 predicate(n->as_Store()->barrier_data() == 0);
7511
7512 ins_cost(VOLATILE_REF_COST);
7513 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7514
7515 ins_encode(aarch64_enc_stlrw(src, mem));
7516
7517 ins_pipe(pipe_class_memory);
7518 %}
7519
7520 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7521 %{
7522 match(Set mem (StoreN mem zero));
7523 predicate(n->as_Store()->barrier_data() == 0);
7524
7525 ins_cost(VOLATILE_REF_COST);
7526 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7527
7528 ins_encode(aarch64_enc_stlrw0(mem));
7529
7530 ins_pipe(pipe_class_memory);
7531 %}
7532
7533 // Store Float
7534 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7535 %{
7536 match(Set mem (StoreF mem src));
7537
7538 ins_cost(VOLATILE_REF_COST);
7539 format %{ "stlrs $src, $mem\t# float" %}
7540
7541 ins_encode( aarch64_enc_fstlrs(src, mem) );
7542
7543 ins_pipe(pipe_class_memory);
7544 %}
7545
7546 // TODO
7547 // implement storeImmF0 and storeFImmPacked
7548
7549 // Store Double
7550 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7551 %{
7552 match(Set mem (StoreD mem src));
7553
7554 ins_cost(VOLATILE_REF_COST);
7555 format %{ "stlrd $src, $mem\t# double" %}
7556
7557 ins_encode( aarch64_enc_fstlrd(src, mem) );
7558
7559 ins_pipe(pipe_class_memory);
7560 %}
7561
7562 // ---------------- end of volatile loads and stores ----------------
7563
7564 instruct cacheWB(indirect addr)
7565 %{
7566 predicate(VM_Version::supports_data_cache_line_flush());
7567 match(CacheWB addr);
7568
7569 ins_cost(100);
7570 format %{"cache wb $addr" %}
7571 ins_encode %{
7572 assert($addr->index_position() < 0, "should be");
7573 assert($addr$$disp == 0, "should be");
7574 __ cache_wb(Address($addr$$base$$Register, 0));
7575 %}
7576 ins_pipe(pipe_slow); // XXX
7577 %}
7578
7579 instruct cacheWBPreSync()
7580 %{
7581 predicate(VM_Version::supports_data_cache_line_flush());
7582 match(CacheWBPreSync);
7583
7584 ins_cost(100);
7585 format %{"cache wb presync" %}
7586 ins_encode %{
7587 __ cache_wbsync(true);
7588 %}
7589 ins_pipe(pipe_slow); // XXX
7590 %}
7591
7592 instruct cacheWBPostSync()
7593 %{
7594 predicate(VM_Version::supports_data_cache_line_flush());
7595 match(CacheWBPostSync);
7596
7597 ins_cost(100);
7598 format %{"cache wb postsync" %}
7599 ins_encode %{
7600 __ cache_wbsync(false);
7601 %}
7602 ins_pipe(pipe_slow); // XXX
7603 %}
7604
7605 // ============================================================================
7606 // BSWAP Instructions
7607
7608 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7609 match(Set dst (ReverseBytesI src));
7610
7611 ins_cost(INSN_COST);
7612 format %{ "revw $dst, $src" %}
7613
7614 ins_encode %{
7615 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7616 %}
7617
7618 ins_pipe(ialu_reg);
7619 %}
7620
7621 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7622 match(Set dst (ReverseBytesL src));
7623
7624 ins_cost(INSN_COST);
7625 format %{ "rev $dst, $src" %}
7626
7627 ins_encode %{
7628 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7629 %}
7630
7631 ins_pipe(ialu_reg);
7632 %}
7633
7634 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7635 match(Set dst (ReverseBytesUS src));
7636
7637 ins_cost(INSN_COST);
7638 format %{ "rev16w $dst, $src" %}
7639
7640 ins_encode %{
7641 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7642 %}
7643
7644 ins_pipe(ialu_reg);
7645 %}
7646
7647 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7648 match(Set dst (ReverseBytesS src));
7649
7650 ins_cost(INSN_COST);
7651 format %{ "rev16w $dst, $src\n\t"
7652 "sbfmw $dst, $dst, #0, #15" %}
7653
7654 ins_encode %{
7655 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7656 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7657 %}
7658
7659 ins_pipe(ialu_reg);
7660 %}
7661
7662 // ============================================================================
7663 // Zero Count Instructions
7664
7665 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7666 match(Set dst (CountLeadingZerosI src));
7667
7668 ins_cost(INSN_COST);
7669 format %{ "clzw $dst, $src" %}
7670 ins_encode %{
7671 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7672 %}
7673
7674 ins_pipe(ialu_reg);
7675 %}
7676
7677 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7678 match(Set dst (CountLeadingZerosL src));
7679
7680 ins_cost(INSN_COST);
7681 format %{ "clz $dst, $src" %}
7682 ins_encode %{
7683 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7684 %}
7685
7686 ins_pipe(ialu_reg);
7687 %}
7688
7689 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7690 match(Set dst (CountTrailingZerosI src));
7691
7692 ins_cost(INSN_COST * 2);
7693 format %{ "rbitw $dst, $src\n\t"
7694 "clzw $dst, $dst" %}
7695 ins_encode %{
7696 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7697 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7698 %}
7699
7700 ins_pipe(ialu_reg);
7701 %}
7702
7703 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7704 match(Set dst (CountTrailingZerosL src));
7705
7706 ins_cost(INSN_COST * 2);
7707 format %{ "rbit $dst, $src\n\t"
7708 "clz $dst, $dst" %}
7709 ins_encode %{
7710 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7711 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7712 %}
7713
7714 ins_pipe(ialu_reg);
7715 %}
7716
7717 //---------- Population Count Instructions -------------------------------------
7718 //
7719
7720 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7721 match(Set dst (PopCountI src));
7722 effect(TEMP tmp);
7723 ins_cost(INSN_COST * 13);
7724
7725 format %{ "movw $src, $src\n\t"
7726 "mov $tmp, $src\t# vector (1D)\n\t"
7727 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7728 "addv $tmp, $tmp\t# vector (8B)\n\t"
7729 "mov $dst, $tmp\t# vector (1D)" %}
7730 ins_encode %{
7731 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7732 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7733 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7734 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7735 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7736 %}
7737
7738 ins_pipe(pipe_class_default);
7739 %}
7740
7741 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7742 match(Set dst (PopCountI (LoadI mem)));
7743 effect(TEMP tmp);
7744 ins_cost(INSN_COST * 13);
7745
7746 format %{ "ldrs $tmp, $mem\n\t"
7747 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7748 "addv $tmp, $tmp\t# vector (8B)\n\t"
7749 "mov $dst, $tmp\t# vector (1D)" %}
7750 ins_encode %{
7751 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7752 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7753 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7754 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7755 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7756 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7757 %}
7758
7759 ins_pipe(pipe_class_default);
7760 %}
7761
7762 // Note: Long.bitCount(long) returns an int.
7763 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7764 match(Set dst (PopCountL src));
7765 effect(TEMP tmp);
7766 ins_cost(INSN_COST * 13);
7767
7768 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7769 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7770 "addv $tmp, $tmp\t# vector (8B)\n\t"
7771 "mov $dst, $tmp\t# vector (1D)" %}
7772 ins_encode %{
7773 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7774 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7775 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7776 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7777 %}
7778
7779 ins_pipe(pipe_class_default);
7780 %}
7781
7782 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7783 match(Set dst (PopCountL (LoadL mem)));
7784 effect(TEMP tmp);
7785 ins_cost(INSN_COST * 13);
7786
7787 format %{ "ldrd $tmp, $mem\n\t"
7788 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7789 "addv $tmp, $tmp\t# vector (8B)\n\t"
7790 "mov $dst, $tmp\t# vector (1D)" %}
7791 ins_encode %{
7792 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7793 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7794 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7795 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7796 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7797 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7798 %}
7799
7800 ins_pipe(pipe_class_default);
7801 %}
7802
7803 // ============================================================================
7804 // VerifyVectorAlignment Instruction
7805
7806 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7807 match(Set addr (VerifyVectorAlignment addr mask));
7808 effect(KILL cr);
7809 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7810 ins_encode %{
7811 Label Lskip;
7812 // check if masked bits of addr are zero
7813 __ tst($addr$$Register, $mask$$constant);
7814 __ br(Assembler::EQ, Lskip);
7815 __ stop("verify_vector_alignment found a misaligned vector memory access");
7816 __ bind(Lskip);
7817 %}
7818 ins_pipe(pipe_slow);
7819 %}
7820
7821 // ============================================================================
7822 // MemBar Instruction
7823
7824 instruct load_fence() %{
7825 match(LoadFence);
7826 ins_cost(VOLATILE_REF_COST);
7827
7828 format %{ "load_fence" %}
7829
7830 ins_encode %{
7831 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7832 %}
7833 ins_pipe(pipe_serial);
7834 %}
7835
7836 instruct unnecessary_membar_acquire() %{
7837 predicate(unnecessary_acquire(n));
7838 match(MemBarAcquire);
7839 ins_cost(0);
7840
7841 format %{ "membar_acquire (elided)" %}
7842
7843 ins_encode %{
7844 __ block_comment("membar_acquire (elided)");
7845 %}
7846
7847 ins_pipe(pipe_class_empty);
7848 %}
7849
7850 instruct membar_acquire() %{
7851 match(MemBarAcquire);
7852 ins_cost(VOLATILE_REF_COST);
7853
7854 format %{ "membar_acquire\n\t"
7855 "dmb ishld" %}
7856
7857 ins_encode %{
7858 __ block_comment("membar_acquire");
7859 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7860 %}
7861
7862 ins_pipe(pipe_serial);
7863 %}
7864
7865
7866 instruct membar_acquire_lock() %{
7867 match(MemBarAcquireLock);
7868 ins_cost(VOLATILE_REF_COST);
7869
7870 format %{ "membar_acquire_lock (elided)" %}
7871
7872 ins_encode %{
7873 __ block_comment("membar_acquire_lock (elided)");
7874 %}
7875
7876 ins_pipe(pipe_serial);
7877 %}
7878
7879 instruct store_fence() %{
7880 match(StoreFence);
7881 ins_cost(VOLATILE_REF_COST);
7882
7883 format %{ "store_fence" %}
7884
7885 ins_encode %{
7886 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7887 %}
7888 ins_pipe(pipe_serial);
7889 %}
7890
7891 instruct unnecessary_membar_release() %{
7892 predicate(unnecessary_release(n));
7893 match(MemBarRelease);
7894 ins_cost(0);
7895
7896 format %{ "membar_release (elided)" %}
7897
7898 ins_encode %{
7899 __ block_comment("membar_release (elided)");
7900 %}
7901 ins_pipe(pipe_serial);
7902 %}
7903
7904 instruct membar_release() %{
7905 match(MemBarRelease);
7906 ins_cost(VOLATILE_REF_COST);
7907
7908 format %{ "membar_release\n\t"
7909 "dmb ishst\n\tdmb ishld" %}
7910
7911 ins_encode %{
7912 __ block_comment("membar_release");
7913 // These will be merged if AlwaysMergeDMB is enabled.
7914 __ membar(Assembler::StoreStore);
7915 __ membar(Assembler::LoadStore);
7916 %}
7917 ins_pipe(pipe_serial);
7918 %}
7919
7920 instruct membar_storestore() %{
7921 match(MemBarStoreStore);
7922 match(StoreStoreFence);
7923 ins_cost(VOLATILE_REF_COST);
7924
7925 format %{ "MEMBAR-store-store" %}
7926
7927 ins_encode %{
7928 __ membar(Assembler::StoreStore);
7929 %}
7930 ins_pipe(pipe_serial);
7931 %}
7932
7933 instruct membar_release_lock() %{
7934 match(MemBarReleaseLock);
7935 ins_cost(VOLATILE_REF_COST);
7936
7937 format %{ "membar_release_lock (elided)" %}
7938
7939 ins_encode %{
7940 __ block_comment("membar_release_lock (elided)");
7941 %}
7942
7943 ins_pipe(pipe_serial);
7944 %}
7945
7946 instruct unnecessary_membar_volatile() %{
7947 predicate(unnecessary_volatile(n));
7948 match(MemBarVolatile);
7949 ins_cost(0);
7950
7951 format %{ "membar_volatile (elided)" %}
7952
7953 ins_encode %{
7954 __ block_comment("membar_volatile (elided)");
7955 %}
7956
7957 ins_pipe(pipe_serial);
7958 %}
7959
7960 instruct membar_volatile() %{
7961 match(MemBarVolatile);
7962 ins_cost(VOLATILE_REF_COST*100);
7963
7964 format %{ "membar_volatile\n\t"
7965 "dmb ish"%}
7966
7967 ins_encode %{
7968 __ block_comment("membar_volatile");
7969 __ membar(Assembler::StoreLoad);
7970 %}
7971
7972 ins_pipe(pipe_serial);
7973 %}
7974
7975 // ============================================================================
7976 // Cast/Convert Instructions
7977
7978 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7979 match(Set dst (CastX2P src));
7980
7981 ins_cost(INSN_COST);
7982 format %{ "mov $dst, $src\t# long -> ptr" %}
7983
7984 ins_encode %{
7985 if ($dst$$reg != $src$$reg) {
7986 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7987 }
7988 %}
7989
7990 ins_pipe(ialu_reg);
7991 %}
7992
7993 instruct castP2X(iRegLNoSp dst, iRegP src) %{
7994 match(Set dst (CastP2X src));
7995
7996 ins_cost(INSN_COST);
7997 format %{ "mov $dst, $src\t# ptr -> long" %}
7998
7999 ins_encode %{
8000 if ($dst$$reg != $src$$reg) {
8001 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8002 }
8003 %}
8004
8005 ins_pipe(ialu_reg);
8006 %}
8007
8008 // Convert oop into int for vectors alignment masking
8009 instruct convP2I(iRegINoSp dst, iRegP src) %{
8010 match(Set dst (ConvL2I (CastP2X src)));
8011
8012 ins_cost(INSN_COST);
8013 format %{ "movw $dst, $src\t# ptr -> int" %}
8014 ins_encode %{
8015 __ movw($dst$$Register, $src$$Register);
8016 %}
8017
8018 ins_pipe(ialu_reg);
8019 %}
8020
8021 // Convert compressed oop into int for vectors alignment masking
8022 // in case of 32bit oops (heap < 4Gb).
8023 instruct convN2I(iRegINoSp dst, iRegN src)
8024 %{
8025 predicate(CompressedOops::shift() == 0);
8026 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8027
8028 ins_cost(INSN_COST);
8029 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8030 ins_encode %{
8031 __ movw($dst$$Register, $src$$Register);
8032 %}
8033
8034 ins_pipe(ialu_reg);
8035 %}
8036
8037
8038 // Convert oop pointer into compressed form
8039 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8040 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8041 match(Set dst (EncodeP src));
8042 effect(KILL cr);
8043 ins_cost(INSN_COST * 3);
8044 format %{ "encode_heap_oop $dst, $src" %}
8045 ins_encode %{
8046 Register s = $src$$Register;
8047 Register d = $dst$$Register;
8048 __ encode_heap_oop(d, s);
8049 %}
8050 ins_pipe(ialu_reg);
8051 %}
8052
8053 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8054 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8055 match(Set dst (EncodeP src));
8056 ins_cost(INSN_COST * 3);
8057 format %{ "encode_heap_oop_not_null $dst, $src" %}
8058 ins_encode %{
8059 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8060 %}
8061 ins_pipe(ialu_reg);
8062 %}
8063
8064 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8065 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8066 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8067 match(Set dst (DecodeN src));
8068 ins_cost(INSN_COST * 3);
8069 format %{ "decode_heap_oop $dst, $src" %}
8070 ins_encode %{
8071 Register s = $src$$Register;
8072 Register d = $dst$$Register;
8073 __ decode_heap_oop(d, s);
8074 %}
8075 ins_pipe(ialu_reg);
8076 %}
8077
8078 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8079 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8080 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8081 match(Set dst (DecodeN src));
8082 ins_cost(INSN_COST * 3);
8083 format %{ "decode_heap_oop_not_null $dst, $src" %}
8084 ins_encode %{
8085 Register s = $src$$Register;
8086 Register d = $dst$$Register;
8087 __ decode_heap_oop_not_null(d, s);
8088 %}
8089 ins_pipe(ialu_reg);
8090 %}
8091
8092 // n.b. AArch64 implementations of encode_klass_not_null and
8093 // decode_klass_not_null do not modify the flags register so, unlike
8094 // Intel, we don't kill CR as a side effect here
8095
8096 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8097 match(Set dst (EncodePKlass src));
8098
8099 ins_cost(INSN_COST * 3);
8100 format %{ "encode_klass_not_null $dst,$src" %}
8101
8102 ins_encode %{
8103 Register src_reg = as_Register($src$$reg);
8104 Register dst_reg = as_Register($dst$$reg);
8105 __ encode_klass_not_null(dst_reg, src_reg);
8106 %}
8107
8108 ins_pipe(ialu_reg);
8109 %}
8110
8111 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8112 match(Set dst (DecodeNKlass src));
8113
8114 ins_cost(INSN_COST * 3);
8115 format %{ "decode_klass_not_null $dst,$src" %}
8116
8117 ins_encode %{
8118 Register src_reg = as_Register($src$$reg);
8119 Register dst_reg = as_Register($dst$$reg);
8120 if (dst_reg != src_reg) {
8121 __ decode_klass_not_null(dst_reg, src_reg);
8122 } else {
8123 __ decode_klass_not_null(dst_reg);
8124 }
8125 %}
8126
8127 ins_pipe(ialu_reg);
8128 %}
8129
8130 instruct checkCastPP(iRegPNoSp dst)
8131 %{
8132 match(Set dst (CheckCastPP dst));
8133
8134 size(0);
8135 format %{ "# checkcastPP of $dst" %}
8136 ins_encode(/* empty encoding */);
8137 ins_pipe(pipe_class_empty);
8138 %}
8139
8140 instruct castPP(iRegPNoSp dst)
8141 %{
8142 match(Set dst (CastPP dst));
8143
8144 size(0);
8145 format %{ "# castPP of $dst" %}
8146 ins_encode(/* empty encoding */);
8147 ins_pipe(pipe_class_empty);
8148 %}
8149
8150 instruct castII(iRegI dst)
8151 %{
8152 match(Set dst (CastII dst));
8153
8154 size(0);
8155 format %{ "# castII of $dst" %}
8156 ins_encode(/* empty encoding */);
8157 ins_cost(0);
8158 ins_pipe(pipe_class_empty);
8159 %}
8160
8161 instruct castLL(iRegL dst)
8162 %{
8163 match(Set dst (CastLL dst));
8164
8165 size(0);
8166 format %{ "# castLL of $dst" %}
8167 ins_encode(/* empty encoding */);
8168 ins_cost(0);
8169 ins_pipe(pipe_class_empty);
8170 %}
8171
8172 instruct castFF(vRegF dst)
8173 %{
8174 match(Set dst (CastFF dst));
8175
8176 size(0);
8177 format %{ "# castFF of $dst" %}
8178 ins_encode(/* empty encoding */);
8179 ins_cost(0);
8180 ins_pipe(pipe_class_empty);
8181 %}
8182
8183 instruct castDD(vRegD dst)
8184 %{
8185 match(Set dst (CastDD dst));
8186
8187 size(0);
8188 format %{ "# castDD of $dst" %}
8189 ins_encode(/* empty encoding */);
8190 ins_cost(0);
8191 ins_pipe(pipe_class_empty);
8192 %}
8193
8194 instruct castVV(vReg dst)
8195 %{
8196 match(Set dst (CastVV dst));
8197
8198 size(0);
8199 format %{ "# castVV of $dst" %}
8200 ins_encode(/* empty encoding */);
8201 ins_cost(0);
8202 ins_pipe(pipe_class_empty);
8203 %}
8204
8205 instruct castVVMask(pRegGov dst)
8206 %{
8207 match(Set dst (CastVV dst));
8208
8209 size(0);
8210 format %{ "# castVV of $dst" %}
8211 ins_encode(/* empty encoding */);
8212 ins_cost(0);
8213 ins_pipe(pipe_class_empty);
8214 %}
8215
8216 // ============================================================================
8217 // Atomic operation instructions
8218 //
8219
8220 // standard CompareAndSwapX when we are using barriers
8221 // these have higher priority than the rules selected by a predicate
8222
8223 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8224 // can't match them
8225
8226 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8227
8228 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8229 ins_cost(2 * VOLATILE_REF_COST);
8230
8231 effect(KILL cr);
8232
8233 format %{
8234 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8235 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8236 %}
8237
8238 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8239 aarch64_enc_cset_eq(res));
8240
8241 ins_pipe(pipe_slow);
8242 %}
8243
8244 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8245
8246 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8247 ins_cost(2 * VOLATILE_REF_COST);
8248
8249 effect(KILL cr);
8250
8251 format %{
8252 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8253 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8254 %}
8255
8256 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8257 aarch64_enc_cset_eq(res));
8258
8259 ins_pipe(pipe_slow);
8260 %}
8261
8262 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8263
8264 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8265 ins_cost(2 * VOLATILE_REF_COST);
8266
8267 effect(KILL cr);
8268
8269 format %{
8270 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8271 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8272 %}
8273
8274 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8275 aarch64_enc_cset_eq(res));
8276
8277 ins_pipe(pipe_slow);
8278 %}
8279
8280 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8281
8282 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8283 ins_cost(2 * VOLATILE_REF_COST);
8284
8285 effect(KILL cr);
8286
8287 format %{
8288 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8289 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8290 %}
8291
8292 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8293 aarch64_enc_cset_eq(res));
8294
8295 ins_pipe(pipe_slow);
8296 %}
8297
8298 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8299
8300 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8301 predicate(n->as_LoadStore()->barrier_data() == 0);
8302 ins_cost(2 * VOLATILE_REF_COST);
8303
8304 effect(KILL cr);
8305
8306 format %{
8307 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8308 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8309 %}
8310
8311 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8312 aarch64_enc_cset_eq(res));
8313
8314 ins_pipe(pipe_slow);
8315 %}
8316
8317 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8318
8319 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8320 predicate(n->as_LoadStore()->barrier_data() == 0);
8321 ins_cost(2 * VOLATILE_REF_COST);
8322
8323 effect(KILL cr);
8324
8325 format %{
8326 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8327 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8328 %}
8329
8330 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8331 aarch64_enc_cset_eq(res));
8332
8333 ins_pipe(pipe_slow);
8334 %}
8335
8336 // alternative CompareAndSwapX when we are eliding barriers
8337
8338 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8339
8340 predicate(needs_acquiring_load_exclusive(n));
8341 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8342 ins_cost(VOLATILE_REF_COST);
8343
8344 effect(KILL cr);
8345
8346 format %{
8347 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8348 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8349 %}
8350
8351 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8352 aarch64_enc_cset_eq(res));
8353
8354 ins_pipe(pipe_slow);
8355 %}
8356
8357 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8358
8359 predicate(needs_acquiring_load_exclusive(n));
8360 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8361 ins_cost(VOLATILE_REF_COST);
8362
8363 effect(KILL cr);
8364
8365 format %{
8366 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8367 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8368 %}
8369
8370 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8371 aarch64_enc_cset_eq(res));
8372
8373 ins_pipe(pipe_slow);
8374 %}
8375
8376 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8377
8378 predicate(needs_acquiring_load_exclusive(n));
8379 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8380 ins_cost(VOLATILE_REF_COST);
8381
8382 effect(KILL cr);
8383
8384 format %{
8385 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8386 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8387 %}
8388
8389 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8390 aarch64_enc_cset_eq(res));
8391
8392 ins_pipe(pipe_slow);
8393 %}
8394
8395 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8396
8397 predicate(needs_acquiring_load_exclusive(n));
8398 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8399 ins_cost(VOLATILE_REF_COST);
8400
8401 effect(KILL cr);
8402
8403 format %{
8404 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8405 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8406 %}
8407
8408 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8409 aarch64_enc_cset_eq(res));
8410
8411 ins_pipe(pipe_slow);
8412 %}
8413
8414 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8415
8416 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8417 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8418 ins_cost(VOLATILE_REF_COST);
8419
8420 effect(KILL cr);
8421
8422 format %{
8423 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8424 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8425 %}
8426
8427 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8428 aarch64_enc_cset_eq(res));
8429
8430 ins_pipe(pipe_slow);
8431 %}
8432
8433 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8434
8435 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8436 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8437 ins_cost(VOLATILE_REF_COST);
8438
8439 effect(KILL cr);
8440
8441 format %{
8442 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8443 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8444 %}
8445
8446 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8447 aarch64_enc_cset_eq(res));
8448
8449 ins_pipe(pipe_slow);
8450 %}
8451
8452
8453 // ---------------------------------------------------------------------
8454
8455 // BEGIN This section of the file is automatically generated. Do not edit --------------
8456
8457 // Sundry CAS operations. Note that release is always true,
8458 // regardless of the memory ordering of the CAS. This is because we
8459 // need the volatile case to be sequentially consistent but there is
8460 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8461 // can't check the type of memory ordering here, so we always emit a
8462 // STLXR.
8463
8464 // This section is generated from cas.m4
8465
8466
8467 // This pattern is generated automatically from cas.m4.
8468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8469 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8470 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8471 ins_cost(2 * VOLATILE_REF_COST);
8472 effect(TEMP_DEF res, KILL cr);
8473 format %{
8474 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8475 %}
8476 ins_encode %{
8477 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8478 Assembler::byte, /*acquire*/ false, /*release*/ true,
8479 /*weak*/ false, $res$$Register);
8480 __ sxtbw($res$$Register, $res$$Register);
8481 %}
8482 ins_pipe(pipe_slow);
8483 %}
8484
8485 // This pattern is generated automatically from cas.m4.
8486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8487 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8488 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8489 ins_cost(2 * VOLATILE_REF_COST);
8490 effect(TEMP_DEF res, KILL cr);
8491 format %{
8492 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8493 %}
8494 ins_encode %{
8495 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8496 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8497 /*weak*/ false, $res$$Register);
8498 __ sxthw($res$$Register, $res$$Register);
8499 %}
8500 ins_pipe(pipe_slow);
8501 %}
8502
8503 // This pattern is generated automatically from cas.m4.
8504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8505 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8506 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8507 ins_cost(2 * VOLATILE_REF_COST);
8508 effect(TEMP_DEF res, KILL cr);
8509 format %{
8510 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8511 %}
8512 ins_encode %{
8513 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8514 Assembler::word, /*acquire*/ false, /*release*/ true,
8515 /*weak*/ false, $res$$Register);
8516 %}
8517 ins_pipe(pipe_slow);
8518 %}
8519
8520 // This pattern is generated automatically from cas.m4.
8521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8522 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8523 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8524 ins_cost(2 * VOLATILE_REF_COST);
8525 effect(TEMP_DEF res, KILL cr);
8526 format %{
8527 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8528 %}
8529 ins_encode %{
8530 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8531 Assembler::xword, /*acquire*/ false, /*release*/ true,
8532 /*weak*/ false, $res$$Register);
8533 %}
8534 ins_pipe(pipe_slow);
8535 %}
8536
8537 // This pattern is generated automatically from cas.m4.
8538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8539 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8540 predicate(n->as_LoadStore()->barrier_data() == 0);
8541 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8542 ins_cost(2 * VOLATILE_REF_COST);
8543 effect(TEMP_DEF res, KILL cr);
8544 format %{
8545 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8546 %}
8547 ins_encode %{
8548 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8549 Assembler::word, /*acquire*/ false, /*release*/ true,
8550 /*weak*/ false, $res$$Register);
8551 %}
8552 ins_pipe(pipe_slow);
8553 %}
8554
8555 // This pattern is generated automatically from cas.m4.
8556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8557 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8558 predicate(n->as_LoadStore()->barrier_data() == 0);
8559 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8560 ins_cost(2 * VOLATILE_REF_COST);
8561 effect(TEMP_DEF res, KILL cr);
8562 format %{
8563 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8564 %}
8565 ins_encode %{
8566 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8567 Assembler::xword, /*acquire*/ false, /*release*/ true,
8568 /*weak*/ false, $res$$Register);
8569 %}
8570 ins_pipe(pipe_slow);
8571 %}
8572
8573 // This pattern is generated automatically from cas.m4.
8574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8575 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8576 predicate(needs_acquiring_load_exclusive(n));
8577 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8578 ins_cost(VOLATILE_REF_COST);
8579 effect(TEMP_DEF res, KILL cr);
8580 format %{
8581 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8582 %}
8583 ins_encode %{
8584 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8585 Assembler::byte, /*acquire*/ true, /*release*/ true,
8586 /*weak*/ false, $res$$Register);
8587 __ sxtbw($res$$Register, $res$$Register);
8588 %}
8589 ins_pipe(pipe_slow);
8590 %}
8591
8592 // This pattern is generated automatically from cas.m4.
8593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8594 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8595 predicate(needs_acquiring_load_exclusive(n));
8596 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8597 ins_cost(VOLATILE_REF_COST);
8598 effect(TEMP_DEF res, KILL cr);
8599 format %{
8600 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8601 %}
8602 ins_encode %{
8603 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8604 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8605 /*weak*/ false, $res$$Register);
8606 __ sxthw($res$$Register, $res$$Register);
8607 %}
8608 ins_pipe(pipe_slow);
8609 %}
8610
8611 // This pattern is generated automatically from cas.m4.
8612 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8613 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8614 predicate(needs_acquiring_load_exclusive(n));
8615 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8616 ins_cost(VOLATILE_REF_COST);
8617 effect(TEMP_DEF res, KILL cr);
8618 format %{
8619 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8620 %}
8621 ins_encode %{
8622 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8623 Assembler::word, /*acquire*/ true, /*release*/ true,
8624 /*weak*/ false, $res$$Register);
8625 %}
8626 ins_pipe(pipe_slow);
8627 %}
8628
8629 // This pattern is generated automatically from cas.m4.
8630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8631 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8632 predicate(needs_acquiring_load_exclusive(n));
8633 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8634 ins_cost(VOLATILE_REF_COST);
8635 effect(TEMP_DEF res, KILL cr);
8636 format %{
8637 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8638 %}
8639 ins_encode %{
8640 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8641 Assembler::xword, /*acquire*/ true, /*release*/ true,
8642 /*weak*/ false, $res$$Register);
8643 %}
8644 ins_pipe(pipe_slow);
8645 %}
8646
8647 // This pattern is generated automatically from cas.m4.
8648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8649 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8650 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8651 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8652 ins_cost(VOLATILE_REF_COST);
8653 effect(TEMP_DEF res, KILL cr);
8654 format %{
8655 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8656 %}
8657 ins_encode %{
8658 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8659 Assembler::word, /*acquire*/ true, /*release*/ true,
8660 /*weak*/ false, $res$$Register);
8661 %}
8662 ins_pipe(pipe_slow);
8663 %}
8664
8665 // This pattern is generated automatically from cas.m4.
8666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8667 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8668 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8669 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8670 ins_cost(VOLATILE_REF_COST);
8671 effect(TEMP_DEF res, KILL cr);
8672 format %{
8673 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8674 %}
8675 ins_encode %{
8676 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8677 Assembler::xword, /*acquire*/ true, /*release*/ true,
8678 /*weak*/ false, $res$$Register);
8679 %}
8680 ins_pipe(pipe_slow);
8681 %}
8682
8683 // This pattern is generated automatically from cas.m4.
8684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8685 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8686 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8687 ins_cost(2 * VOLATILE_REF_COST);
8688 effect(KILL cr);
8689 format %{
8690 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8691 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8692 %}
8693 ins_encode %{
8694 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8695 Assembler::byte, /*acquire*/ false, /*release*/ true,
8696 /*weak*/ true, noreg);
8697 __ csetw($res$$Register, Assembler::EQ);
8698 %}
8699 ins_pipe(pipe_slow);
8700 %}
8701
8702 // This pattern is generated automatically from cas.m4.
8703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8704 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8705 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8706 ins_cost(2 * VOLATILE_REF_COST);
8707 effect(KILL cr);
8708 format %{
8709 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8710 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8711 %}
8712 ins_encode %{
8713 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8714 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8715 /*weak*/ true, noreg);
8716 __ csetw($res$$Register, Assembler::EQ);
8717 %}
8718 ins_pipe(pipe_slow);
8719 %}
8720
8721 // This pattern is generated automatically from cas.m4.
8722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8723 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8724 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8725 ins_cost(2 * VOLATILE_REF_COST);
8726 effect(KILL cr);
8727 format %{
8728 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8729 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8730 %}
8731 ins_encode %{
8732 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8733 Assembler::word, /*acquire*/ false, /*release*/ true,
8734 /*weak*/ true, noreg);
8735 __ csetw($res$$Register, Assembler::EQ);
8736 %}
8737 ins_pipe(pipe_slow);
8738 %}
8739
8740 // This pattern is generated automatically from cas.m4.
8741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8742 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8743 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8744 ins_cost(2 * VOLATILE_REF_COST);
8745 effect(KILL cr);
8746 format %{
8747 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8748 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8749 %}
8750 ins_encode %{
8751 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8752 Assembler::xword, /*acquire*/ false, /*release*/ true,
8753 /*weak*/ true, noreg);
8754 __ csetw($res$$Register, Assembler::EQ);
8755 %}
8756 ins_pipe(pipe_slow);
8757 %}
8758
8759 // This pattern is generated automatically from cas.m4.
8760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8761 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8762 predicate(n->as_LoadStore()->barrier_data() == 0);
8763 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8764 ins_cost(2 * VOLATILE_REF_COST);
8765 effect(KILL cr);
8766 format %{
8767 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8768 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8769 %}
8770 ins_encode %{
8771 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8772 Assembler::word, /*acquire*/ false, /*release*/ true,
8773 /*weak*/ true, noreg);
8774 __ csetw($res$$Register, Assembler::EQ);
8775 %}
8776 ins_pipe(pipe_slow);
8777 %}
8778
8779 // This pattern is generated automatically from cas.m4.
8780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8781 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8782 predicate(n->as_LoadStore()->barrier_data() == 0);
8783 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8784 ins_cost(2 * VOLATILE_REF_COST);
8785 effect(KILL cr);
8786 format %{
8787 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8788 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8789 %}
8790 ins_encode %{
8791 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8792 Assembler::xword, /*acquire*/ false, /*release*/ true,
8793 /*weak*/ true, noreg);
8794 __ csetw($res$$Register, Assembler::EQ);
8795 %}
8796 ins_pipe(pipe_slow);
8797 %}
8798
8799 // This pattern is generated automatically from cas.m4.
8800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8801 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8802 predicate(needs_acquiring_load_exclusive(n));
8803 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8804 ins_cost(VOLATILE_REF_COST);
8805 effect(KILL cr);
8806 format %{
8807 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8808 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8809 %}
8810 ins_encode %{
8811 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8812 Assembler::byte, /*acquire*/ true, /*release*/ true,
8813 /*weak*/ true, noreg);
8814 __ csetw($res$$Register, Assembler::EQ);
8815 %}
8816 ins_pipe(pipe_slow);
8817 %}
8818
8819 // This pattern is generated automatically from cas.m4.
8820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8821 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8822 predicate(needs_acquiring_load_exclusive(n));
8823 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8824 ins_cost(VOLATILE_REF_COST);
8825 effect(KILL cr);
8826 format %{
8827 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8828 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8829 %}
8830 ins_encode %{
8831 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8832 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8833 /*weak*/ true, noreg);
8834 __ csetw($res$$Register, Assembler::EQ);
8835 %}
8836 ins_pipe(pipe_slow);
8837 %}
8838
8839 // This pattern is generated automatically from cas.m4.
8840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8841 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8842 predicate(needs_acquiring_load_exclusive(n));
8843 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8844 ins_cost(VOLATILE_REF_COST);
8845 effect(KILL cr);
8846 format %{
8847 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8848 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8849 %}
8850 ins_encode %{
8851 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8852 Assembler::word, /*acquire*/ true, /*release*/ true,
8853 /*weak*/ true, noreg);
8854 __ csetw($res$$Register, Assembler::EQ);
8855 %}
8856 ins_pipe(pipe_slow);
8857 %}
8858
8859 // This pattern is generated automatically from cas.m4.
8860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8861 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8862 predicate(needs_acquiring_load_exclusive(n));
8863 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8864 ins_cost(VOLATILE_REF_COST);
8865 effect(KILL cr);
8866 format %{
8867 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8868 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8869 %}
8870 ins_encode %{
8871 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8872 Assembler::xword, /*acquire*/ true, /*release*/ true,
8873 /*weak*/ true, noreg);
8874 __ csetw($res$$Register, Assembler::EQ);
8875 %}
8876 ins_pipe(pipe_slow);
8877 %}
8878
8879 // This pattern is generated automatically from cas.m4.
8880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8881 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8882 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8883 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8884 ins_cost(VOLATILE_REF_COST);
8885 effect(KILL cr);
8886 format %{
8887 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8888 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8889 %}
8890 ins_encode %{
8891 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8892 Assembler::word, /*acquire*/ true, /*release*/ true,
8893 /*weak*/ true, noreg);
8894 __ csetw($res$$Register, Assembler::EQ);
8895 %}
8896 ins_pipe(pipe_slow);
8897 %}
8898
8899 // This pattern is generated automatically from cas.m4.
8900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8901 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8902 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8903 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8904 ins_cost(VOLATILE_REF_COST);
8905 effect(KILL cr);
8906 format %{
8907 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8908 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8909 %}
8910 ins_encode %{
8911 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8912 Assembler::xword, /*acquire*/ true, /*release*/ true,
8913 /*weak*/ true, noreg);
8914 __ csetw($res$$Register, Assembler::EQ);
8915 %}
8916 ins_pipe(pipe_slow);
8917 %}
8918
8919 // END This section of the file is automatically generated. Do not edit --------------
8920 // ---------------------------------------------------------------------
8921
8922 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8923 match(Set prev (GetAndSetI mem newv));
8924 ins_cost(2 * VOLATILE_REF_COST);
8925 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8926 ins_encode %{
8927 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8928 %}
8929 ins_pipe(pipe_serial);
8930 %}
8931
8932 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8933 match(Set prev (GetAndSetL mem newv));
8934 ins_cost(2 * VOLATILE_REF_COST);
8935 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8936 ins_encode %{
8937 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8938 %}
8939 ins_pipe(pipe_serial);
8940 %}
8941
8942 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8943 predicate(n->as_LoadStore()->barrier_data() == 0);
8944 match(Set prev (GetAndSetN mem newv));
8945 ins_cost(2 * VOLATILE_REF_COST);
8946 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8947 ins_encode %{
8948 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8949 %}
8950 ins_pipe(pipe_serial);
8951 %}
8952
8953 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
8954 predicate(n->as_LoadStore()->barrier_data() == 0);
8955 match(Set prev (GetAndSetP mem newv));
8956 ins_cost(2 * VOLATILE_REF_COST);
8957 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8958 ins_encode %{
8959 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8960 %}
8961 ins_pipe(pipe_serial);
8962 %}
8963
8964 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
8965 predicate(needs_acquiring_load_exclusive(n));
8966 match(Set prev (GetAndSetI mem newv));
8967 ins_cost(VOLATILE_REF_COST);
8968 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8969 ins_encode %{
8970 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8971 %}
8972 ins_pipe(pipe_serial);
8973 %}
8974
8975 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
8976 predicate(needs_acquiring_load_exclusive(n));
8977 match(Set prev (GetAndSetL mem newv));
8978 ins_cost(VOLATILE_REF_COST);
8979 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8980 ins_encode %{
8981 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8982 %}
8983 ins_pipe(pipe_serial);
8984 %}
8985
8986 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
8987 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8988 match(Set prev (GetAndSetN mem newv));
8989 ins_cost(VOLATILE_REF_COST);
8990 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8991 ins_encode %{
8992 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8993 %}
8994 ins_pipe(pipe_serial);
8995 %}
8996
8997 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
8998 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8999 match(Set prev (GetAndSetP mem newv));
9000 ins_cost(VOLATILE_REF_COST);
9001 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9002 ins_encode %{
9003 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9004 %}
9005 ins_pipe(pipe_serial);
9006 %}
9007
9008
9009 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9010 match(Set newval (GetAndAddL mem incr));
9011 ins_cost(2 * VOLATILE_REF_COST + 1);
9012 format %{ "get_and_addL $newval, [$mem], $incr" %}
9013 ins_encode %{
9014 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9015 %}
9016 ins_pipe(pipe_serial);
9017 %}
9018
9019 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9020 predicate(n->as_LoadStore()->result_not_used());
9021 match(Set dummy (GetAndAddL mem incr));
9022 ins_cost(2 * VOLATILE_REF_COST);
9023 format %{ "get_and_addL [$mem], $incr" %}
9024 ins_encode %{
9025 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9026 %}
9027 ins_pipe(pipe_serial);
9028 %}
9029
9030 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9031 match(Set newval (GetAndAddL mem incr));
9032 ins_cost(2 * VOLATILE_REF_COST + 1);
9033 format %{ "get_and_addL $newval, [$mem], $incr" %}
9034 ins_encode %{
9035 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9036 %}
9037 ins_pipe(pipe_serial);
9038 %}
9039
9040 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9041 predicate(n->as_LoadStore()->result_not_used());
9042 match(Set dummy (GetAndAddL mem incr));
9043 ins_cost(2 * VOLATILE_REF_COST);
9044 format %{ "get_and_addL [$mem], $incr" %}
9045 ins_encode %{
9046 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9047 %}
9048 ins_pipe(pipe_serial);
9049 %}
9050
9051 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9052 match(Set newval (GetAndAddI mem incr));
9053 ins_cost(2 * VOLATILE_REF_COST + 1);
9054 format %{ "get_and_addI $newval, [$mem], $incr" %}
9055 ins_encode %{
9056 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9057 %}
9058 ins_pipe(pipe_serial);
9059 %}
9060
9061 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9062 predicate(n->as_LoadStore()->result_not_used());
9063 match(Set dummy (GetAndAddI mem incr));
9064 ins_cost(2 * VOLATILE_REF_COST);
9065 format %{ "get_and_addI [$mem], $incr" %}
9066 ins_encode %{
9067 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9068 %}
9069 ins_pipe(pipe_serial);
9070 %}
9071
9072 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9073 match(Set newval (GetAndAddI mem incr));
9074 ins_cost(2 * VOLATILE_REF_COST + 1);
9075 format %{ "get_and_addI $newval, [$mem], $incr" %}
9076 ins_encode %{
9077 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9078 %}
9079 ins_pipe(pipe_serial);
9080 %}
9081
9082 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9083 predicate(n->as_LoadStore()->result_not_used());
9084 match(Set dummy (GetAndAddI mem incr));
9085 ins_cost(2 * VOLATILE_REF_COST);
9086 format %{ "get_and_addI [$mem], $incr" %}
9087 ins_encode %{
9088 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9089 %}
9090 ins_pipe(pipe_serial);
9091 %}
9092
9093 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9094 predicate(needs_acquiring_load_exclusive(n));
9095 match(Set newval (GetAndAddL mem incr));
9096 ins_cost(VOLATILE_REF_COST + 1);
9097 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9098 ins_encode %{
9099 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9100 %}
9101 ins_pipe(pipe_serial);
9102 %}
9103
9104 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9105 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9106 match(Set dummy (GetAndAddL mem incr));
9107 ins_cost(VOLATILE_REF_COST);
9108 format %{ "get_and_addL_acq [$mem], $incr" %}
9109 ins_encode %{
9110 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9111 %}
9112 ins_pipe(pipe_serial);
9113 %}
9114
9115 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9116 predicate(needs_acquiring_load_exclusive(n));
9117 match(Set newval (GetAndAddL mem incr));
9118 ins_cost(VOLATILE_REF_COST + 1);
9119 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9120 ins_encode %{
9121 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9122 %}
9123 ins_pipe(pipe_serial);
9124 %}
9125
9126 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9127 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9128 match(Set dummy (GetAndAddL mem incr));
9129 ins_cost(VOLATILE_REF_COST);
9130 format %{ "get_and_addL_acq [$mem], $incr" %}
9131 ins_encode %{
9132 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9133 %}
9134 ins_pipe(pipe_serial);
9135 %}
9136
9137 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9138 predicate(needs_acquiring_load_exclusive(n));
9139 match(Set newval (GetAndAddI mem incr));
9140 ins_cost(VOLATILE_REF_COST + 1);
9141 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9142 ins_encode %{
9143 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9144 %}
9145 ins_pipe(pipe_serial);
9146 %}
9147
9148 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9149 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9150 match(Set dummy (GetAndAddI mem incr));
9151 ins_cost(VOLATILE_REF_COST);
9152 format %{ "get_and_addI_acq [$mem], $incr" %}
9153 ins_encode %{
9154 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9155 %}
9156 ins_pipe(pipe_serial);
9157 %}
9158
9159 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9160 predicate(needs_acquiring_load_exclusive(n));
9161 match(Set newval (GetAndAddI mem incr));
9162 ins_cost(VOLATILE_REF_COST + 1);
9163 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9164 ins_encode %{
9165 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9166 %}
9167 ins_pipe(pipe_serial);
9168 %}
9169
9170 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9171 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9172 match(Set dummy (GetAndAddI mem incr));
9173 ins_cost(VOLATILE_REF_COST);
9174 format %{ "get_and_addI_acq [$mem], $incr" %}
9175 ins_encode %{
9176 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9177 %}
9178 ins_pipe(pipe_serial);
9179 %}
9180
9181 // Manifest a CmpU result in an integer register.
9182 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9183 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9184 %{
9185 match(Set dst (CmpU3 src1 src2));
9186 effect(KILL flags);
9187
9188 ins_cost(INSN_COST * 3);
9189 format %{
9190 "cmpw $src1, $src2\n\t"
9191 "csetw $dst, ne\n\t"
9192 "cnegw $dst, lo\t# CmpU3(reg)"
9193 %}
9194 ins_encode %{
9195 __ cmpw($src1$$Register, $src2$$Register);
9196 __ csetw($dst$$Register, Assembler::NE);
9197 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9198 %}
9199
9200 ins_pipe(pipe_class_default);
9201 %}
9202
9203 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9204 %{
9205 match(Set dst (CmpU3 src1 src2));
9206 effect(KILL flags);
9207
9208 ins_cost(INSN_COST * 3);
9209 format %{
9210 "subsw zr, $src1, $src2\n\t"
9211 "csetw $dst, ne\n\t"
9212 "cnegw $dst, lo\t# CmpU3(imm)"
9213 %}
9214 ins_encode %{
9215 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9216 __ csetw($dst$$Register, Assembler::NE);
9217 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9218 %}
9219
9220 ins_pipe(pipe_class_default);
9221 %}
9222
9223 // Manifest a CmpUL result in an integer register.
9224 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9225 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9226 %{
9227 match(Set dst (CmpUL3 src1 src2));
9228 effect(KILL flags);
9229
9230 ins_cost(INSN_COST * 3);
9231 format %{
9232 "cmp $src1, $src2\n\t"
9233 "csetw $dst, ne\n\t"
9234 "cnegw $dst, lo\t# CmpUL3(reg)"
9235 %}
9236 ins_encode %{
9237 __ cmp($src1$$Register, $src2$$Register);
9238 __ csetw($dst$$Register, Assembler::NE);
9239 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9240 %}
9241
9242 ins_pipe(pipe_class_default);
9243 %}
9244
9245 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9246 %{
9247 match(Set dst (CmpUL3 src1 src2));
9248 effect(KILL flags);
9249
9250 ins_cost(INSN_COST * 3);
9251 format %{
9252 "subs zr, $src1, $src2\n\t"
9253 "csetw $dst, ne\n\t"
9254 "cnegw $dst, lo\t# CmpUL3(imm)"
9255 %}
9256 ins_encode %{
9257 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9258 __ csetw($dst$$Register, Assembler::NE);
9259 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9260 %}
9261
9262 ins_pipe(pipe_class_default);
9263 %}
9264
9265 // Manifest a CmpL result in an integer register.
9266 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9267 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9268 %{
9269 match(Set dst (CmpL3 src1 src2));
9270 effect(KILL flags);
9271
9272 ins_cost(INSN_COST * 3);
9273 format %{
9274 "cmp $src1, $src2\n\t"
9275 "csetw $dst, ne\n\t"
9276 "cnegw $dst, lt\t# CmpL3(reg)"
9277 %}
9278 ins_encode %{
9279 __ cmp($src1$$Register, $src2$$Register);
9280 __ csetw($dst$$Register, Assembler::NE);
9281 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9282 %}
9283
9284 ins_pipe(pipe_class_default);
9285 %}
9286
9287 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9288 %{
9289 match(Set dst (CmpL3 src1 src2));
9290 effect(KILL flags);
9291
9292 ins_cost(INSN_COST * 3);
9293 format %{
9294 "subs zr, $src1, $src2\n\t"
9295 "csetw $dst, ne\n\t"
9296 "cnegw $dst, lt\t# CmpL3(imm)"
9297 %}
9298 ins_encode %{
9299 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9300 __ csetw($dst$$Register, Assembler::NE);
9301 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9302 %}
9303
9304 ins_pipe(pipe_class_default);
9305 %}
9306
9307 // ============================================================================
9308 // Conditional Move Instructions
9309
9310 // n.b. we have identical rules for both a signed compare op (cmpOp)
9311 // and an unsigned compare op (cmpOpU). it would be nice if we could
9312 // define an op class which merged both inputs and use it to type the
9313 // argument to a single rule. unfortunatelyt his fails because the
9314 // opclass does not live up to the COND_INTER interface of its
9315 // component operands. When the generic code tries to negate the
9316 // operand it ends up running the generci Machoper::negate method
9317 // which throws a ShouldNotHappen. So, we have to provide two flavours
9318 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9319
9320 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9321 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9322
9323 ins_cost(INSN_COST * 2);
9324 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9325
9326 ins_encode %{
9327 __ cselw(as_Register($dst$$reg),
9328 as_Register($src2$$reg),
9329 as_Register($src1$$reg),
9330 (Assembler::Condition)$cmp$$cmpcode);
9331 %}
9332
9333 ins_pipe(icond_reg_reg);
9334 %}
9335
9336 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9337 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9338
9339 ins_cost(INSN_COST * 2);
9340 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9341
9342 ins_encode %{
9343 __ cselw(as_Register($dst$$reg),
9344 as_Register($src2$$reg),
9345 as_Register($src1$$reg),
9346 (Assembler::Condition)$cmp$$cmpcode);
9347 %}
9348
9349 ins_pipe(icond_reg_reg);
9350 %}
9351
9352 // special cases where one arg is zero
9353
9354 // n.b. this is selected in preference to the rule above because it
9355 // avoids loading constant 0 into a source register
9356
9357 // TODO
9358 // we ought only to be able to cull one of these variants as the ideal
9359 // transforms ought always to order the zero consistently (to left/right?)
9360
9361 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9362 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9363
9364 ins_cost(INSN_COST * 2);
9365 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9366
9367 ins_encode %{
9368 __ cselw(as_Register($dst$$reg),
9369 as_Register($src$$reg),
9370 zr,
9371 (Assembler::Condition)$cmp$$cmpcode);
9372 %}
9373
9374 ins_pipe(icond_reg);
9375 %}
9376
9377 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9378 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9379
9380 ins_cost(INSN_COST * 2);
9381 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9382
9383 ins_encode %{
9384 __ cselw(as_Register($dst$$reg),
9385 as_Register($src$$reg),
9386 zr,
9387 (Assembler::Condition)$cmp$$cmpcode);
9388 %}
9389
9390 ins_pipe(icond_reg);
9391 %}
9392
9393 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9394 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9395
9396 ins_cost(INSN_COST * 2);
9397 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9398
9399 ins_encode %{
9400 __ cselw(as_Register($dst$$reg),
9401 zr,
9402 as_Register($src$$reg),
9403 (Assembler::Condition)$cmp$$cmpcode);
9404 %}
9405
9406 ins_pipe(icond_reg);
9407 %}
9408
9409 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9410 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9411
9412 ins_cost(INSN_COST * 2);
9413 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9414
9415 ins_encode %{
9416 __ cselw(as_Register($dst$$reg),
9417 zr,
9418 as_Register($src$$reg),
9419 (Assembler::Condition)$cmp$$cmpcode);
9420 %}
9421
9422 ins_pipe(icond_reg);
9423 %}
9424
9425 // special case for creating a boolean 0 or 1
9426
9427 // n.b. this is selected in preference to the rule above because it
9428 // avoids loading constants 0 and 1 into a source register
9429
9430 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9431 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9432
9433 ins_cost(INSN_COST * 2);
9434 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9435
9436 ins_encode %{
9437 // equivalently
9438 // cset(as_Register($dst$$reg),
9439 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9440 __ csincw(as_Register($dst$$reg),
9441 zr,
9442 zr,
9443 (Assembler::Condition)$cmp$$cmpcode);
9444 %}
9445
9446 ins_pipe(icond_none);
9447 %}
9448
9449 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9450 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9451
9452 ins_cost(INSN_COST * 2);
9453 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9454
9455 ins_encode %{
9456 // equivalently
9457 // cset(as_Register($dst$$reg),
9458 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9459 __ csincw(as_Register($dst$$reg),
9460 zr,
9461 zr,
9462 (Assembler::Condition)$cmp$$cmpcode);
9463 %}
9464
9465 ins_pipe(icond_none);
9466 %}
9467
9468 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9469 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9470
9471 ins_cost(INSN_COST * 2);
9472 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9473
9474 ins_encode %{
9475 __ csel(as_Register($dst$$reg),
9476 as_Register($src2$$reg),
9477 as_Register($src1$$reg),
9478 (Assembler::Condition)$cmp$$cmpcode);
9479 %}
9480
9481 ins_pipe(icond_reg_reg);
9482 %}
9483
9484 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9485 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9486
9487 ins_cost(INSN_COST * 2);
9488 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9489
9490 ins_encode %{
9491 __ csel(as_Register($dst$$reg),
9492 as_Register($src2$$reg),
9493 as_Register($src1$$reg),
9494 (Assembler::Condition)$cmp$$cmpcode);
9495 %}
9496
9497 ins_pipe(icond_reg_reg);
9498 %}
9499
9500 // special cases where one arg is zero
9501
9502 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9503 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9504
9505 ins_cost(INSN_COST * 2);
9506 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9507
9508 ins_encode %{
9509 __ csel(as_Register($dst$$reg),
9510 zr,
9511 as_Register($src$$reg),
9512 (Assembler::Condition)$cmp$$cmpcode);
9513 %}
9514
9515 ins_pipe(icond_reg);
9516 %}
9517
9518 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9519 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9520
9521 ins_cost(INSN_COST * 2);
9522 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9523
9524 ins_encode %{
9525 __ csel(as_Register($dst$$reg),
9526 zr,
9527 as_Register($src$$reg),
9528 (Assembler::Condition)$cmp$$cmpcode);
9529 %}
9530
9531 ins_pipe(icond_reg);
9532 %}
9533
9534 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9535 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9536
9537 ins_cost(INSN_COST * 2);
9538 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9539
9540 ins_encode %{
9541 __ csel(as_Register($dst$$reg),
9542 as_Register($src$$reg),
9543 zr,
9544 (Assembler::Condition)$cmp$$cmpcode);
9545 %}
9546
9547 ins_pipe(icond_reg);
9548 %}
9549
9550 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9551 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9552
9553 ins_cost(INSN_COST * 2);
9554 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9555
9556 ins_encode %{
9557 __ csel(as_Register($dst$$reg),
9558 as_Register($src$$reg),
9559 zr,
9560 (Assembler::Condition)$cmp$$cmpcode);
9561 %}
9562
9563 ins_pipe(icond_reg);
9564 %}
9565
9566 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9567 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9568
9569 ins_cost(INSN_COST * 2);
9570 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9571
9572 ins_encode %{
9573 __ csel(as_Register($dst$$reg),
9574 as_Register($src2$$reg),
9575 as_Register($src1$$reg),
9576 (Assembler::Condition)$cmp$$cmpcode);
9577 %}
9578
9579 ins_pipe(icond_reg_reg);
9580 %}
9581
9582 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9583 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9584
9585 ins_cost(INSN_COST * 2);
9586 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9587
9588 ins_encode %{
9589 __ csel(as_Register($dst$$reg),
9590 as_Register($src2$$reg),
9591 as_Register($src1$$reg),
9592 (Assembler::Condition)$cmp$$cmpcode);
9593 %}
9594
9595 ins_pipe(icond_reg_reg);
9596 %}
9597
9598 // special cases where one arg is zero
9599
9600 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9601 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9602
9603 ins_cost(INSN_COST * 2);
9604 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9605
9606 ins_encode %{
9607 __ csel(as_Register($dst$$reg),
9608 zr,
9609 as_Register($src$$reg),
9610 (Assembler::Condition)$cmp$$cmpcode);
9611 %}
9612
9613 ins_pipe(icond_reg);
9614 %}
9615
9616 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9617 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9618
9619 ins_cost(INSN_COST * 2);
9620 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9621
9622 ins_encode %{
9623 __ csel(as_Register($dst$$reg),
9624 zr,
9625 as_Register($src$$reg),
9626 (Assembler::Condition)$cmp$$cmpcode);
9627 %}
9628
9629 ins_pipe(icond_reg);
9630 %}
9631
9632 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9633 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9634
9635 ins_cost(INSN_COST * 2);
9636 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9637
9638 ins_encode %{
9639 __ csel(as_Register($dst$$reg),
9640 as_Register($src$$reg),
9641 zr,
9642 (Assembler::Condition)$cmp$$cmpcode);
9643 %}
9644
9645 ins_pipe(icond_reg);
9646 %}
9647
9648 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9649 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9650
9651 ins_cost(INSN_COST * 2);
9652 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9653
9654 ins_encode %{
9655 __ csel(as_Register($dst$$reg),
9656 as_Register($src$$reg),
9657 zr,
9658 (Assembler::Condition)$cmp$$cmpcode);
9659 %}
9660
9661 ins_pipe(icond_reg);
9662 %}
9663
9664 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9665 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9666
9667 ins_cost(INSN_COST * 2);
9668 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9669
9670 ins_encode %{
9671 __ cselw(as_Register($dst$$reg),
9672 as_Register($src2$$reg),
9673 as_Register($src1$$reg),
9674 (Assembler::Condition)$cmp$$cmpcode);
9675 %}
9676
9677 ins_pipe(icond_reg_reg);
9678 %}
9679
9680 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9681 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9682
9683 ins_cost(INSN_COST * 2);
9684 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9685
9686 ins_encode %{
9687 __ cselw(as_Register($dst$$reg),
9688 as_Register($src2$$reg),
9689 as_Register($src1$$reg),
9690 (Assembler::Condition)$cmp$$cmpcode);
9691 %}
9692
9693 ins_pipe(icond_reg_reg);
9694 %}
9695
9696 // special cases where one arg is zero
9697
9698 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9699 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9700
9701 ins_cost(INSN_COST * 2);
9702 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9703
9704 ins_encode %{
9705 __ cselw(as_Register($dst$$reg),
9706 zr,
9707 as_Register($src$$reg),
9708 (Assembler::Condition)$cmp$$cmpcode);
9709 %}
9710
9711 ins_pipe(icond_reg);
9712 %}
9713
9714 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9715 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9716
9717 ins_cost(INSN_COST * 2);
9718 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9719
9720 ins_encode %{
9721 __ cselw(as_Register($dst$$reg),
9722 zr,
9723 as_Register($src$$reg),
9724 (Assembler::Condition)$cmp$$cmpcode);
9725 %}
9726
9727 ins_pipe(icond_reg);
9728 %}
9729
9730 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9731 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9732
9733 ins_cost(INSN_COST * 2);
9734 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9735
9736 ins_encode %{
9737 __ cselw(as_Register($dst$$reg),
9738 as_Register($src$$reg),
9739 zr,
9740 (Assembler::Condition)$cmp$$cmpcode);
9741 %}
9742
9743 ins_pipe(icond_reg);
9744 %}
9745
9746 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9747 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9748
9749 ins_cost(INSN_COST * 2);
9750 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9751
9752 ins_encode %{
9753 __ cselw(as_Register($dst$$reg),
9754 as_Register($src$$reg),
9755 zr,
9756 (Assembler::Condition)$cmp$$cmpcode);
9757 %}
9758
9759 ins_pipe(icond_reg);
9760 %}
9761
9762 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9763 %{
9764 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9765
9766 ins_cost(INSN_COST * 3);
9767
9768 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9769 ins_encode %{
9770 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9771 __ fcsels(as_FloatRegister($dst$$reg),
9772 as_FloatRegister($src2$$reg),
9773 as_FloatRegister($src1$$reg),
9774 cond);
9775 %}
9776
9777 ins_pipe(fp_cond_reg_reg_s);
9778 %}
9779
9780 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9781 %{
9782 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9783
9784 ins_cost(INSN_COST * 3);
9785
9786 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9787 ins_encode %{
9788 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9789 __ fcsels(as_FloatRegister($dst$$reg),
9790 as_FloatRegister($src2$$reg),
9791 as_FloatRegister($src1$$reg),
9792 cond);
9793 %}
9794
9795 ins_pipe(fp_cond_reg_reg_s);
9796 %}
9797
9798 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9799 %{
9800 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9801
9802 ins_cost(INSN_COST * 3);
9803
9804 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9805 ins_encode %{
9806 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9807 __ fcseld(as_FloatRegister($dst$$reg),
9808 as_FloatRegister($src2$$reg),
9809 as_FloatRegister($src1$$reg),
9810 cond);
9811 %}
9812
9813 ins_pipe(fp_cond_reg_reg_d);
9814 %}
9815
9816 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9817 %{
9818 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9819
9820 ins_cost(INSN_COST * 3);
9821
9822 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9823 ins_encode %{
9824 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9825 __ fcseld(as_FloatRegister($dst$$reg),
9826 as_FloatRegister($src2$$reg),
9827 as_FloatRegister($src1$$reg),
9828 cond);
9829 %}
9830
9831 ins_pipe(fp_cond_reg_reg_d);
9832 %}
9833
9834 // ============================================================================
9835 // Arithmetic Instructions
9836 //
9837
9838 // Integer Addition
9839
9840 // TODO
9841 // these currently employ operations which do not set CR and hence are
9842 // not flagged as killing CR but we would like to isolate the cases
9843 // where we want to set flags from those where we don't. need to work
9844 // out how to do that.
9845
9846 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9847 match(Set dst (AddI src1 src2));
9848
9849 ins_cost(INSN_COST);
9850 format %{ "addw $dst, $src1, $src2" %}
9851
9852 ins_encode %{
9853 __ addw(as_Register($dst$$reg),
9854 as_Register($src1$$reg),
9855 as_Register($src2$$reg));
9856 %}
9857
9858 ins_pipe(ialu_reg_reg);
9859 %}
9860
9861 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9862 match(Set dst (AddI src1 src2));
9863
9864 ins_cost(INSN_COST);
9865 format %{ "addw $dst, $src1, $src2" %}
9866
9867 // use opcode to indicate that this is an add not a sub
9868 opcode(0x0);
9869
9870 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9871
9872 ins_pipe(ialu_reg_imm);
9873 %}
9874
9875 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9876 match(Set dst (AddI (ConvL2I src1) src2));
9877
9878 ins_cost(INSN_COST);
9879 format %{ "addw $dst, $src1, $src2" %}
9880
9881 // use opcode to indicate that this is an add not a sub
9882 opcode(0x0);
9883
9884 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9885
9886 ins_pipe(ialu_reg_imm);
9887 %}
9888
9889 // Pointer Addition
9890 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9891 match(Set dst (AddP src1 src2));
9892
9893 ins_cost(INSN_COST);
9894 format %{ "add $dst, $src1, $src2\t# ptr" %}
9895
9896 ins_encode %{
9897 __ add(as_Register($dst$$reg),
9898 as_Register($src1$$reg),
9899 as_Register($src2$$reg));
9900 %}
9901
9902 ins_pipe(ialu_reg_reg);
9903 %}
9904
9905 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9906 match(Set dst (AddP src1 (ConvI2L src2)));
9907
9908 ins_cost(1.9 * INSN_COST);
9909 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9910
9911 ins_encode %{
9912 __ add(as_Register($dst$$reg),
9913 as_Register($src1$$reg),
9914 as_Register($src2$$reg), ext::sxtw);
9915 %}
9916
9917 ins_pipe(ialu_reg_reg);
9918 %}
9919
9920 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9921 match(Set dst (AddP src1 (LShiftL src2 scale)));
9922
9923 ins_cost(1.9 * INSN_COST);
9924 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9925
9926 ins_encode %{
9927 __ lea(as_Register($dst$$reg),
9928 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9929 Address::lsl($scale$$constant)));
9930 %}
9931
9932 ins_pipe(ialu_reg_reg_shift);
9933 %}
9934
9935 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9936 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9937
9938 ins_cost(1.9 * INSN_COST);
9939 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9940
9941 ins_encode %{
9942 __ lea(as_Register($dst$$reg),
9943 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9944 Address::sxtw($scale$$constant)));
9945 %}
9946
9947 ins_pipe(ialu_reg_reg_shift);
9948 %}
9949
9950 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9951 match(Set dst (LShiftL (ConvI2L src) scale));
9952
9953 ins_cost(INSN_COST);
9954 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9955
9956 ins_encode %{
9957 __ sbfiz(as_Register($dst$$reg),
9958 as_Register($src$$reg),
9959 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9960 %}
9961
9962 ins_pipe(ialu_reg_shift);
9963 %}
9964
9965 // Pointer Immediate Addition
9966 // n.b. this needs to be more expensive than using an indirect memory
9967 // operand
9968 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9969 match(Set dst (AddP src1 src2));
9970
9971 ins_cost(INSN_COST);
9972 format %{ "add $dst, $src1, $src2\t# ptr" %}
9973
9974 // use opcode to indicate that this is an add not a sub
9975 opcode(0x0);
9976
9977 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9978
9979 ins_pipe(ialu_reg_imm);
9980 %}
9981
9982 // Long Addition
9983 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9984
9985 match(Set dst (AddL src1 src2));
9986
9987 ins_cost(INSN_COST);
9988 format %{ "add $dst, $src1, $src2" %}
9989
9990 ins_encode %{
9991 __ add(as_Register($dst$$reg),
9992 as_Register($src1$$reg),
9993 as_Register($src2$$reg));
9994 %}
9995
9996 ins_pipe(ialu_reg_reg);
9997 %}
9998
9999 // No constant pool entries requiredLong Immediate Addition.
10000 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10001 match(Set dst (AddL src1 src2));
10002
10003 ins_cost(INSN_COST);
10004 format %{ "add $dst, $src1, $src2" %}
10005
10006 // use opcode to indicate that this is an add not a sub
10007 opcode(0x0);
10008
10009 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10010
10011 ins_pipe(ialu_reg_imm);
10012 %}
10013
10014 // Integer Subtraction
10015 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10016 match(Set dst (SubI src1 src2));
10017
10018 ins_cost(INSN_COST);
10019 format %{ "subw $dst, $src1, $src2" %}
10020
10021 ins_encode %{
10022 __ subw(as_Register($dst$$reg),
10023 as_Register($src1$$reg),
10024 as_Register($src2$$reg));
10025 %}
10026
10027 ins_pipe(ialu_reg_reg);
10028 %}
10029
10030 // Immediate Subtraction
10031 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10032 match(Set dst (SubI src1 src2));
10033
10034 ins_cost(INSN_COST);
10035 format %{ "subw $dst, $src1, $src2" %}
10036
10037 // use opcode to indicate that this is a sub not an add
10038 opcode(0x1);
10039
10040 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10041
10042 ins_pipe(ialu_reg_imm);
10043 %}
10044
10045 // Long Subtraction
10046 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10047
10048 match(Set dst (SubL src1 src2));
10049
10050 ins_cost(INSN_COST);
10051 format %{ "sub $dst, $src1, $src2" %}
10052
10053 ins_encode %{
10054 __ sub(as_Register($dst$$reg),
10055 as_Register($src1$$reg),
10056 as_Register($src2$$reg));
10057 %}
10058
10059 ins_pipe(ialu_reg_reg);
10060 %}
10061
10062 // No constant pool entries requiredLong Immediate Subtraction.
10063 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10064 match(Set dst (SubL src1 src2));
10065
10066 ins_cost(INSN_COST);
10067 format %{ "sub$dst, $src1, $src2" %}
10068
10069 // use opcode to indicate that this is a sub not an add
10070 opcode(0x1);
10071
10072 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10073
10074 ins_pipe(ialu_reg_imm);
10075 %}
10076
10077 // Integer Negation (special case for sub)
10078
10079 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10080 match(Set dst (SubI zero src));
10081
10082 ins_cost(INSN_COST);
10083 format %{ "negw $dst, $src\t# int" %}
10084
10085 ins_encode %{
10086 __ negw(as_Register($dst$$reg),
10087 as_Register($src$$reg));
10088 %}
10089
10090 ins_pipe(ialu_reg);
10091 %}
10092
10093 // Long Negation
10094
10095 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10096 match(Set dst (SubL zero src));
10097
10098 ins_cost(INSN_COST);
10099 format %{ "neg $dst, $src\t# long" %}
10100
10101 ins_encode %{
10102 __ neg(as_Register($dst$$reg),
10103 as_Register($src$$reg));
10104 %}
10105
10106 ins_pipe(ialu_reg);
10107 %}
10108
10109 // Integer Multiply
10110
10111 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10112 match(Set dst (MulI src1 src2));
10113
10114 ins_cost(INSN_COST * 3);
10115 format %{ "mulw $dst, $src1, $src2" %}
10116
10117 ins_encode %{
10118 __ mulw(as_Register($dst$$reg),
10119 as_Register($src1$$reg),
10120 as_Register($src2$$reg));
10121 %}
10122
10123 ins_pipe(imul_reg_reg);
10124 %}
10125
10126 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10127 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10128
10129 ins_cost(INSN_COST * 3);
10130 format %{ "smull $dst, $src1, $src2" %}
10131
10132 ins_encode %{
10133 __ smull(as_Register($dst$$reg),
10134 as_Register($src1$$reg),
10135 as_Register($src2$$reg));
10136 %}
10137
10138 ins_pipe(imul_reg_reg);
10139 %}
10140
10141 // Long Multiply
10142
10143 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10144 match(Set dst (MulL src1 src2));
10145
10146 ins_cost(INSN_COST * 5);
10147 format %{ "mul $dst, $src1, $src2" %}
10148
10149 ins_encode %{
10150 __ mul(as_Register($dst$$reg),
10151 as_Register($src1$$reg),
10152 as_Register($src2$$reg));
10153 %}
10154
10155 ins_pipe(lmul_reg_reg);
10156 %}
10157
10158 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10159 %{
10160 match(Set dst (MulHiL src1 src2));
10161
10162 ins_cost(INSN_COST * 7);
10163 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10164
10165 ins_encode %{
10166 __ smulh(as_Register($dst$$reg),
10167 as_Register($src1$$reg),
10168 as_Register($src2$$reg));
10169 %}
10170
10171 ins_pipe(lmul_reg_reg);
10172 %}
10173
10174 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10175 %{
10176 match(Set dst (UMulHiL src1 src2));
10177
10178 ins_cost(INSN_COST * 7);
10179 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10180
10181 ins_encode %{
10182 __ umulh(as_Register($dst$$reg),
10183 as_Register($src1$$reg),
10184 as_Register($src2$$reg));
10185 %}
10186
10187 ins_pipe(lmul_reg_reg);
10188 %}
10189
10190 // Combined Integer Multiply & Add/Sub
10191
10192 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10193 match(Set dst (AddI src3 (MulI src1 src2)));
10194
10195 ins_cost(INSN_COST * 3);
10196 format %{ "madd $dst, $src1, $src2, $src3" %}
10197
10198 ins_encode %{
10199 __ maddw(as_Register($dst$$reg),
10200 as_Register($src1$$reg),
10201 as_Register($src2$$reg),
10202 as_Register($src3$$reg));
10203 %}
10204
10205 ins_pipe(imac_reg_reg);
10206 %}
10207
10208 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10209 match(Set dst (SubI src3 (MulI src1 src2)));
10210
10211 ins_cost(INSN_COST * 3);
10212 format %{ "msub $dst, $src1, $src2, $src3" %}
10213
10214 ins_encode %{
10215 __ msubw(as_Register($dst$$reg),
10216 as_Register($src1$$reg),
10217 as_Register($src2$$reg),
10218 as_Register($src3$$reg));
10219 %}
10220
10221 ins_pipe(imac_reg_reg);
10222 %}
10223
10224 // Combined Integer Multiply & Neg
10225
10226 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10227 match(Set dst (MulI (SubI zero src1) src2));
10228
10229 ins_cost(INSN_COST * 3);
10230 format %{ "mneg $dst, $src1, $src2" %}
10231
10232 ins_encode %{
10233 __ mnegw(as_Register($dst$$reg),
10234 as_Register($src1$$reg),
10235 as_Register($src2$$reg));
10236 %}
10237
10238 ins_pipe(imac_reg_reg);
10239 %}
10240
10241 // Combined Long Multiply & Add/Sub
10242
10243 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10244 match(Set dst (AddL src3 (MulL src1 src2)));
10245
10246 ins_cost(INSN_COST * 5);
10247 format %{ "madd $dst, $src1, $src2, $src3" %}
10248
10249 ins_encode %{
10250 __ madd(as_Register($dst$$reg),
10251 as_Register($src1$$reg),
10252 as_Register($src2$$reg),
10253 as_Register($src3$$reg));
10254 %}
10255
10256 ins_pipe(lmac_reg_reg);
10257 %}
10258
10259 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10260 match(Set dst (SubL src3 (MulL src1 src2)));
10261
10262 ins_cost(INSN_COST * 5);
10263 format %{ "msub $dst, $src1, $src2, $src3" %}
10264
10265 ins_encode %{
10266 __ msub(as_Register($dst$$reg),
10267 as_Register($src1$$reg),
10268 as_Register($src2$$reg),
10269 as_Register($src3$$reg));
10270 %}
10271
10272 ins_pipe(lmac_reg_reg);
10273 %}
10274
10275 // Combined Long Multiply & Neg
10276
10277 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10278 match(Set dst (MulL (SubL zero src1) src2));
10279
10280 ins_cost(INSN_COST * 5);
10281 format %{ "mneg $dst, $src1, $src2" %}
10282
10283 ins_encode %{
10284 __ mneg(as_Register($dst$$reg),
10285 as_Register($src1$$reg),
10286 as_Register($src2$$reg));
10287 %}
10288
10289 ins_pipe(lmac_reg_reg);
10290 %}
10291
10292 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10293
10294 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10295 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10296
10297 ins_cost(INSN_COST * 3);
10298 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10299
10300 ins_encode %{
10301 __ smaddl(as_Register($dst$$reg),
10302 as_Register($src1$$reg),
10303 as_Register($src2$$reg),
10304 as_Register($src3$$reg));
10305 %}
10306
10307 ins_pipe(imac_reg_reg);
10308 %}
10309
10310 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10311 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10312
10313 ins_cost(INSN_COST * 3);
10314 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10315
10316 ins_encode %{
10317 __ smsubl(as_Register($dst$$reg),
10318 as_Register($src1$$reg),
10319 as_Register($src2$$reg),
10320 as_Register($src3$$reg));
10321 %}
10322
10323 ins_pipe(imac_reg_reg);
10324 %}
10325
10326 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10327 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10328
10329 ins_cost(INSN_COST * 3);
10330 format %{ "smnegl $dst, $src1, $src2" %}
10331
10332 ins_encode %{
10333 __ smnegl(as_Register($dst$$reg),
10334 as_Register($src1$$reg),
10335 as_Register($src2$$reg));
10336 %}
10337
10338 ins_pipe(imac_reg_reg);
10339 %}
10340
10341 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10342
10343 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10344 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10345
10346 ins_cost(INSN_COST * 5);
10347 format %{ "mulw rscratch1, $src1, $src2\n\t"
10348 "maddw $dst, $src3, $src4, rscratch1" %}
10349
10350 ins_encode %{
10351 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10352 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10353
10354 ins_pipe(imac_reg_reg);
10355 %}
10356
10357 // Integer Divide
10358
10359 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10360 match(Set dst (DivI src1 src2));
10361
10362 ins_cost(INSN_COST * 19);
10363 format %{ "sdivw $dst, $src1, $src2" %}
10364
10365 ins_encode(aarch64_enc_divw(dst, src1, src2));
10366 ins_pipe(idiv_reg_reg);
10367 %}
10368
10369 // Long Divide
10370
10371 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10372 match(Set dst (DivL src1 src2));
10373
10374 ins_cost(INSN_COST * 35);
10375 format %{ "sdiv $dst, $src1, $src2" %}
10376
10377 ins_encode(aarch64_enc_div(dst, src1, src2));
10378 ins_pipe(ldiv_reg_reg);
10379 %}
10380
10381 // Integer Remainder
10382
10383 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10384 match(Set dst (ModI src1 src2));
10385
10386 ins_cost(INSN_COST * 22);
10387 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10388 "msubw $dst, rscratch1, $src2, $src1" %}
10389
10390 ins_encode(aarch64_enc_modw(dst, src1, src2));
10391 ins_pipe(idiv_reg_reg);
10392 %}
10393
10394 // Long Remainder
10395
10396 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10397 match(Set dst (ModL src1 src2));
10398
10399 ins_cost(INSN_COST * 38);
10400 format %{ "sdiv rscratch1, $src1, $src2\n"
10401 "msub $dst, rscratch1, $src2, $src1" %}
10402
10403 ins_encode(aarch64_enc_mod(dst, src1, src2));
10404 ins_pipe(ldiv_reg_reg);
10405 %}
10406
10407 // Unsigned Integer Divide
10408
10409 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10410 match(Set dst (UDivI src1 src2));
10411
10412 ins_cost(INSN_COST * 19);
10413 format %{ "udivw $dst, $src1, $src2" %}
10414
10415 ins_encode %{
10416 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10417 %}
10418
10419 ins_pipe(idiv_reg_reg);
10420 %}
10421
10422 // Unsigned Long Divide
10423
10424 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10425 match(Set dst (UDivL src1 src2));
10426
10427 ins_cost(INSN_COST * 35);
10428 format %{ "udiv $dst, $src1, $src2" %}
10429
10430 ins_encode %{
10431 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10432 %}
10433
10434 ins_pipe(ldiv_reg_reg);
10435 %}
10436
10437 // Unsigned Integer Remainder
10438
10439 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10440 match(Set dst (UModI src1 src2));
10441
10442 ins_cost(INSN_COST * 22);
10443 format %{ "udivw rscratch1, $src1, $src2\n\t"
10444 "msubw $dst, rscratch1, $src2, $src1" %}
10445
10446 ins_encode %{
10447 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10448 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10449 %}
10450
10451 ins_pipe(idiv_reg_reg);
10452 %}
10453
10454 // Unsigned Long Remainder
10455
10456 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10457 match(Set dst (UModL src1 src2));
10458
10459 ins_cost(INSN_COST * 38);
10460 format %{ "udiv rscratch1, $src1, $src2\n"
10461 "msub $dst, rscratch1, $src2, $src1" %}
10462
10463 ins_encode %{
10464 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10465 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10466 %}
10467
10468 ins_pipe(ldiv_reg_reg);
10469 %}
10470
10471 // Integer Shifts
10472
10473 // Shift Left Register
10474 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10475 match(Set dst (LShiftI src1 src2));
10476
10477 ins_cost(INSN_COST * 2);
10478 format %{ "lslvw $dst, $src1, $src2" %}
10479
10480 ins_encode %{
10481 __ lslvw(as_Register($dst$$reg),
10482 as_Register($src1$$reg),
10483 as_Register($src2$$reg));
10484 %}
10485
10486 ins_pipe(ialu_reg_reg_vshift);
10487 %}
10488
10489 // Shift Left Immediate
10490 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10491 match(Set dst (LShiftI src1 src2));
10492
10493 ins_cost(INSN_COST);
10494 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10495
10496 ins_encode %{
10497 __ lslw(as_Register($dst$$reg),
10498 as_Register($src1$$reg),
10499 $src2$$constant & 0x1f);
10500 %}
10501
10502 ins_pipe(ialu_reg_shift);
10503 %}
10504
10505 // Shift Right Logical Register
10506 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10507 match(Set dst (URShiftI src1 src2));
10508
10509 ins_cost(INSN_COST * 2);
10510 format %{ "lsrvw $dst, $src1, $src2" %}
10511
10512 ins_encode %{
10513 __ lsrvw(as_Register($dst$$reg),
10514 as_Register($src1$$reg),
10515 as_Register($src2$$reg));
10516 %}
10517
10518 ins_pipe(ialu_reg_reg_vshift);
10519 %}
10520
10521 // Shift Right Logical Immediate
10522 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10523 match(Set dst (URShiftI src1 src2));
10524
10525 ins_cost(INSN_COST);
10526 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10527
10528 ins_encode %{
10529 __ lsrw(as_Register($dst$$reg),
10530 as_Register($src1$$reg),
10531 $src2$$constant & 0x1f);
10532 %}
10533
10534 ins_pipe(ialu_reg_shift);
10535 %}
10536
10537 // Shift Right Arithmetic Register
10538 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10539 match(Set dst (RShiftI src1 src2));
10540
10541 ins_cost(INSN_COST * 2);
10542 format %{ "asrvw $dst, $src1, $src2" %}
10543
10544 ins_encode %{
10545 __ asrvw(as_Register($dst$$reg),
10546 as_Register($src1$$reg),
10547 as_Register($src2$$reg));
10548 %}
10549
10550 ins_pipe(ialu_reg_reg_vshift);
10551 %}
10552
10553 // Shift Right Arithmetic Immediate
10554 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10555 match(Set dst (RShiftI src1 src2));
10556
10557 ins_cost(INSN_COST);
10558 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10559
10560 ins_encode %{
10561 __ asrw(as_Register($dst$$reg),
10562 as_Register($src1$$reg),
10563 $src2$$constant & 0x1f);
10564 %}
10565
10566 ins_pipe(ialu_reg_shift);
10567 %}
10568
10569 // Combined Int Mask and Right Shift (using UBFM)
10570 // TODO
10571
10572 // Long Shifts
10573
10574 // Shift Left Register
10575 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10576 match(Set dst (LShiftL src1 src2));
10577
10578 ins_cost(INSN_COST * 2);
10579 format %{ "lslv $dst, $src1, $src2" %}
10580
10581 ins_encode %{
10582 __ lslv(as_Register($dst$$reg),
10583 as_Register($src1$$reg),
10584 as_Register($src2$$reg));
10585 %}
10586
10587 ins_pipe(ialu_reg_reg_vshift);
10588 %}
10589
10590 // Shift Left Immediate
10591 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10592 match(Set dst (LShiftL src1 src2));
10593
10594 ins_cost(INSN_COST);
10595 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10596
10597 ins_encode %{
10598 __ lsl(as_Register($dst$$reg),
10599 as_Register($src1$$reg),
10600 $src2$$constant & 0x3f);
10601 %}
10602
10603 ins_pipe(ialu_reg_shift);
10604 %}
10605
10606 // Shift Right Logical Register
10607 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10608 match(Set dst (URShiftL src1 src2));
10609
10610 ins_cost(INSN_COST * 2);
10611 format %{ "lsrv $dst, $src1, $src2" %}
10612
10613 ins_encode %{
10614 __ lsrv(as_Register($dst$$reg),
10615 as_Register($src1$$reg),
10616 as_Register($src2$$reg));
10617 %}
10618
10619 ins_pipe(ialu_reg_reg_vshift);
10620 %}
10621
10622 // Shift Right Logical Immediate
10623 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10624 match(Set dst (URShiftL src1 src2));
10625
10626 ins_cost(INSN_COST);
10627 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10628
10629 ins_encode %{
10630 __ lsr(as_Register($dst$$reg),
10631 as_Register($src1$$reg),
10632 $src2$$constant & 0x3f);
10633 %}
10634
10635 ins_pipe(ialu_reg_shift);
10636 %}
10637
10638 // A special-case pattern for card table stores.
10639 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10640 match(Set dst (URShiftL (CastP2X src1) src2));
10641
10642 ins_cost(INSN_COST);
10643 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10644
10645 ins_encode %{
10646 __ lsr(as_Register($dst$$reg),
10647 as_Register($src1$$reg),
10648 $src2$$constant & 0x3f);
10649 %}
10650
10651 ins_pipe(ialu_reg_shift);
10652 %}
10653
10654 // Shift Right Arithmetic Register
10655 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10656 match(Set dst (RShiftL src1 src2));
10657
10658 ins_cost(INSN_COST * 2);
10659 format %{ "asrv $dst, $src1, $src2" %}
10660
10661 ins_encode %{
10662 __ asrv(as_Register($dst$$reg),
10663 as_Register($src1$$reg),
10664 as_Register($src2$$reg));
10665 %}
10666
10667 ins_pipe(ialu_reg_reg_vshift);
10668 %}
10669
10670 // Shift Right Arithmetic Immediate
10671 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10672 match(Set dst (RShiftL src1 src2));
10673
10674 ins_cost(INSN_COST);
10675 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10676
10677 ins_encode %{
10678 __ asr(as_Register($dst$$reg),
10679 as_Register($src1$$reg),
10680 $src2$$constant & 0x3f);
10681 %}
10682
10683 ins_pipe(ialu_reg_shift);
10684 %}
10685
10686 // BEGIN This section of the file is automatically generated. Do not edit --------------
10687 // This section is generated from aarch64_ad.m4
10688
10689 // This pattern is automatically generated from aarch64_ad.m4
10690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10691 instruct regL_not_reg(iRegLNoSp dst,
10692 iRegL src1, immL_M1 m1,
10693 rFlagsReg cr) %{
10694 match(Set dst (XorL src1 m1));
10695 ins_cost(INSN_COST);
10696 format %{ "eon $dst, $src1, zr" %}
10697
10698 ins_encode %{
10699 __ eon(as_Register($dst$$reg),
10700 as_Register($src1$$reg),
10701 zr,
10702 Assembler::LSL, 0);
10703 %}
10704
10705 ins_pipe(ialu_reg);
10706 %}
10707
10708 // This pattern is automatically generated from aarch64_ad.m4
10709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10710 instruct regI_not_reg(iRegINoSp dst,
10711 iRegIorL2I src1, immI_M1 m1,
10712 rFlagsReg cr) %{
10713 match(Set dst (XorI src1 m1));
10714 ins_cost(INSN_COST);
10715 format %{ "eonw $dst, $src1, zr" %}
10716
10717 ins_encode %{
10718 __ eonw(as_Register($dst$$reg),
10719 as_Register($src1$$reg),
10720 zr,
10721 Assembler::LSL, 0);
10722 %}
10723
10724 ins_pipe(ialu_reg);
10725 %}
10726
10727 // This pattern is automatically generated from aarch64_ad.m4
10728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10729 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10730 immI0 zero, iRegIorL2I src1, immI src2) %{
10731 match(Set dst (SubI zero (URShiftI src1 src2)));
10732
10733 ins_cost(1.9 * INSN_COST);
10734 format %{ "negw $dst, $src1, LSR $src2" %}
10735
10736 ins_encode %{
10737 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10738 Assembler::LSR, $src2$$constant & 0x1f);
10739 %}
10740
10741 ins_pipe(ialu_reg_shift);
10742 %}
10743
10744 // This pattern is automatically generated from aarch64_ad.m4
10745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10746 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10747 immI0 zero, iRegIorL2I src1, immI src2) %{
10748 match(Set dst (SubI zero (RShiftI src1 src2)));
10749
10750 ins_cost(1.9 * INSN_COST);
10751 format %{ "negw $dst, $src1, ASR $src2" %}
10752
10753 ins_encode %{
10754 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10755 Assembler::ASR, $src2$$constant & 0x1f);
10756 %}
10757
10758 ins_pipe(ialu_reg_shift);
10759 %}
10760
10761 // This pattern is automatically generated from aarch64_ad.m4
10762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10763 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10764 immI0 zero, iRegIorL2I src1, immI src2) %{
10765 match(Set dst (SubI zero (LShiftI src1 src2)));
10766
10767 ins_cost(1.9 * INSN_COST);
10768 format %{ "negw $dst, $src1, LSL $src2" %}
10769
10770 ins_encode %{
10771 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10772 Assembler::LSL, $src2$$constant & 0x1f);
10773 %}
10774
10775 ins_pipe(ialu_reg_shift);
10776 %}
10777
10778 // This pattern is automatically generated from aarch64_ad.m4
10779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10780 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10781 immL0 zero, iRegL src1, immI src2) %{
10782 match(Set dst (SubL zero (URShiftL src1 src2)));
10783
10784 ins_cost(1.9 * INSN_COST);
10785 format %{ "neg $dst, $src1, LSR $src2" %}
10786
10787 ins_encode %{
10788 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10789 Assembler::LSR, $src2$$constant & 0x3f);
10790 %}
10791
10792 ins_pipe(ialu_reg_shift);
10793 %}
10794
10795 // This pattern is automatically generated from aarch64_ad.m4
10796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10797 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10798 immL0 zero, iRegL src1, immI src2) %{
10799 match(Set dst (SubL zero (RShiftL src1 src2)));
10800
10801 ins_cost(1.9 * INSN_COST);
10802 format %{ "neg $dst, $src1, ASR $src2" %}
10803
10804 ins_encode %{
10805 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10806 Assembler::ASR, $src2$$constant & 0x3f);
10807 %}
10808
10809 ins_pipe(ialu_reg_shift);
10810 %}
10811
10812 // This pattern is automatically generated from aarch64_ad.m4
10813 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10814 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10815 immL0 zero, iRegL src1, immI src2) %{
10816 match(Set dst (SubL zero (LShiftL src1 src2)));
10817
10818 ins_cost(1.9 * INSN_COST);
10819 format %{ "neg $dst, $src1, LSL $src2" %}
10820
10821 ins_encode %{
10822 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10823 Assembler::LSL, $src2$$constant & 0x3f);
10824 %}
10825
10826 ins_pipe(ialu_reg_shift);
10827 %}
10828
10829 // This pattern is automatically generated from aarch64_ad.m4
10830 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10831 instruct AndI_reg_not_reg(iRegINoSp dst,
10832 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10833 match(Set dst (AndI src1 (XorI src2 m1)));
10834 ins_cost(INSN_COST);
10835 format %{ "bicw $dst, $src1, $src2" %}
10836
10837 ins_encode %{
10838 __ bicw(as_Register($dst$$reg),
10839 as_Register($src1$$reg),
10840 as_Register($src2$$reg),
10841 Assembler::LSL, 0);
10842 %}
10843
10844 ins_pipe(ialu_reg_reg);
10845 %}
10846
10847 // This pattern is automatically generated from aarch64_ad.m4
10848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10849 instruct AndL_reg_not_reg(iRegLNoSp dst,
10850 iRegL src1, iRegL src2, immL_M1 m1) %{
10851 match(Set dst (AndL src1 (XorL src2 m1)));
10852 ins_cost(INSN_COST);
10853 format %{ "bic $dst, $src1, $src2" %}
10854
10855 ins_encode %{
10856 __ bic(as_Register($dst$$reg),
10857 as_Register($src1$$reg),
10858 as_Register($src2$$reg),
10859 Assembler::LSL, 0);
10860 %}
10861
10862 ins_pipe(ialu_reg_reg);
10863 %}
10864
10865 // This pattern is automatically generated from aarch64_ad.m4
10866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10867 instruct OrI_reg_not_reg(iRegINoSp dst,
10868 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10869 match(Set dst (OrI src1 (XorI src2 m1)));
10870 ins_cost(INSN_COST);
10871 format %{ "ornw $dst, $src1, $src2" %}
10872
10873 ins_encode %{
10874 __ ornw(as_Register($dst$$reg),
10875 as_Register($src1$$reg),
10876 as_Register($src2$$reg),
10877 Assembler::LSL, 0);
10878 %}
10879
10880 ins_pipe(ialu_reg_reg);
10881 %}
10882
10883 // This pattern is automatically generated from aarch64_ad.m4
10884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10885 instruct OrL_reg_not_reg(iRegLNoSp dst,
10886 iRegL src1, iRegL src2, immL_M1 m1) %{
10887 match(Set dst (OrL src1 (XorL src2 m1)));
10888 ins_cost(INSN_COST);
10889 format %{ "orn $dst, $src1, $src2" %}
10890
10891 ins_encode %{
10892 __ orn(as_Register($dst$$reg),
10893 as_Register($src1$$reg),
10894 as_Register($src2$$reg),
10895 Assembler::LSL, 0);
10896 %}
10897
10898 ins_pipe(ialu_reg_reg);
10899 %}
10900
10901 // This pattern is automatically generated from aarch64_ad.m4
10902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10903 instruct XorI_reg_not_reg(iRegINoSp dst,
10904 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10905 match(Set dst (XorI m1 (XorI src2 src1)));
10906 ins_cost(INSN_COST);
10907 format %{ "eonw $dst, $src1, $src2" %}
10908
10909 ins_encode %{
10910 __ eonw(as_Register($dst$$reg),
10911 as_Register($src1$$reg),
10912 as_Register($src2$$reg),
10913 Assembler::LSL, 0);
10914 %}
10915
10916 ins_pipe(ialu_reg_reg);
10917 %}
10918
10919 // This pattern is automatically generated from aarch64_ad.m4
10920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10921 instruct XorL_reg_not_reg(iRegLNoSp dst,
10922 iRegL src1, iRegL src2, immL_M1 m1) %{
10923 match(Set dst (XorL m1 (XorL src2 src1)));
10924 ins_cost(INSN_COST);
10925 format %{ "eon $dst, $src1, $src2" %}
10926
10927 ins_encode %{
10928 __ eon(as_Register($dst$$reg),
10929 as_Register($src1$$reg),
10930 as_Register($src2$$reg),
10931 Assembler::LSL, 0);
10932 %}
10933
10934 ins_pipe(ialu_reg_reg);
10935 %}
10936
10937 // This pattern is automatically generated from aarch64_ad.m4
10938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10939 // val & (-1 ^ (val >>> shift)) ==> bicw
10940 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10941 iRegIorL2I src1, iRegIorL2I src2,
10942 immI src3, immI_M1 src4) %{
10943 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10944 ins_cost(1.9 * INSN_COST);
10945 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10946
10947 ins_encode %{
10948 __ bicw(as_Register($dst$$reg),
10949 as_Register($src1$$reg),
10950 as_Register($src2$$reg),
10951 Assembler::LSR,
10952 $src3$$constant & 0x1f);
10953 %}
10954
10955 ins_pipe(ialu_reg_reg_shift);
10956 %}
10957
10958 // This pattern is automatically generated from aarch64_ad.m4
10959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10960 // val & (-1 ^ (val >>> shift)) ==> bic
10961 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10962 iRegL src1, iRegL src2,
10963 immI src3, immL_M1 src4) %{
10964 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10965 ins_cost(1.9 * INSN_COST);
10966 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10967
10968 ins_encode %{
10969 __ bic(as_Register($dst$$reg),
10970 as_Register($src1$$reg),
10971 as_Register($src2$$reg),
10972 Assembler::LSR,
10973 $src3$$constant & 0x3f);
10974 %}
10975
10976 ins_pipe(ialu_reg_reg_shift);
10977 %}
10978
10979 // This pattern is automatically generated from aarch64_ad.m4
10980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10981 // val & (-1 ^ (val >> shift)) ==> bicw
10982 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10983 iRegIorL2I src1, iRegIorL2I src2,
10984 immI src3, immI_M1 src4) %{
10985 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10986 ins_cost(1.9 * INSN_COST);
10987 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10988
10989 ins_encode %{
10990 __ bicw(as_Register($dst$$reg),
10991 as_Register($src1$$reg),
10992 as_Register($src2$$reg),
10993 Assembler::ASR,
10994 $src3$$constant & 0x1f);
10995 %}
10996
10997 ins_pipe(ialu_reg_reg_shift);
10998 %}
10999
11000 // This pattern is automatically generated from aarch64_ad.m4
11001 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11002 // val & (-1 ^ (val >> shift)) ==> bic
11003 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11004 iRegL src1, iRegL src2,
11005 immI src3, immL_M1 src4) %{
11006 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11007 ins_cost(1.9 * INSN_COST);
11008 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11009
11010 ins_encode %{
11011 __ bic(as_Register($dst$$reg),
11012 as_Register($src1$$reg),
11013 as_Register($src2$$reg),
11014 Assembler::ASR,
11015 $src3$$constant & 0x3f);
11016 %}
11017
11018 ins_pipe(ialu_reg_reg_shift);
11019 %}
11020
11021 // This pattern is automatically generated from aarch64_ad.m4
11022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11023 // val & (-1 ^ (val ror shift)) ==> bicw
11024 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11025 iRegIorL2I src1, iRegIorL2I src2,
11026 immI src3, immI_M1 src4) %{
11027 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11028 ins_cost(1.9 * INSN_COST);
11029 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11030
11031 ins_encode %{
11032 __ bicw(as_Register($dst$$reg),
11033 as_Register($src1$$reg),
11034 as_Register($src2$$reg),
11035 Assembler::ROR,
11036 $src3$$constant & 0x1f);
11037 %}
11038
11039 ins_pipe(ialu_reg_reg_shift);
11040 %}
11041
11042 // This pattern is automatically generated from aarch64_ad.m4
11043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11044 // val & (-1 ^ (val ror shift)) ==> bic
11045 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11046 iRegL src1, iRegL src2,
11047 immI src3, immL_M1 src4) %{
11048 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11049 ins_cost(1.9 * INSN_COST);
11050 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11051
11052 ins_encode %{
11053 __ bic(as_Register($dst$$reg),
11054 as_Register($src1$$reg),
11055 as_Register($src2$$reg),
11056 Assembler::ROR,
11057 $src3$$constant & 0x3f);
11058 %}
11059
11060 ins_pipe(ialu_reg_reg_shift);
11061 %}
11062
11063 // This pattern is automatically generated from aarch64_ad.m4
11064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11065 // val & (-1 ^ (val << shift)) ==> bicw
11066 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11067 iRegIorL2I src1, iRegIorL2I src2,
11068 immI src3, immI_M1 src4) %{
11069 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11070 ins_cost(1.9 * INSN_COST);
11071 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11072
11073 ins_encode %{
11074 __ bicw(as_Register($dst$$reg),
11075 as_Register($src1$$reg),
11076 as_Register($src2$$reg),
11077 Assembler::LSL,
11078 $src3$$constant & 0x1f);
11079 %}
11080
11081 ins_pipe(ialu_reg_reg_shift);
11082 %}
11083
11084 // This pattern is automatically generated from aarch64_ad.m4
11085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11086 // val & (-1 ^ (val << shift)) ==> bic
11087 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11088 iRegL src1, iRegL src2,
11089 immI src3, immL_M1 src4) %{
11090 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11091 ins_cost(1.9 * INSN_COST);
11092 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11093
11094 ins_encode %{
11095 __ bic(as_Register($dst$$reg),
11096 as_Register($src1$$reg),
11097 as_Register($src2$$reg),
11098 Assembler::LSL,
11099 $src3$$constant & 0x3f);
11100 %}
11101
11102 ins_pipe(ialu_reg_reg_shift);
11103 %}
11104
11105 // This pattern is automatically generated from aarch64_ad.m4
11106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11107 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11108 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11109 iRegIorL2I src1, iRegIorL2I src2,
11110 immI src3, immI_M1 src4) %{
11111 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11112 ins_cost(1.9 * INSN_COST);
11113 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11114
11115 ins_encode %{
11116 __ eonw(as_Register($dst$$reg),
11117 as_Register($src1$$reg),
11118 as_Register($src2$$reg),
11119 Assembler::LSR,
11120 $src3$$constant & 0x1f);
11121 %}
11122
11123 ins_pipe(ialu_reg_reg_shift);
11124 %}
11125
11126 // This pattern is automatically generated from aarch64_ad.m4
11127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11128 // val ^ (-1 ^ (val >>> shift)) ==> eon
11129 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11130 iRegL src1, iRegL src2,
11131 immI src3, immL_M1 src4) %{
11132 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11133 ins_cost(1.9 * INSN_COST);
11134 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11135
11136 ins_encode %{
11137 __ eon(as_Register($dst$$reg),
11138 as_Register($src1$$reg),
11139 as_Register($src2$$reg),
11140 Assembler::LSR,
11141 $src3$$constant & 0x3f);
11142 %}
11143
11144 ins_pipe(ialu_reg_reg_shift);
11145 %}
11146
11147 // This pattern is automatically generated from aarch64_ad.m4
11148 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11149 // val ^ (-1 ^ (val >> shift)) ==> eonw
11150 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11151 iRegIorL2I src1, iRegIorL2I src2,
11152 immI src3, immI_M1 src4) %{
11153 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11154 ins_cost(1.9 * INSN_COST);
11155 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11156
11157 ins_encode %{
11158 __ eonw(as_Register($dst$$reg),
11159 as_Register($src1$$reg),
11160 as_Register($src2$$reg),
11161 Assembler::ASR,
11162 $src3$$constant & 0x1f);
11163 %}
11164
11165 ins_pipe(ialu_reg_reg_shift);
11166 %}
11167
11168 // This pattern is automatically generated from aarch64_ad.m4
11169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11170 // val ^ (-1 ^ (val >> shift)) ==> eon
11171 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11172 iRegL src1, iRegL src2,
11173 immI src3, immL_M1 src4) %{
11174 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11175 ins_cost(1.9 * INSN_COST);
11176 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11177
11178 ins_encode %{
11179 __ eon(as_Register($dst$$reg),
11180 as_Register($src1$$reg),
11181 as_Register($src2$$reg),
11182 Assembler::ASR,
11183 $src3$$constant & 0x3f);
11184 %}
11185
11186 ins_pipe(ialu_reg_reg_shift);
11187 %}
11188
11189 // This pattern is automatically generated from aarch64_ad.m4
11190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11191 // val ^ (-1 ^ (val ror shift)) ==> eonw
11192 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11193 iRegIorL2I src1, iRegIorL2I src2,
11194 immI src3, immI_M1 src4) %{
11195 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11196 ins_cost(1.9 * INSN_COST);
11197 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11198
11199 ins_encode %{
11200 __ eonw(as_Register($dst$$reg),
11201 as_Register($src1$$reg),
11202 as_Register($src2$$reg),
11203 Assembler::ROR,
11204 $src3$$constant & 0x1f);
11205 %}
11206
11207 ins_pipe(ialu_reg_reg_shift);
11208 %}
11209
11210 // This pattern is automatically generated from aarch64_ad.m4
11211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11212 // val ^ (-1 ^ (val ror shift)) ==> eon
11213 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11214 iRegL src1, iRegL src2,
11215 immI src3, immL_M1 src4) %{
11216 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11217 ins_cost(1.9 * INSN_COST);
11218 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11219
11220 ins_encode %{
11221 __ eon(as_Register($dst$$reg),
11222 as_Register($src1$$reg),
11223 as_Register($src2$$reg),
11224 Assembler::ROR,
11225 $src3$$constant & 0x3f);
11226 %}
11227
11228 ins_pipe(ialu_reg_reg_shift);
11229 %}
11230
11231 // This pattern is automatically generated from aarch64_ad.m4
11232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11233 // val ^ (-1 ^ (val << shift)) ==> eonw
11234 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11235 iRegIorL2I src1, iRegIorL2I src2,
11236 immI src3, immI_M1 src4) %{
11237 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11238 ins_cost(1.9 * INSN_COST);
11239 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11240
11241 ins_encode %{
11242 __ eonw(as_Register($dst$$reg),
11243 as_Register($src1$$reg),
11244 as_Register($src2$$reg),
11245 Assembler::LSL,
11246 $src3$$constant & 0x1f);
11247 %}
11248
11249 ins_pipe(ialu_reg_reg_shift);
11250 %}
11251
11252 // This pattern is automatically generated from aarch64_ad.m4
11253 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11254 // val ^ (-1 ^ (val << shift)) ==> eon
11255 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11256 iRegL src1, iRegL src2,
11257 immI src3, immL_M1 src4) %{
11258 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11259 ins_cost(1.9 * INSN_COST);
11260 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11261
11262 ins_encode %{
11263 __ eon(as_Register($dst$$reg),
11264 as_Register($src1$$reg),
11265 as_Register($src2$$reg),
11266 Assembler::LSL,
11267 $src3$$constant & 0x3f);
11268 %}
11269
11270 ins_pipe(ialu_reg_reg_shift);
11271 %}
11272
11273 // This pattern is automatically generated from aarch64_ad.m4
11274 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11275 // val | (-1 ^ (val >>> shift)) ==> ornw
11276 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11277 iRegIorL2I src1, iRegIorL2I src2,
11278 immI src3, immI_M1 src4) %{
11279 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11280 ins_cost(1.9 * INSN_COST);
11281 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11282
11283 ins_encode %{
11284 __ ornw(as_Register($dst$$reg),
11285 as_Register($src1$$reg),
11286 as_Register($src2$$reg),
11287 Assembler::LSR,
11288 $src3$$constant & 0x1f);
11289 %}
11290
11291 ins_pipe(ialu_reg_reg_shift);
11292 %}
11293
11294 // This pattern is automatically generated from aarch64_ad.m4
11295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11296 // val | (-1 ^ (val >>> shift)) ==> orn
11297 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11298 iRegL src1, iRegL src2,
11299 immI src3, immL_M1 src4) %{
11300 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11301 ins_cost(1.9 * INSN_COST);
11302 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11303
11304 ins_encode %{
11305 __ orn(as_Register($dst$$reg),
11306 as_Register($src1$$reg),
11307 as_Register($src2$$reg),
11308 Assembler::LSR,
11309 $src3$$constant & 0x3f);
11310 %}
11311
11312 ins_pipe(ialu_reg_reg_shift);
11313 %}
11314
11315 // This pattern is automatically generated from aarch64_ad.m4
11316 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11317 // val | (-1 ^ (val >> shift)) ==> ornw
11318 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11319 iRegIorL2I src1, iRegIorL2I src2,
11320 immI src3, immI_M1 src4) %{
11321 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11322 ins_cost(1.9 * INSN_COST);
11323 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11324
11325 ins_encode %{
11326 __ ornw(as_Register($dst$$reg),
11327 as_Register($src1$$reg),
11328 as_Register($src2$$reg),
11329 Assembler::ASR,
11330 $src3$$constant & 0x1f);
11331 %}
11332
11333 ins_pipe(ialu_reg_reg_shift);
11334 %}
11335
11336 // This pattern is automatically generated from aarch64_ad.m4
11337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11338 // val | (-1 ^ (val >> shift)) ==> orn
11339 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11340 iRegL src1, iRegL src2,
11341 immI src3, immL_M1 src4) %{
11342 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11343 ins_cost(1.9 * INSN_COST);
11344 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11345
11346 ins_encode %{
11347 __ orn(as_Register($dst$$reg),
11348 as_Register($src1$$reg),
11349 as_Register($src2$$reg),
11350 Assembler::ASR,
11351 $src3$$constant & 0x3f);
11352 %}
11353
11354 ins_pipe(ialu_reg_reg_shift);
11355 %}
11356
11357 // This pattern is automatically generated from aarch64_ad.m4
11358 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11359 // val | (-1 ^ (val ror shift)) ==> ornw
11360 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11361 iRegIorL2I src1, iRegIorL2I src2,
11362 immI src3, immI_M1 src4) %{
11363 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11364 ins_cost(1.9 * INSN_COST);
11365 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11366
11367 ins_encode %{
11368 __ ornw(as_Register($dst$$reg),
11369 as_Register($src1$$reg),
11370 as_Register($src2$$reg),
11371 Assembler::ROR,
11372 $src3$$constant & 0x1f);
11373 %}
11374
11375 ins_pipe(ialu_reg_reg_shift);
11376 %}
11377
11378 // This pattern is automatically generated from aarch64_ad.m4
11379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11380 // val | (-1 ^ (val ror shift)) ==> orn
11381 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11382 iRegL src1, iRegL src2,
11383 immI src3, immL_M1 src4) %{
11384 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11385 ins_cost(1.9 * INSN_COST);
11386 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11387
11388 ins_encode %{
11389 __ orn(as_Register($dst$$reg),
11390 as_Register($src1$$reg),
11391 as_Register($src2$$reg),
11392 Assembler::ROR,
11393 $src3$$constant & 0x3f);
11394 %}
11395
11396 ins_pipe(ialu_reg_reg_shift);
11397 %}
11398
11399 // This pattern is automatically generated from aarch64_ad.m4
11400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11401 // val | (-1 ^ (val << shift)) ==> ornw
11402 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11403 iRegIorL2I src1, iRegIorL2I src2,
11404 immI src3, immI_M1 src4) %{
11405 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11406 ins_cost(1.9 * INSN_COST);
11407 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11408
11409 ins_encode %{
11410 __ ornw(as_Register($dst$$reg),
11411 as_Register($src1$$reg),
11412 as_Register($src2$$reg),
11413 Assembler::LSL,
11414 $src3$$constant & 0x1f);
11415 %}
11416
11417 ins_pipe(ialu_reg_reg_shift);
11418 %}
11419
11420 // This pattern is automatically generated from aarch64_ad.m4
11421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11422 // val | (-1 ^ (val << shift)) ==> orn
11423 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11424 iRegL src1, iRegL src2,
11425 immI src3, immL_M1 src4) %{
11426 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11427 ins_cost(1.9 * INSN_COST);
11428 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11429
11430 ins_encode %{
11431 __ orn(as_Register($dst$$reg),
11432 as_Register($src1$$reg),
11433 as_Register($src2$$reg),
11434 Assembler::LSL,
11435 $src3$$constant & 0x3f);
11436 %}
11437
11438 ins_pipe(ialu_reg_reg_shift);
11439 %}
11440
11441 // This pattern is automatically generated from aarch64_ad.m4
11442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11443 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11444 iRegIorL2I src1, iRegIorL2I src2,
11445 immI src3) %{
11446 match(Set dst (AndI src1 (URShiftI src2 src3)));
11447
11448 ins_cost(1.9 * INSN_COST);
11449 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11450
11451 ins_encode %{
11452 __ andw(as_Register($dst$$reg),
11453 as_Register($src1$$reg),
11454 as_Register($src2$$reg),
11455 Assembler::LSR,
11456 $src3$$constant & 0x1f);
11457 %}
11458
11459 ins_pipe(ialu_reg_reg_shift);
11460 %}
11461
11462 // This pattern is automatically generated from aarch64_ad.m4
11463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11464 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11465 iRegL src1, iRegL src2,
11466 immI src3) %{
11467 match(Set dst (AndL src1 (URShiftL src2 src3)));
11468
11469 ins_cost(1.9 * INSN_COST);
11470 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11471
11472 ins_encode %{
11473 __ andr(as_Register($dst$$reg),
11474 as_Register($src1$$reg),
11475 as_Register($src2$$reg),
11476 Assembler::LSR,
11477 $src3$$constant & 0x3f);
11478 %}
11479
11480 ins_pipe(ialu_reg_reg_shift);
11481 %}
11482
11483 // This pattern is automatically generated from aarch64_ad.m4
11484 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11485 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11486 iRegIorL2I src1, iRegIorL2I src2,
11487 immI src3) %{
11488 match(Set dst (AndI src1 (RShiftI src2 src3)));
11489
11490 ins_cost(1.9 * INSN_COST);
11491 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11492
11493 ins_encode %{
11494 __ andw(as_Register($dst$$reg),
11495 as_Register($src1$$reg),
11496 as_Register($src2$$reg),
11497 Assembler::ASR,
11498 $src3$$constant & 0x1f);
11499 %}
11500
11501 ins_pipe(ialu_reg_reg_shift);
11502 %}
11503
11504 // This pattern is automatically generated from aarch64_ad.m4
11505 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11506 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11507 iRegL src1, iRegL src2,
11508 immI src3) %{
11509 match(Set dst (AndL src1 (RShiftL src2 src3)));
11510
11511 ins_cost(1.9 * INSN_COST);
11512 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11513
11514 ins_encode %{
11515 __ andr(as_Register($dst$$reg),
11516 as_Register($src1$$reg),
11517 as_Register($src2$$reg),
11518 Assembler::ASR,
11519 $src3$$constant & 0x3f);
11520 %}
11521
11522 ins_pipe(ialu_reg_reg_shift);
11523 %}
11524
11525 // This pattern is automatically generated from aarch64_ad.m4
11526 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11527 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11528 iRegIorL2I src1, iRegIorL2I src2,
11529 immI src3) %{
11530 match(Set dst (AndI src1 (LShiftI src2 src3)));
11531
11532 ins_cost(1.9 * INSN_COST);
11533 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11534
11535 ins_encode %{
11536 __ andw(as_Register($dst$$reg),
11537 as_Register($src1$$reg),
11538 as_Register($src2$$reg),
11539 Assembler::LSL,
11540 $src3$$constant & 0x1f);
11541 %}
11542
11543 ins_pipe(ialu_reg_reg_shift);
11544 %}
11545
11546 // This pattern is automatically generated from aarch64_ad.m4
11547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11548 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11549 iRegL src1, iRegL src2,
11550 immI src3) %{
11551 match(Set dst (AndL src1 (LShiftL src2 src3)));
11552
11553 ins_cost(1.9 * INSN_COST);
11554 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11555
11556 ins_encode %{
11557 __ andr(as_Register($dst$$reg),
11558 as_Register($src1$$reg),
11559 as_Register($src2$$reg),
11560 Assembler::LSL,
11561 $src3$$constant & 0x3f);
11562 %}
11563
11564 ins_pipe(ialu_reg_reg_shift);
11565 %}
11566
11567 // This pattern is automatically generated from aarch64_ad.m4
11568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11569 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11570 iRegIorL2I src1, iRegIorL2I src2,
11571 immI src3) %{
11572 match(Set dst (AndI src1 (RotateRight src2 src3)));
11573
11574 ins_cost(1.9 * INSN_COST);
11575 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11576
11577 ins_encode %{
11578 __ andw(as_Register($dst$$reg),
11579 as_Register($src1$$reg),
11580 as_Register($src2$$reg),
11581 Assembler::ROR,
11582 $src3$$constant & 0x1f);
11583 %}
11584
11585 ins_pipe(ialu_reg_reg_shift);
11586 %}
11587
11588 // This pattern is automatically generated from aarch64_ad.m4
11589 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11590 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11591 iRegL src1, iRegL src2,
11592 immI src3) %{
11593 match(Set dst (AndL src1 (RotateRight src2 src3)));
11594
11595 ins_cost(1.9 * INSN_COST);
11596 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11597
11598 ins_encode %{
11599 __ andr(as_Register($dst$$reg),
11600 as_Register($src1$$reg),
11601 as_Register($src2$$reg),
11602 Assembler::ROR,
11603 $src3$$constant & 0x3f);
11604 %}
11605
11606 ins_pipe(ialu_reg_reg_shift);
11607 %}
11608
11609 // This pattern is automatically generated from aarch64_ad.m4
11610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11611 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11612 iRegIorL2I src1, iRegIorL2I src2,
11613 immI src3) %{
11614 match(Set dst (XorI src1 (URShiftI src2 src3)));
11615
11616 ins_cost(1.9 * INSN_COST);
11617 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11618
11619 ins_encode %{
11620 __ eorw(as_Register($dst$$reg),
11621 as_Register($src1$$reg),
11622 as_Register($src2$$reg),
11623 Assembler::LSR,
11624 $src3$$constant & 0x1f);
11625 %}
11626
11627 ins_pipe(ialu_reg_reg_shift);
11628 %}
11629
11630 // This pattern is automatically generated from aarch64_ad.m4
11631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11632 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11633 iRegL src1, iRegL src2,
11634 immI src3) %{
11635 match(Set dst (XorL src1 (URShiftL src2 src3)));
11636
11637 ins_cost(1.9 * INSN_COST);
11638 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11639
11640 ins_encode %{
11641 __ eor(as_Register($dst$$reg),
11642 as_Register($src1$$reg),
11643 as_Register($src2$$reg),
11644 Assembler::LSR,
11645 $src3$$constant & 0x3f);
11646 %}
11647
11648 ins_pipe(ialu_reg_reg_shift);
11649 %}
11650
11651 // This pattern is automatically generated from aarch64_ad.m4
11652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11653 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11654 iRegIorL2I src1, iRegIorL2I src2,
11655 immI src3) %{
11656 match(Set dst (XorI src1 (RShiftI src2 src3)));
11657
11658 ins_cost(1.9 * INSN_COST);
11659 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11660
11661 ins_encode %{
11662 __ eorw(as_Register($dst$$reg),
11663 as_Register($src1$$reg),
11664 as_Register($src2$$reg),
11665 Assembler::ASR,
11666 $src3$$constant & 0x1f);
11667 %}
11668
11669 ins_pipe(ialu_reg_reg_shift);
11670 %}
11671
11672 // This pattern is automatically generated from aarch64_ad.m4
11673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11674 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11675 iRegL src1, iRegL src2,
11676 immI src3) %{
11677 match(Set dst (XorL src1 (RShiftL src2 src3)));
11678
11679 ins_cost(1.9 * INSN_COST);
11680 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11681
11682 ins_encode %{
11683 __ eor(as_Register($dst$$reg),
11684 as_Register($src1$$reg),
11685 as_Register($src2$$reg),
11686 Assembler::ASR,
11687 $src3$$constant & 0x3f);
11688 %}
11689
11690 ins_pipe(ialu_reg_reg_shift);
11691 %}
11692
11693 // This pattern is automatically generated from aarch64_ad.m4
11694 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11695 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11696 iRegIorL2I src1, iRegIorL2I src2,
11697 immI src3) %{
11698 match(Set dst (XorI src1 (LShiftI src2 src3)));
11699
11700 ins_cost(1.9 * INSN_COST);
11701 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11702
11703 ins_encode %{
11704 __ eorw(as_Register($dst$$reg),
11705 as_Register($src1$$reg),
11706 as_Register($src2$$reg),
11707 Assembler::LSL,
11708 $src3$$constant & 0x1f);
11709 %}
11710
11711 ins_pipe(ialu_reg_reg_shift);
11712 %}
11713
11714 // This pattern is automatically generated from aarch64_ad.m4
11715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11716 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11717 iRegL src1, iRegL src2,
11718 immI src3) %{
11719 match(Set dst (XorL src1 (LShiftL src2 src3)));
11720
11721 ins_cost(1.9 * INSN_COST);
11722 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11723
11724 ins_encode %{
11725 __ eor(as_Register($dst$$reg),
11726 as_Register($src1$$reg),
11727 as_Register($src2$$reg),
11728 Assembler::LSL,
11729 $src3$$constant & 0x3f);
11730 %}
11731
11732 ins_pipe(ialu_reg_reg_shift);
11733 %}
11734
11735 // This pattern is automatically generated from aarch64_ad.m4
11736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11737 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11738 iRegIorL2I src1, iRegIorL2I src2,
11739 immI src3) %{
11740 match(Set dst (XorI src1 (RotateRight src2 src3)));
11741
11742 ins_cost(1.9 * INSN_COST);
11743 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11744
11745 ins_encode %{
11746 __ eorw(as_Register($dst$$reg),
11747 as_Register($src1$$reg),
11748 as_Register($src2$$reg),
11749 Assembler::ROR,
11750 $src3$$constant & 0x1f);
11751 %}
11752
11753 ins_pipe(ialu_reg_reg_shift);
11754 %}
11755
11756 // This pattern is automatically generated from aarch64_ad.m4
11757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11758 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11759 iRegL src1, iRegL src2,
11760 immI src3) %{
11761 match(Set dst (XorL src1 (RotateRight src2 src3)));
11762
11763 ins_cost(1.9 * INSN_COST);
11764 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11765
11766 ins_encode %{
11767 __ eor(as_Register($dst$$reg),
11768 as_Register($src1$$reg),
11769 as_Register($src2$$reg),
11770 Assembler::ROR,
11771 $src3$$constant & 0x3f);
11772 %}
11773
11774 ins_pipe(ialu_reg_reg_shift);
11775 %}
11776
11777 // This pattern is automatically generated from aarch64_ad.m4
11778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11779 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11780 iRegIorL2I src1, iRegIorL2I src2,
11781 immI src3) %{
11782 match(Set dst (OrI src1 (URShiftI src2 src3)));
11783
11784 ins_cost(1.9 * INSN_COST);
11785 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11786
11787 ins_encode %{
11788 __ orrw(as_Register($dst$$reg),
11789 as_Register($src1$$reg),
11790 as_Register($src2$$reg),
11791 Assembler::LSR,
11792 $src3$$constant & 0x1f);
11793 %}
11794
11795 ins_pipe(ialu_reg_reg_shift);
11796 %}
11797
11798 // This pattern is automatically generated from aarch64_ad.m4
11799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11800 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11801 iRegL src1, iRegL src2,
11802 immI src3) %{
11803 match(Set dst (OrL src1 (URShiftL src2 src3)));
11804
11805 ins_cost(1.9 * INSN_COST);
11806 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11807
11808 ins_encode %{
11809 __ orr(as_Register($dst$$reg),
11810 as_Register($src1$$reg),
11811 as_Register($src2$$reg),
11812 Assembler::LSR,
11813 $src3$$constant & 0x3f);
11814 %}
11815
11816 ins_pipe(ialu_reg_reg_shift);
11817 %}
11818
11819 // This pattern is automatically generated from aarch64_ad.m4
11820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11821 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11822 iRegIorL2I src1, iRegIorL2I src2,
11823 immI src3) %{
11824 match(Set dst (OrI src1 (RShiftI src2 src3)));
11825
11826 ins_cost(1.9 * INSN_COST);
11827 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11828
11829 ins_encode %{
11830 __ orrw(as_Register($dst$$reg),
11831 as_Register($src1$$reg),
11832 as_Register($src2$$reg),
11833 Assembler::ASR,
11834 $src3$$constant & 0x1f);
11835 %}
11836
11837 ins_pipe(ialu_reg_reg_shift);
11838 %}
11839
11840 // This pattern is automatically generated from aarch64_ad.m4
11841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11842 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11843 iRegL src1, iRegL src2,
11844 immI src3) %{
11845 match(Set dst (OrL src1 (RShiftL src2 src3)));
11846
11847 ins_cost(1.9 * INSN_COST);
11848 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11849
11850 ins_encode %{
11851 __ orr(as_Register($dst$$reg),
11852 as_Register($src1$$reg),
11853 as_Register($src2$$reg),
11854 Assembler::ASR,
11855 $src3$$constant & 0x3f);
11856 %}
11857
11858 ins_pipe(ialu_reg_reg_shift);
11859 %}
11860
11861 // This pattern is automatically generated from aarch64_ad.m4
11862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11863 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11864 iRegIorL2I src1, iRegIorL2I src2,
11865 immI src3) %{
11866 match(Set dst (OrI src1 (LShiftI src2 src3)));
11867
11868 ins_cost(1.9 * INSN_COST);
11869 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11870
11871 ins_encode %{
11872 __ orrw(as_Register($dst$$reg),
11873 as_Register($src1$$reg),
11874 as_Register($src2$$reg),
11875 Assembler::LSL,
11876 $src3$$constant & 0x1f);
11877 %}
11878
11879 ins_pipe(ialu_reg_reg_shift);
11880 %}
11881
11882 // This pattern is automatically generated from aarch64_ad.m4
11883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11884 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11885 iRegL src1, iRegL src2,
11886 immI src3) %{
11887 match(Set dst (OrL src1 (LShiftL src2 src3)));
11888
11889 ins_cost(1.9 * INSN_COST);
11890 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11891
11892 ins_encode %{
11893 __ orr(as_Register($dst$$reg),
11894 as_Register($src1$$reg),
11895 as_Register($src2$$reg),
11896 Assembler::LSL,
11897 $src3$$constant & 0x3f);
11898 %}
11899
11900 ins_pipe(ialu_reg_reg_shift);
11901 %}
11902
11903 // This pattern is automatically generated from aarch64_ad.m4
11904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11905 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11906 iRegIorL2I src1, iRegIorL2I src2,
11907 immI src3) %{
11908 match(Set dst (OrI src1 (RotateRight src2 src3)));
11909
11910 ins_cost(1.9 * INSN_COST);
11911 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11912
11913 ins_encode %{
11914 __ orrw(as_Register($dst$$reg),
11915 as_Register($src1$$reg),
11916 as_Register($src2$$reg),
11917 Assembler::ROR,
11918 $src3$$constant & 0x1f);
11919 %}
11920
11921 ins_pipe(ialu_reg_reg_shift);
11922 %}
11923
11924 // This pattern is automatically generated from aarch64_ad.m4
11925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11926 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11927 iRegL src1, iRegL src2,
11928 immI src3) %{
11929 match(Set dst (OrL src1 (RotateRight src2 src3)));
11930
11931 ins_cost(1.9 * INSN_COST);
11932 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11933
11934 ins_encode %{
11935 __ orr(as_Register($dst$$reg),
11936 as_Register($src1$$reg),
11937 as_Register($src2$$reg),
11938 Assembler::ROR,
11939 $src3$$constant & 0x3f);
11940 %}
11941
11942 ins_pipe(ialu_reg_reg_shift);
11943 %}
11944
11945 // This pattern is automatically generated from aarch64_ad.m4
11946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11947 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11948 iRegIorL2I src1, iRegIorL2I src2,
11949 immI src3) %{
11950 match(Set dst (AddI src1 (URShiftI src2 src3)));
11951
11952 ins_cost(1.9 * INSN_COST);
11953 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11954
11955 ins_encode %{
11956 __ addw(as_Register($dst$$reg),
11957 as_Register($src1$$reg),
11958 as_Register($src2$$reg),
11959 Assembler::LSR,
11960 $src3$$constant & 0x1f);
11961 %}
11962
11963 ins_pipe(ialu_reg_reg_shift);
11964 %}
11965
11966 // This pattern is automatically generated from aarch64_ad.m4
11967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11968 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11969 iRegL src1, iRegL src2,
11970 immI src3) %{
11971 match(Set dst (AddL src1 (URShiftL src2 src3)));
11972
11973 ins_cost(1.9 * INSN_COST);
11974 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11975
11976 ins_encode %{
11977 __ add(as_Register($dst$$reg),
11978 as_Register($src1$$reg),
11979 as_Register($src2$$reg),
11980 Assembler::LSR,
11981 $src3$$constant & 0x3f);
11982 %}
11983
11984 ins_pipe(ialu_reg_reg_shift);
11985 %}
11986
11987 // This pattern is automatically generated from aarch64_ad.m4
11988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11989 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11990 iRegIorL2I src1, iRegIorL2I src2,
11991 immI src3) %{
11992 match(Set dst (AddI src1 (RShiftI src2 src3)));
11993
11994 ins_cost(1.9 * INSN_COST);
11995 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11996
11997 ins_encode %{
11998 __ addw(as_Register($dst$$reg),
11999 as_Register($src1$$reg),
12000 as_Register($src2$$reg),
12001 Assembler::ASR,
12002 $src3$$constant & 0x1f);
12003 %}
12004
12005 ins_pipe(ialu_reg_reg_shift);
12006 %}
12007
12008 // This pattern is automatically generated from aarch64_ad.m4
12009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12010 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12011 iRegL src1, iRegL src2,
12012 immI src3) %{
12013 match(Set dst (AddL src1 (RShiftL src2 src3)));
12014
12015 ins_cost(1.9 * INSN_COST);
12016 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12017
12018 ins_encode %{
12019 __ add(as_Register($dst$$reg),
12020 as_Register($src1$$reg),
12021 as_Register($src2$$reg),
12022 Assembler::ASR,
12023 $src3$$constant & 0x3f);
12024 %}
12025
12026 ins_pipe(ialu_reg_reg_shift);
12027 %}
12028
12029 // This pattern is automatically generated from aarch64_ad.m4
12030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12031 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12032 iRegIorL2I src1, iRegIorL2I src2,
12033 immI src3) %{
12034 match(Set dst (AddI src1 (LShiftI src2 src3)));
12035
12036 ins_cost(1.9 * INSN_COST);
12037 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12038
12039 ins_encode %{
12040 __ addw(as_Register($dst$$reg),
12041 as_Register($src1$$reg),
12042 as_Register($src2$$reg),
12043 Assembler::LSL,
12044 $src3$$constant & 0x1f);
12045 %}
12046
12047 ins_pipe(ialu_reg_reg_shift);
12048 %}
12049
12050 // This pattern is automatically generated from aarch64_ad.m4
12051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12052 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12053 iRegL src1, iRegL src2,
12054 immI src3) %{
12055 match(Set dst (AddL src1 (LShiftL src2 src3)));
12056
12057 ins_cost(1.9 * INSN_COST);
12058 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12059
12060 ins_encode %{
12061 __ add(as_Register($dst$$reg),
12062 as_Register($src1$$reg),
12063 as_Register($src2$$reg),
12064 Assembler::LSL,
12065 $src3$$constant & 0x3f);
12066 %}
12067
12068 ins_pipe(ialu_reg_reg_shift);
12069 %}
12070
12071 // This pattern is automatically generated from aarch64_ad.m4
12072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12073 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12074 iRegIorL2I src1, iRegIorL2I src2,
12075 immI src3) %{
12076 match(Set dst (SubI src1 (URShiftI src2 src3)));
12077
12078 ins_cost(1.9 * INSN_COST);
12079 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12080
12081 ins_encode %{
12082 __ subw(as_Register($dst$$reg),
12083 as_Register($src1$$reg),
12084 as_Register($src2$$reg),
12085 Assembler::LSR,
12086 $src3$$constant & 0x1f);
12087 %}
12088
12089 ins_pipe(ialu_reg_reg_shift);
12090 %}
12091
12092 // This pattern is automatically generated from aarch64_ad.m4
12093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12094 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12095 iRegL src1, iRegL src2,
12096 immI src3) %{
12097 match(Set dst (SubL src1 (URShiftL src2 src3)));
12098
12099 ins_cost(1.9 * INSN_COST);
12100 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12101
12102 ins_encode %{
12103 __ sub(as_Register($dst$$reg),
12104 as_Register($src1$$reg),
12105 as_Register($src2$$reg),
12106 Assembler::LSR,
12107 $src3$$constant & 0x3f);
12108 %}
12109
12110 ins_pipe(ialu_reg_reg_shift);
12111 %}
12112
12113 // This pattern is automatically generated from aarch64_ad.m4
12114 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12115 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12116 iRegIorL2I src1, iRegIorL2I src2,
12117 immI src3) %{
12118 match(Set dst (SubI src1 (RShiftI src2 src3)));
12119
12120 ins_cost(1.9 * INSN_COST);
12121 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12122
12123 ins_encode %{
12124 __ subw(as_Register($dst$$reg),
12125 as_Register($src1$$reg),
12126 as_Register($src2$$reg),
12127 Assembler::ASR,
12128 $src3$$constant & 0x1f);
12129 %}
12130
12131 ins_pipe(ialu_reg_reg_shift);
12132 %}
12133
12134 // This pattern is automatically generated from aarch64_ad.m4
12135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12136 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12137 iRegL src1, iRegL src2,
12138 immI src3) %{
12139 match(Set dst (SubL src1 (RShiftL src2 src3)));
12140
12141 ins_cost(1.9 * INSN_COST);
12142 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12143
12144 ins_encode %{
12145 __ sub(as_Register($dst$$reg),
12146 as_Register($src1$$reg),
12147 as_Register($src2$$reg),
12148 Assembler::ASR,
12149 $src3$$constant & 0x3f);
12150 %}
12151
12152 ins_pipe(ialu_reg_reg_shift);
12153 %}
12154
12155 // This pattern is automatically generated from aarch64_ad.m4
12156 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12157 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12158 iRegIorL2I src1, iRegIorL2I src2,
12159 immI src3) %{
12160 match(Set dst (SubI src1 (LShiftI src2 src3)));
12161
12162 ins_cost(1.9 * INSN_COST);
12163 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12164
12165 ins_encode %{
12166 __ subw(as_Register($dst$$reg),
12167 as_Register($src1$$reg),
12168 as_Register($src2$$reg),
12169 Assembler::LSL,
12170 $src3$$constant & 0x1f);
12171 %}
12172
12173 ins_pipe(ialu_reg_reg_shift);
12174 %}
12175
12176 // This pattern is automatically generated from aarch64_ad.m4
12177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12178 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12179 iRegL src1, iRegL src2,
12180 immI src3) %{
12181 match(Set dst (SubL src1 (LShiftL src2 src3)));
12182
12183 ins_cost(1.9 * INSN_COST);
12184 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12185
12186 ins_encode %{
12187 __ sub(as_Register($dst$$reg),
12188 as_Register($src1$$reg),
12189 as_Register($src2$$reg),
12190 Assembler::LSL,
12191 $src3$$constant & 0x3f);
12192 %}
12193
12194 ins_pipe(ialu_reg_reg_shift);
12195 %}
12196
12197 // This pattern is automatically generated from aarch64_ad.m4
12198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12199
12200 // Shift Left followed by Shift Right.
12201 // This idiom is used by the compiler for the i2b bytecode etc.
12202 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12203 %{
12204 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12205 ins_cost(INSN_COST * 2);
12206 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12207 ins_encode %{
12208 int lshift = $lshift_count$$constant & 63;
12209 int rshift = $rshift_count$$constant & 63;
12210 int s = 63 - lshift;
12211 int r = (rshift - lshift) & 63;
12212 __ sbfm(as_Register($dst$$reg),
12213 as_Register($src$$reg),
12214 r, s);
12215 %}
12216
12217 ins_pipe(ialu_reg_shift);
12218 %}
12219
12220 // This pattern is automatically generated from aarch64_ad.m4
12221 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12222
12223 // Shift Left followed by Shift Right.
12224 // This idiom is used by the compiler for the i2b bytecode etc.
12225 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12226 %{
12227 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12228 ins_cost(INSN_COST * 2);
12229 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12230 ins_encode %{
12231 int lshift = $lshift_count$$constant & 31;
12232 int rshift = $rshift_count$$constant & 31;
12233 int s = 31 - lshift;
12234 int r = (rshift - lshift) & 31;
12235 __ sbfmw(as_Register($dst$$reg),
12236 as_Register($src$$reg),
12237 r, s);
12238 %}
12239
12240 ins_pipe(ialu_reg_shift);
12241 %}
12242
12243 // This pattern is automatically generated from aarch64_ad.m4
12244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12245
12246 // Shift Left followed by Shift Right.
12247 // This idiom is used by the compiler for the i2b bytecode etc.
12248 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12249 %{
12250 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12251 ins_cost(INSN_COST * 2);
12252 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12253 ins_encode %{
12254 int lshift = $lshift_count$$constant & 63;
12255 int rshift = $rshift_count$$constant & 63;
12256 int s = 63 - lshift;
12257 int r = (rshift - lshift) & 63;
12258 __ ubfm(as_Register($dst$$reg),
12259 as_Register($src$$reg),
12260 r, s);
12261 %}
12262
12263 ins_pipe(ialu_reg_shift);
12264 %}
12265
12266 // This pattern is automatically generated from aarch64_ad.m4
12267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12268
12269 // Shift Left followed by Shift Right.
12270 // This idiom is used by the compiler for the i2b bytecode etc.
12271 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12272 %{
12273 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12274 ins_cost(INSN_COST * 2);
12275 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12276 ins_encode %{
12277 int lshift = $lshift_count$$constant & 31;
12278 int rshift = $rshift_count$$constant & 31;
12279 int s = 31 - lshift;
12280 int r = (rshift - lshift) & 31;
12281 __ ubfmw(as_Register($dst$$reg),
12282 as_Register($src$$reg),
12283 r, s);
12284 %}
12285
12286 ins_pipe(ialu_reg_shift);
12287 %}
12288
12289 // Bitfield extract with shift & mask
12290
12291 // This pattern is automatically generated from aarch64_ad.m4
12292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12293 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12294 %{
12295 match(Set dst (AndI (URShiftI src rshift) mask));
12296 // Make sure we are not going to exceed what ubfxw can do.
12297 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12298
12299 ins_cost(INSN_COST);
12300 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12301 ins_encode %{
12302 int rshift = $rshift$$constant & 31;
12303 intptr_t mask = $mask$$constant;
12304 int width = exact_log2(mask+1);
12305 __ ubfxw(as_Register($dst$$reg),
12306 as_Register($src$$reg), rshift, width);
12307 %}
12308 ins_pipe(ialu_reg_shift);
12309 %}
12310
12311 // This pattern is automatically generated from aarch64_ad.m4
12312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12313 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12314 %{
12315 match(Set dst (AndL (URShiftL src rshift) mask));
12316 // Make sure we are not going to exceed what ubfx can do.
12317 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12318
12319 ins_cost(INSN_COST);
12320 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12321 ins_encode %{
12322 int rshift = $rshift$$constant & 63;
12323 intptr_t mask = $mask$$constant;
12324 int width = exact_log2_long(mask+1);
12325 __ ubfx(as_Register($dst$$reg),
12326 as_Register($src$$reg), rshift, width);
12327 %}
12328 ins_pipe(ialu_reg_shift);
12329 %}
12330
12331
12332 // This pattern is automatically generated from aarch64_ad.m4
12333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12334
12335 // We can use ubfx when extending an And with a mask when we know mask
12336 // is positive. We know that because immI_bitmask guarantees it.
12337 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12338 %{
12339 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12340 // Make sure we are not going to exceed what ubfxw can do.
12341 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12342
12343 ins_cost(INSN_COST * 2);
12344 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12345 ins_encode %{
12346 int rshift = $rshift$$constant & 31;
12347 intptr_t mask = $mask$$constant;
12348 int width = exact_log2(mask+1);
12349 __ ubfx(as_Register($dst$$reg),
12350 as_Register($src$$reg), rshift, width);
12351 %}
12352 ins_pipe(ialu_reg_shift);
12353 %}
12354
12355
12356 // This pattern is automatically generated from aarch64_ad.m4
12357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12358
12359 // We can use ubfiz when masking by a positive number and then left shifting the result.
12360 // We know that the mask is positive because immI_bitmask guarantees it.
12361 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12362 %{
12363 match(Set dst (LShiftI (AndI src mask) lshift));
12364 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12365
12366 ins_cost(INSN_COST);
12367 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12368 ins_encode %{
12369 int lshift = $lshift$$constant & 31;
12370 intptr_t mask = $mask$$constant;
12371 int width = exact_log2(mask+1);
12372 __ ubfizw(as_Register($dst$$reg),
12373 as_Register($src$$reg), lshift, width);
12374 %}
12375 ins_pipe(ialu_reg_shift);
12376 %}
12377
12378 // This pattern is automatically generated from aarch64_ad.m4
12379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12380
12381 // We can use ubfiz when masking by a positive number and then left shifting the result.
12382 // We know that the mask is positive because immL_bitmask guarantees it.
12383 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12384 %{
12385 match(Set dst (LShiftL (AndL src mask) lshift));
12386 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12387
12388 ins_cost(INSN_COST);
12389 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12390 ins_encode %{
12391 int lshift = $lshift$$constant & 63;
12392 intptr_t mask = $mask$$constant;
12393 int width = exact_log2_long(mask+1);
12394 __ ubfiz(as_Register($dst$$reg),
12395 as_Register($src$$reg), lshift, width);
12396 %}
12397 ins_pipe(ialu_reg_shift);
12398 %}
12399
12400 // This pattern is automatically generated from aarch64_ad.m4
12401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12402
12403 // We can use ubfiz when masking by a positive number and then left shifting the result.
12404 // We know that the mask is positive because immI_bitmask guarantees it.
12405 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12406 %{
12407 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12408 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12409
12410 ins_cost(INSN_COST);
12411 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12412 ins_encode %{
12413 int lshift = $lshift$$constant & 31;
12414 intptr_t mask = $mask$$constant;
12415 int width = exact_log2(mask+1);
12416 __ ubfizw(as_Register($dst$$reg),
12417 as_Register($src$$reg), lshift, width);
12418 %}
12419 ins_pipe(ialu_reg_shift);
12420 %}
12421
12422 // This pattern is automatically generated from aarch64_ad.m4
12423 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12424
12425 // We can use ubfiz when masking by a positive number and then left shifting the result.
12426 // We know that the mask is positive because immL_bitmask guarantees it.
12427 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12428 %{
12429 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12430 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12431
12432 ins_cost(INSN_COST);
12433 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12434 ins_encode %{
12435 int lshift = $lshift$$constant & 63;
12436 intptr_t mask = $mask$$constant;
12437 int width = exact_log2_long(mask+1);
12438 __ ubfiz(as_Register($dst$$reg),
12439 as_Register($src$$reg), lshift, width);
12440 %}
12441 ins_pipe(ialu_reg_shift);
12442 %}
12443
12444
12445 // This pattern is automatically generated from aarch64_ad.m4
12446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12447
12448 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12449 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12450 %{
12451 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12452 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12453
12454 ins_cost(INSN_COST);
12455 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12456 ins_encode %{
12457 int lshift = $lshift$$constant & 63;
12458 intptr_t mask = $mask$$constant;
12459 int width = exact_log2(mask+1);
12460 __ ubfiz(as_Register($dst$$reg),
12461 as_Register($src$$reg), lshift, width);
12462 %}
12463 ins_pipe(ialu_reg_shift);
12464 %}
12465
12466 // This pattern is automatically generated from aarch64_ad.m4
12467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12468
12469 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12470 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12471 %{
12472 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12473 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12474
12475 ins_cost(INSN_COST);
12476 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12477 ins_encode %{
12478 int lshift = $lshift$$constant & 31;
12479 intptr_t mask = $mask$$constant;
12480 int width = exact_log2(mask+1);
12481 __ ubfiz(as_Register($dst$$reg),
12482 as_Register($src$$reg), lshift, width);
12483 %}
12484 ins_pipe(ialu_reg_shift);
12485 %}
12486
12487 // This pattern is automatically generated from aarch64_ad.m4
12488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12489
12490 // Can skip int2long conversions after AND with small bitmask
12491 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12492 %{
12493 match(Set dst (ConvI2L (AndI src msk)));
12494 ins_cost(INSN_COST);
12495 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12496 ins_encode %{
12497 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12498 %}
12499 ins_pipe(ialu_reg_shift);
12500 %}
12501
12502
12503 // Rotations
12504
12505 // This pattern is automatically generated from aarch64_ad.m4
12506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12507 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12508 %{
12509 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12510 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12511
12512 ins_cost(INSN_COST);
12513 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12514
12515 ins_encode %{
12516 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12517 $rshift$$constant & 63);
12518 %}
12519 ins_pipe(ialu_reg_reg_extr);
12520 %}
12521
12522
12523 // This pattern is automatically generated from aarch64_ad.m4
12524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12525 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12526 %{
12527 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12528 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12529
12530 ins_cost(INSN_COST);
12531 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12532
12533 ins_encode %{
12534 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12535 $rshift$$constant & 31);
12536 %}
12537 ins_pipe(ialu_reg_reg_extr);
12538 %}
12539
12540
12541 // This pattern is automatically generated from aarch64_ad.m4
12542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12543 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12544 %{
12545 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12546 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12547
12548 ins_cost(INSN_COST);
12549 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12550
12551 ins_encode %{
12552 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12553 $rshift$$constant & 63);
12554 %}
12555 ins_pipe(ialu_reg_reg_extr);
12556 %}
12557
12558
12559 // This pattern is automatically generated from aarch64_ad.m4
12560 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12561 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12562 %{
12563 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12564 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12565
12566 ins_cost(INSN_COST);
12567 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12568
12569 ins_encode %{
12570 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12571 $rshift$$constant & 31);
12572 %}
12573 ins_pipe(ialu_reg_reg_extr);
12574 %}
12575
12576 // This pattern is automatically generated from aarch64_ad.m4
12577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12578 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12579 %{
12580 match(Set dst (RotateRight src shift));
12581
12582 ins_cost(INSN_COST);
12583 format %{ "ror $dst, $src, $shift" %}
12584
12585 ins_encode %{
12586 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12587 $shift$$constant & 0x1f);
12588 %}
12589 ins_pipe(ialu_reg_reg_vshift);
12590 %}
12591
12592 // This pattern is automatically generated from aarch64_ad.m4
12593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12594 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12595 %{
12596 match(Set dst (RotateRight src shift));
12597
12598 ins_cost(INSN_COST);
12599 format %{ "ror $dst, $src, $shift" %}
12600
12601 ins_encode %{
12602 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12603 $shift$$constant & 0x3f);
12604 %}
12605 ins_pipe(ialu_reg_reg_vshift);
12606 %}
12607
12608 // This pattern is automatically generated from aarch64_ad.m4
12609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12610 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12611 %{
12612 match(Set dst (RotateRight src shift));
12613
12614 ins_cost(INSN_COST);
12615 format %{ "ror $dst, $src, $shift" %}
12616
12617 ins_encode %{
12618 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12619 %}
12620 ins_pipe(ialu_reg_reg_vshift);
12621 %}
12622
12623 // This pattern is automatically generated from aarch64_ad.m4
12624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12625 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12626 %{
12627 match(Set dst (RotateRight src shift));
12628
12629 ins_cost(INSN_COST);
12630 format %{ "ror $dst, $src, $shift" %}
12631
12632 ins_encode %{
12633 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12634 %}
12635 ins_pipe(ialu_reg_reg_vshift);
12636 %}
12637
12638 // This pattern is automatically generated from aarch64_ad.m4
12639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12640 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12641 %{
12642 match(Set dst (RotateLeft src shift));
12643
12644 ins_cost(INSN_COST);
12645 format %{ "rol $dst, $src, $shift" %}
12646
12647 ins_encode %{
12648 __ subw(rscratch1, zr, as_Register($shift$$reg));
12649 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12650 %}
12651 ins_pipe(ialu_reg_reg_vshift);
12652 %}
12653
12654 // This pattern is automatically generated from aarch64_ad.m4
12655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12656 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12657 %{
12658 match(Set dst (RotateLeft src shift));
12659
12660 ins_cost(INSN_COST);
12661 format %{ "rol $dst, $src, $shift" %}
12662
12663 ins_encode %{
12664 __ subw(rscratch1, zr, as_Register($shift$$reg));
12665 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12666 %}
12667 ins_pipe(ialu_reg_reg_vshift);
12668 %}
12669
12670
12671 // Add/subtract (extended)
12672
12673 // This pattern is automatically generated from aarch64_ad.m4
12674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12675 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12676 %{
12677 match(Set dst (AddL src1 (ConvI2L src2)));
12678 ins_cost(INSN_COST);
12679 format %{ "add $dst, $src1, $src2, sxtw" %}
12680
12681 ins_encode %{
12682 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12683 as_Register($src2$$reg), ext::sxtw);
12684 %}
12685 ins_pipe(ialu_reg_reg);
12686 %}
12687
12688 // This pattern is automatically generated from aarch64_ad.m4
12689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12690 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12691 %{
12692 match(Set dst (SubL src1 (ConvI2L src2)));
12693 ins_cost(INSN_COST);
12694 format %{ "sub $dst, $src1, $src2, sxtw" %}
12695
12696 ins_encode %{
12697 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12698 as_Register($src2$$reg), ext::sxtw);
12699 %}
12700 ins_pipe(ialu_reg_reg);
12701 %}
12702
12703 // This pattern is automatically generated from aarch64_ad.m4
12704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12705 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12706 %{
12707 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12708 ins_cost(INSN_COST);
12709 format %{ "add $dst, $src1, $src2, sxth" %}
12710
12711 ins_encode %{
12712 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12713 as_Register($src2$$reg), ext::sxth);
12714 %}
12715 ins_pipe(ialu_reg_reg);
12716 %}
12717
12718 // This pattern is automatically generated from aarch64_ad.m4
12719 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12720 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12721 %{
12722 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12723 ins_cost(INSN_COST);
12724 format %{ "add $dst, $src1, $src2, sxtb" %}
12725
12726 ins_encode %{
12727 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12728 as_Register($src2$$reg), ext::sxtb);
12729 %}
12730 ins_pipe(ialu_reg_reg);
12731 %}
12732
12733 // This pattern is automatically generated from aarch64_ad.m4
12734 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12735 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12736 %{
12737 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12738 ins_cost(INSN_COST);
12739 format %{ "add $dst, $src1, $src2, uxtb" %}
12740
12741 ins_encode %{
12742 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12743 as_Register($src2$$reg), ext::uxtb);
12744 %}
12745 ins_pipe(ialu_reg_reg);
12746 %}
12747
12748 // This pattern is automatically generated from aarch64_ad.m4
12749 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12750 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12751 %{
12752 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12753 ins_cost(INSN_COST);
12754 format %{ "add $dst, $src1, $src2, sxth" %}
12755
12756 ins_encode %{
12757 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12758 as_Register($src2$$reg), ext::sxth);
12759 %}
12760 ins_pipe(ialu_reg_reg);
12761 %}
12762
12763 // This pattern is automatically generated from aarch64_ad.m4
12764 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12765 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12766 %{
12767 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12768 ins_cost(INSN_COST);
12769 format %{ "add $dst, $src1, $src2, sxtw" %}
12770
12771 ins_encode %{
12772 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12773 as_Register($src2$$reg), ext::sxtw);
12774 %}
12775 ins_pipe(ialu_reg_reg);
12776 %}
12777
12778 // This pattern is automatically generated from aarch64_ad.m4
12779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12780 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12781 %{
12782 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12783 ins_cost(INSN_COST);
12784 format %{ "add $dst, $src1, $src2, sxtb" %}
12785
12786 ins_encode %{
12787 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12788 as_Register($src2$$reg), ext::sxtb);
12789 %}
12790 ins_pipe(ialu_reg_reg);
12791 %}
12792
12793 // This pattern is automatically generated from aarch64_ad.m4
12794 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12795 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12796 %{
12797 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12798 ins_cost(INSN_COST);
12799 format %{ "add $dst, $src1, $src2, uxtb" %}
12800
12801 ins_encode %{
12802 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12803 as_Register($src2$$reg), ext::uxtb);
12804 %}
12805 ins_pipe(ialu_reg_reg);
12806 %}
12807
12808 // This pattern is automatically generated from aarch64_ad.m4
12809 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12810 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12811 %{
12812 match(Set dst (AddI src1 (AndI src2 mask)));
12813 ins_cost(INSN_COST);
12814 format %{ "addw $dst, $src1, $src2, uxtb" %}
12815
12816 ins_encode %{
12817 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12818 as_Register($src2$$reg), ext::uxtb);
12819 %}
12820 ins_pipe(ialu_reg_reg);
12821 %}
12822
12823 // This pattern is automatically generated from aarch64_ad.m4
12824 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12825 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12826 %{
12827 match(Set dst (AddI src1 (AndI src2 mask)));
12828 ins_cost(INSN_COST);
12829 format %{ "addw $dst, $src1, $src2, uxth" %}
12830
12831 ins_encode %{
12832 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12833 as_Register($src2$$reg), ext::uxth);
12834 %}
12835 ins_pipe(ialu_reg_reg);
12836 %}
12837
12838 // This pattern is automatically generated from aarch64_ad.m4
12839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12840 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12841 %{
12842 match(Set dst (AddL src1 (AndL src2 mask)));
12843 ins_cost(INSN_COST);
12844 format %{ "add $dst, $src1, $src2, uxtb" %}
12845
12846 ins_encode %{
12847 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12848 as_Register($src2$$reg), ext::uxtb);
12849 %}
12850 ins_pipe(ialu_reg_reg);
12851 %}
12852
12853 // This pattern is automatically generated from aarch64_ad.m4
12854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12855 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12856 %{
12857 match(Set dst (AddL src1 (AndL src2 mask)));
12858 ins_cost(INSN_COST);
12859 format %{ "add $dst, $src1, $src2, uxth" %}
12860
12861 ins_encode %{
12862 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12863 as_Register($src2$$reg), ext::uxth);
12864 %}
12865 ins_pipe(ialu_reg_reg);
12866 %}
12867
12868 // This pattern is automatically generated from aarch64_ad.m4
12869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12870 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12871 %{
12872 match(Set dst (AddL src1 (AndL src2 mask)));
12873 ins_cost(INSN_COST);
12874 format %{ "add $dst, $src1, $src2, uxtw" %}
12875
12876 ins_encode %{
12877 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12878 as_Register($src2$$reg), ext::uxtw);
12879 %}
12880 ins_pipe(ialu_reg_reg);
12881 %}
12882
12883 // This pattern is automatically generated from aarch64_ad.m4
12884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12885 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12886 %{
12887 match(Set dst (SubI src1 (AndI src2 mask)));
12888 ins_cost(INSN_COST);
12889 format %{ "subw $dst, $src1, $src2, uxtb" %}
12890
12891 ins_encode %{
12892 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12893 as_Register($src2$$reg), ext::uxtb);
12894 %}
12895 ins_pipe(ialu_reg_reg);
12896 %}
12897
12898 // This pattern is automatically generated from aarch64_ad.m4
12899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12900 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12901 %{
12902 match(Set dst (SubI src1 (AndI src2 mask)));
12903 ins_cost(INSN_COST);
12904 format %{ "subw $dst, $src1, $src2, uxth" %}
12905
12906 ins_encode %{
12907 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12908 as_Register($src2$$reg), ext::uxth);
12909 %}
12910 ins_pipe(ialu_reg_reg);
12911 %}
12912
12913 // This pattern is automatically generated from aarch64_ad.m4
12914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12915 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12916 %{
12917 match(Set dst (SubL src1 (AndL src2 mask)));
12918 ins_cost(INSN_COST);
12919 format %{ "sub $dst, $src1, $src2, uxtb" %}
12920
12921 ins_encode %{
12922 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12923 as_Register($src2$$reg), ext::uxtb);
12924 %}
12925 ins_pipe(ialu_reg_reg);
12926 %}
12927
12928 // This pattern is automatically generated from aarch64_ad.m4
12929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12930 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12931 %{
12932 match(Set dst (SubL src1 (AndL src2 mask)));
12933 ins_cost(INSN_COST);
12934 format %{ "sub $dst, $src1, $src2, uxth" %}
12935
12936 ins_encode %{
12937 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12938 as_Register($src2$$reg), ext::uxth);
12939 %}
12940 ins_pipe(ialu_reg_reg);
12941 %}
12942
12943 // This pattern is automatically generated from aarch64_ad.m4
12944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12945 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12946 %{
12947 match(Set dst (SubL src1 (AndL src2 mask)));
12948 ins_cost(INSN_COST);
12949 format %{ "sub $dst, $src1, $src2, uxtw" %}
12950
12951 ins_encode %{
12952 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12953 as_Register($src2$$reg), ext::uxtw);
12954 %}
12955 ins_pipe(ialu_reg_reg);
12956 %}
12957
12958
12959 // This pattern is automatically generated from aarch64_ad.m4
12960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12961 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12962 %{
12963 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12964 ins_cost(1.9 * INSN_COST);
12965 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12966
12967 ins_encode %{
12968 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12969 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12970 %}
12971 ins_pipe(ialu_reg_reg_shift);
12972 %}
12973
12974 // This pattern is automatically generated from aarch64_ad.m4
12975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12976 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12977 %{
12978 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12979 ins_cost(1.9 * INSN_COST);
12980 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12981
12982 ins_encode %{
12983 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12984 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12985 %}
12986 ins_pipe(ialu_reg_reg_shift);
12987 %}
12988
12989 // This pattern is automatically generated from aarch64_ad.m4
12990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12991 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12992 %{
12993 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12994 ins_cost(1.9 * INSN_COST);
12995 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12996
12997 ins_encode %{
12998 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12999 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13000 %}
13001 ins_pipe(ialu_reg_reg_shift);
13002 %}
13003
13004 // This pattern is automatically generated from aarch64_ad.m4
13005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13006 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13007 %{
13008 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13009 ins_cost(1.9 * INSN_COST);
13010 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13011
13012 ins_encode %{
13013 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13014 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13015 %}
13016 ins_pipe(ialu_reg_reg_shift);
13017 %}
13018
13019 // This pattern is automatically generated from aarch64_ad.m4
13020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13021 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13022 %{
13023 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13024 ins_cost(1.9 * INSN_COST);
13025 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13026
13027 ins_encode %{
13028 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13029 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13030 %}
13031 ins_pipe(ialu_reg_reg_shift);
13032 %}
13033
13034 // This pattern is automatically generated from aarch64_ad.m4
13035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13036 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13037 %{
13038 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13039 ins_cost(1.9 * INSN_COST);
13040 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13041
13042 ins_encode %{
13043 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13044 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13045 %}
13046 ins_pipe(ialu_reg_reg_shift);
13047 %}
13048
13049 // This pattern is automatically generated from aarch64_ad.m4
13050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13051 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13052 %{
13053 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13054 ins_cost(1.9 * INSN_COST);
13055 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13056
13057 ins_encode %{
13058 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13059 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13060 %}
13061 ins_pipe(ialu_reg_reg_shift);
13062 %}
13063
13064 // This pattern is automatically generated from aarch64_ad.m4
13065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13066 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13067 %{
13068 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13069 ins_cost(1.9 * INSN_COST);
13070 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13071
13072 ins_encode %{
13073 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13074 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13075 %}
13076 ins_pipe(ialu_reg_reg_shift);
13077 %}
13078
13079 // This pattern is automatically generated from aarch64_ad.m4
13080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13081 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13082 %{
13083 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13084 ins_cost(1.9 * INSN_COST);
13085 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13086
13087 ins_encode %{
13088 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13089 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13090 %}
13091 ins_pipe(ialu_reg_reg_shift);
13092 %}
13093
13094 // This pattern is automatically generated from aarch64_ad.m4
13095 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13096 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13097 %{
13098 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13099 ins_cost(1.9 * INSN_COST);
13100 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13101
13102 ins_encode %{
13103 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13104 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13105 %}
13106 ins_pipe(ialu_reg_reg_shift);
13107 %}
13108
13109 // This pattern is automatically generated from aarch64_ad.m4
13110 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13111 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13112 %{
13113 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13114 ins_cost(1.9 * INSN_COST);
13115 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13116
13117 ins_encode %{
13118 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13119 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13120 %}
13121 ins_pipe(ialu_reg_reg_shift);
13122 %}
13123
13124 // This pattern is automatically generated from aarch64_ad.m4
13125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13126 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13127 %{
13128 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13129 ins_cost(1.9 * INSN_COST);
13130 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13131
13132 ins_encode %{
13133 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13134 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13135 %}
13136 ins_pipe(ialu_reg_reg_shift);
13137 %}
13138
13139 // This pattern is automatically generated from aarch64_ad.m4
13140 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13141 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13142 %{
13143 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13144 ins_cost(1.9 * INSN_COST);
13145 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13146
13147 ins_encode %{
13148 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13149 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13150 %}
13151 ins_pipe(ialu_reg_reg_shift);
13152 %}
13153
13154 // This pattern is automatically generated from aarch64_ad.m4
13155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13156 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13157 %{
13158 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13159 ins_cost(1.9 * INSN_COST);
13160 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13161
13162 ins_encode %{
13163 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13164 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13165 %}
13166 ins_pipe(ialu_reg_reg_shift);
13167 %}
13168
13169 // This pattern is automatically generated from aarch64_ad.m4
13170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13171 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13172 %{
13173 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13174 ins_cost(1.9 * INSN_COST);
13175 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13176
13177 ins_encode %{
13178 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13179 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13180 %}
13181 ins_pipe(ialu_reg_reg_shift);
13182 %}
13183
13184 // This pattern is automatically generated from aarch64_ad.m4
13185 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13186 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13187 %{
13188 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13189 ins_cost(1.9 * INSN_COST);
13190 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13191
13192 ins_encode %{
13193 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13194 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13195 %}
13196 ins_pipe(ialu_reg_reg_shift);
13197 %}
13198
13199 // This pattern is automatically generated from aarch64_ad.m4
13200 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13201 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13202 %{
13203 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13204 ins_cost(1.9 * INSN_COST);
13205 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13206
13207 ins_encode %{
13208 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13209 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13210 %}
13211 ins_pipe(ialu_reg_reg_shift);
13212 %}
13213
13214 // This pattern is automatically generated from aarch64_ad.m4
13215 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13216 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13217 %{
13218 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13219 ins_cost(1.9 * INSN_COST);
13220 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13221
13222 ins_encode %{
13223 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13224 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13225 %}
13226 ins_pipe(ialu_reg_reg_shift);
13227 %}
13228
13229 // This pattern is automatically generated from aarch64_ad.m4
13230 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13231 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13232 %{
13233 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13234 ins_cost(1.9 * INSN_COST);
13235 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13236
13237 ins_encode %{
13238 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13239 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13240 %}
13241 ins_pipe(ialu_reg_reg_shift);
13242 %}
13243
13244 // This pattern is automatically generated from aarch64_ad.m4
13245 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13246 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13247 %{
13248 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13249 ins_cost(1.9 * INSN_COST);
13250 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13251
13252 ins_encode %{
13253 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13254 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13255 %}
13256 ins_pipe(ialu_reg_reg_shift);
13257 %}
13258
13259 // This pattern is automatically generated from aarch64_ad.m4
13260 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13261 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13262 %{
13263 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13264 ins_cost(1.9 * INSN_COST);
13265 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13266
13267 ins_encode %{
13268 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13269 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13270 %}
13271 ins_pipe(ialu_reg_reg_shift);
13272 %}
13273
13274 // This pattern is automatically generated from aarch64_ad.m4
13275 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13276 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13277 %{
13278 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13279 ins_cost(1.9 * INSN_COST);
13280 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13281
13282 ins_encode %{
13283 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13284 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13285 %}
13286 ins_pipe(ialu_reg_reg_shift);
13287 %}
13288
13289 // This pattern is automatically generated from aarch64_ad.m4
13290 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13291 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13292 %{
13293 effect(DEF dst, USE src1, USE src2, USE cr);
13294 ins_cost(INSN_COST * 2);
13295 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13296
13297 ins_encode %{
13298 __ cselw($dst$$Register,
13299 $src1$$Register,
13300 $src2$$Register,
13301 Assembler::LT);
13302 %}
13303 ins_pipe(icond_reg_reg);
13304 %}
13305
13306 // This pattern is automatically generated from aarch64_ad.m4
13307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13308 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13309 %{
13310 effect(DEF dst, USE src1, USE src2, USE cr);
13311 ins_cost(INSN_COST * 2);
13312 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13313
13314 ins_encode %{
13315 __ cselw($dst$$Register,
13316 $src1$$Register,
13317 $src2$$Register,
13318 Assembler::GT);
13319 %}
13320 ins_pipe(icond_reg_reg);
13321 %}
13322
13323 // This pattern is automatically generated from aarch64_ad.m4
13324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13325 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13326 %{
13327 effect(DEF dst, USE src1, USE cr);
13328 ins_cost(INSN_COST * 2);
13329 format %{ "cselw $dst, $src1, zr lt\t" %}
13330
13331 ins_encode %{
13332 __ cselw($dst$$Register,
13333 $src1$$Register,
13334 zr,
13335 Assembler::LT);
13336 %}
13337 ins_pipe(icond_reg);
13338 %}
13339
13340 // This pattern is automatically generated from aarch64_ad.m4
13341 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13342 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13343 %{
13344 effect(DEF dst, USE src1, USE cr);
13345 ins_cost(INSN_COST * 2);
13346 format %{ "cselw $dst, $src1, zr gt\t" %}
13347
13348 ins_encode %{
13349 __ cselw($dst$$Register,
13350 $src1$$Register,
13351 zr,
13352 Assembler::GT);
13353 %}
13354 ins_pipe(icond_reg);
13355 %}
13356
13357 // This pattern is automatically generated from aarch64_ad.m4
13358 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13359 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13360 %{
13361 effect(DEF dst, USE src1, USE cr);
13362 ins_cost(INSN_COST * 2);
13363 format %{ "csincw $dst, $src1, zr le\t" %}
13364
13365 ins_encode %{
13366 __ csincw($dst$$Register,
13367 $src1$$Register,
13368 zr,
13369 Assembler::LE);
13370 %}
13371 ins_pipe(icond_reg);
13372 %}
13373
13374 // This pattern is automatically generated from aarch64_ad.m4
13375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13376 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13377 %{
13378 effect(DEF dst, USE src1, USE cr);
13379 ins_cost(INSN_COST * 2);
13380 format %{ "csincw $dst, $src1, zr gt\t" %}
13381
13382 ins_encode %{
13383 __ csincw($dst$$Register,
13384 $src1$$Register,
13385 zr,
13386 Assembler::GT);
13387 %}
13388 ins_pipe(icond_reg);
13389 %}
13390
13391 // This pattern is automatically generated from aarch64_ad.m4
13392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13393 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13394 %{
13395 effect(DEF dst, USE src1, USE cr);
13396 ins_cost(INSN_COST * 2);
13397 format %{ "csinvw $dst, $src1, zr lt\t" %}
13398
13399 ins_encode %{
13400 __ csinvw($dst$$Register,
13401 $src1$$Register,
13402 zr,
13403 Assembler::LT);
13404 %}
13405 ins_pipe(icond_reg);
13406 %}
13407
13408 // This pattern is automatically generated from aarch64_ad.m4
13409 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13410 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13411 %{
13412 effect(DEF dst, USE src1, USE cr);
13413 ins_cost(INSN_COST * 2);
13414 format %{ "csinvw $dst, $src1, zr ge\t" %}
13415
13416 ins_encode %{
13417 __ csinvw($dst$$Register,
13418 $src1$$Register,
13419 zr,
13420 Assembler::GE);
13421 %}
13422 ins_pipe(icond_reg);
13423 %}
13424
13425 // This pattern is automatically generated from aarch64_ad.m4
13426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13427 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13428 %{
13429 match(Set dst (MinI src imm));
13430 ins_cost(INSN_COST * 3);
13431 expand %{
13432 rFlagsReg cr;
13433 compI_reg_imm0(cr, src);
13434 cmovI_reg_imm0_lt(dst, src, cr);
13435 %}
13436 %}
13437
13438 // This pattern is automatically generated from aarch64_ad.m4
13439 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13440 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13441 %{
13442 match(Set dst (MinI imm src));
13443 ins_cost(INSN_COST * 3);
13444 expand %{
13445 rFlagsReg cr;
13446 compI_reg_imm0(cr, src);
13447 cmovI_reg_imm0_lt(dst, src, cr);
13448 %}
13449 %}
13450
13451 // This pattern is automatically generated from aarch64_ad.m4
13452 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13453 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13454 %{
13455 match(Set dst (MinI src imm));
13456 ins_cost(INSN_COST * 3);
13457 expand %{
13458 rFlagsReg cr;
13459 compI_reg_imm0(cr, src);
13460 cmovI_reg_imm1_le(dst, src, cr);
13461 %}
13462 %}
13463
13464 // This pattern is automatically generated from aarch64_ad.m4
13465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13466 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13467 %{
13468 match(Set dst (MinI imm src));
13469 ins_cost(INSN_COST * 3);
13470 expand %{
13471 rFlagsReg cr;
13472 compI_reg_imm0(cr, src);
13473 cmovI_reg_imm1_le(dst, src, cr);
13474 %}
13475 %}
13476
13477 // This pattern is automatically generated from aarch64_ad.m4
13478 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13479 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13480 %{
13481 match(Set dst (MinI src imm));
13482 ins_cost(INSN_COST * 3);
13483 expand %{
13484 rFlagsReg cr;
13485 compI_reg_imm0(cr, src);
13486 cmovI_reg_immM1_lt(dst, src, cr);
13487 %}
13488 %}
13489
13490 // This pattern is automatically generated from aarch64_ad.m4
13491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13492 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13493 %{
13494 match(Set dst (MinI imm src));
13495 ins_cost(INSN_COST * 3);
13496 expand %{
13497 rFlagsReg cr;
13498 compI_reg_imm0(cr, src);
13499 cmovI_reg_immM1_lt(dst, src, cr);
13500 %}
13501 %}
13502
13503 // This pattern is automatically generated from aarch64_ad.m4
13504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13505 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13506 %{
13507 match(Set dst (MaxI src imm));
13508 ins_cost(INSN_COST * 3);
13509 expand %{
13510 rFlagsReg cr;
13511 compI_reg_imm0(cr, src);
13512 cmovI_reg_imm0_gt(dst, src, cr);
13513 %}
13514 %}
13515
13516 // This pattern is automatically generated from aarch64_ad.m4
13517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13518 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13519 %{
13520 match(Set dst (MaxI imm src));
13521 ins_cost(INSN_COST * 3);
13522 expand %{
13523 rFlagsReg cr;
13524 compI_reg_imm0(cr, src);
13525 cmovI_reg_imm0_gt(dst, src, cr);
13526 %}
13527 %}
13528
13529 // This pattern is automatically generated from aarch64_ad.m4
13530 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13531 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13532 %{
13533 match(Set dst (MaxI src imm));
13534 ins_cost(INSN_COST * 3);
13535 expand %{
13536 rFlagsReg cr;
13537 compI_reg_imm0(cr, src);
13538 cmovI_reg_imm1_gt(dst, src, cr);
13539 %}
13540 %}
13541
13542 // This pattern is automatically generated from aarch64_ad.m4
13543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13544 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13545 %{
13546 match(Set dst (MaxI imm src));
13547 ins_cost(INSN_COST * 3);
13548 expand %{
13549 rFlagsReg cr;
13550 compI_reg_imm0(cr, src);
13551 cmovI_reg_imm1_gt(dst, src, cr);
13552 %}
13553 %}
13554
13555 // This pattern is automatically generated from aarch64_ad.m4
13556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13557 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13558 %{
13559 match(Set dst (MaxI src imm));
13560 ins_cost(INSN_COST * 3);
13561 expand %{
13562 rFlagsReg cr;
13563 compI_reg_imm0(cr, src);
13564 cmovI_reg_immM1_ge(dst, src, cr);
13565 %}
13566 %}
13567
13568 // This pattern is automatically generated from aarch64_ad.m4
13569 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13570 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13571 %{
13572 match(Set dst (MaxI imm src));
13573 ins_cost(INSN_COST * 3);
13574 expand %{
13575 rFlagsReg cr;
13576 compI_reg_imm0(cr, src);
13577 cmovI_reg_immM1_ge(dst, src, cr);
13578 %}
13579 %}
13580
13581 // This pattern is automatically generated from aarch64_ad.m4
13582 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13583 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13584 %{
13585 match(Set dst (ReverseI src));
13586 ins_cost(INSN_COST);
13587 format %{ "rbitw $dst, $src" %}
13588 ins_encode %{
13589 __ rbitw($dst$$Register, $src$$Register);
13590 %}
13591 ins_pipe(ialu_reg);
13592 %}
13593
13594 // This pattern is automatically generated from aarch64_ad.m4
13595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13596 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13597 %{
13598 match(Set dst (ReverseL src));
13599 ins_cost(INSN_COST);
13600 format %{ "rbit $dst, $src" %}
13601 ins_encode %{
13602 __ rbit($dst$$Register, $src$$Register);
13603 %}
13604 ins_pipe(ialu_reg);
13605 %}
13606
13607
13608 // END This section of the file is automatically generated. Do not edit --------------
13609
13610
13611 // ============================================================================
13612 // Floating Point Arithmetic Instructions
13613
13614 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13615 match(Set dst (AddF src1 src2));
13616
13617 ins_cost(INSN_COST * 5);
13618 format %{ "fadds $dst, $src1, $src2" %}
13619
13620 ins_encode %{
13621 __ fadds(as_FloatRegister($dst$$reg),
13622 as_FloatRegister($src1$$reg),
13623 as_FloatRegister($src2$$reg));
13624 %}
13625
13626 ins_pipe(fp_dop_reg_reg_s);
13627 %}
13628
13629 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13630 match(Set dst (AddD src1 src2));
13631
13632 ins_cost(INSN_COST * 5);
13633 format %{ "faddd $dst, $src1, $src2" %}
13634
13635 ins_encode %{
13636 __ faddd(as_FloatRegister($dst$$reg),
13637 as_FloatRegister($src1$$reg),
13638 as_FloatRegister($src2$$reg));
13639 %}
13640
13641 ins_pipe(fp_dop_reg_reg_d);
13642 %}
13643
13644 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13645 match(Set dst (SubF src1 src2));
13646
13647 ins_cost(INSN_COST * 5);
13648 format %{ "fsubs $dst, $src1, $src2" %}
13649
13650 ins_encode %{
13651 __ fsubs(as_FloatRegister($dst$$reg),
13652 as_FloatRegister($src1$$reg),
13653 as_FloatRegister($src2$$reg));
13654 %}
13655
13656 ins_pipe(fp_dop_reg_reg_s);
13657 %}
13658
13659 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13660 match(Set dst (SubD src1 src2));
13661
13662 ins_cost(INSN_COST * 5);
13663 format %{ "fsubd $dst, $src1, $src2" %}
13664
13665 ins_encode %{
13666 __ fsubd(as_FloatRegister($dst$$reg),
13667 as_FloatRegister($src1$$reg),
13668 as_FloatRegister($src2$$reg));
13669 %}
13670
13671 ins_pipe(fp_dop_reg_reg_d);
13672 %}
13673
13674 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13675 match(Set dst (MulF src1 src2));
13676
13677 ins_cost(INSN_COST * 6);
13678 format %{ "fmuls $dst, $src1, $src2" %}
13679
13680 ins_encode %{
13681 __ fmuls(as_FloatRegister($dst$$reg),
13682 as_FloatRegister($src1$$reg),
13683 as_FloatRegister($src2$$reg));
13684 %}
13685
13686 ins_pipe(fp_dop_reg_reg_s);
13687 %}
13688
13689 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13690 match(Set dst (MulD src1 src2));
13691
13692 ins_cost(INSN_COST * 6);
13693 format %{ "fmuld $dst, $src1, $src2" %}
13694
13695 ins_encode %{
13696 __ fmuld(as_FloatRegister($dst$$reg),
13697 as_FloatRegister($src1$$reg),
13698 as_FloatRegister($src2$$reg));
13699 %}
13700
13701 ins_pipe(fp_dop_reg_reg_d);
13702 %}
13703
13704 // src1 * src2 + src3
13705 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13706 match(Set dst (FmaF src3 (Binary src1 src2)));
13707
13708 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13709
13710 ins_encode %{
13711 assert(UseFMA, "Needs FMA instructions support.");
13712 __ fmadds(as_FloatRegister($dst$$reg),
13713 as_FloatRegister($src1$$reg),
13714 as_FloatRegister($src2$$reg),
13715 as_FloatRegister($src3$$reg));
13716 %}
13717
13718 ins_pipe(pipe_class_default);
13719 %}
13720
13721 // src1 * src2 + src3
13722 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13723 match(Set dst (FmaD src3 (Binary src1 src2)));
13724
13725 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13726
13727 ins_encode %{
13728 assert(UseFMA, "Needs FMA instructions support.");
13729 __ fmaddd(as_FloatRegister($dst$$reg),
13730 as_FloatRegister($src1$$reg),
13731 as_FloatRegister($src2$$reg),
13732 as_FloatRegister($src3$$reg));
13733 %}
13734
13735 ins_pipe(pipe_class_default);
13736 %}
13737
13738 // src1 * (-src2) + src3
13739 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13740 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13741 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13742
13743 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13744
13745 ins_encode %{
13746 assert(UseFMA, "Needs FMA instructions support.");
13747 __ fmsubs(as_FloatRegister($dst$$reg),
13748 as_FloatRegister($src1$$reg),
13749 as_FloatRegister($src2$$reg),
13750 as_FloatRegister($src3$$reg));
13751 %}
13752
13753 ins_pipe(pipe_class_default);
13754 %}
13755
13756 // src1 * (-src2) + src3
13757 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13758 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13759 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13760
13761 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13762
13763 ins_encode %{
13764 assert(UseFMA, "Needs FMA instructions support.");
13765 __ fmsubd(as_FloatRegister($dst$$reg),
13766 as_FloatRegister($src1$$reg),
13767 as_FloatRegister($src2$$reg),
13768 as_FloatRegister($src3$$reg));
13769 %}
13770
13771 ins_pipe(pipe_class_default);
13772 %}
13773
13774 // src1 * (-src2) - src3
13775 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13776 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13777 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13778
13779 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13780
13781 ins_encode %{
13782 assert(UseFMA, "Needs FMA instructions support.");
13783 __ fnmadds(as_FloatRegister($dst$$reg),
13784 as_FloatRegister($src1$$reg),
13785 as_FloatRegister($src2$$reg),
13786 as_FloatRegister($src3$$reg));
13787 %}
13788
13789 ins_pipe(pipe_class_default);
13790 %}
13791
13792 // src1 * (-src2) - src3
13793 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13794 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13795 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13796
13797 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13798
13799 ins_encode %{
13800 assert(UseFMA, "Needs FMA instructions support.");
13801 __ fnmaddd(as_FloatRegister($dst$$reg),
13802 as_FloatRegister($src1$$reg),
13803 as_FloatRegister($src2$$reg),
13804 as_FloatRegister($src3$$reg));
13805 %}
13806
13807 ins_pipe(pipe_class_default);
13808 %}
13809
13810 // src1 * src2 - src3
13811 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13812 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13813
13814 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13815
13816 ins_encode %{
13817 assert(UseFMA, "Needs FMA instructions support.");
13818 __ fnmsubs(as_FloatRegister($dst$$reg),
13819 as_FloatRegister($src1$$reg),
13820 as_FloatRegister($src2$$reg),
13821 as_FloatRegister($src3$$reg));
13822 %}
13823
13824 ins_pipe(pipe_class_default);
13825 %}
13826
13827 // src1 * src2 - src3
13828 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13829 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13830
13831 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13832
13833 ins_encode %{
13834 assert(UseFMA, "Needs FMA instructions support.");
13835 // n.b. insn name should be fnmsubd
13836 __ fnmsub(as_FloatRegister($dst$$reg),
13837 as_FloatRegister($src1$$reg),
13838 as_FloatRegister($src2$$reg),
13839 as_FloatRegister($src3$$reg));
13840 %}
13841
13842 ins_pipe(pipe_class_default);
13843 %}
13844
13845
13846 // Math.max(FF)F
13847 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13848 match(Set dst (MaxF src1 src2));
13849
13850 format %{ "fmaxs $dst, $src1, $src2" %}
13851 ins_encode %{
13852 __ fmaxs(as_FloatRegister($dst$$reg),
13853 as_FloatRegister($src1$$reg),
13854 as_FloatRegister($src2$$reg));
13855 %}
13856
13857 ins_pipe(fp_dop_reg_reg_s);
13858 %}
13859
13860 // Math.min(FF)F
13861 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13862 match(Set dst (MinF src1 src2));
13863
13864 format %{ "fmins $dst, $src1, $src2" %}
13865 ins_encode %{
13866 __ fmins(as_FloatRegister($dst$$reg),
13867 as_FloatRegister($src1$$reg),
13868 as_FloatRegister($src2$$reg));
13869 %}
13870
13871 ins_pipe(fp_dop_reg_reg_s);
13872 %}
13873
13874 // Math.max(DD)D
13875 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13876 match(Set dst (MaxD src1 src2));
13877
13878 format %{ "fmaxd $dst, $src1, $src2" %}
13879 ins_encode %{
13880 __ fmaxd(as_FloatRegister($dst$$reg),
13881 as_FloatRegister($src1$$reg),
13882 as_FloatRegister($src2$$reg));
13883 %}
13884
13885 ins_pipe(fp_dop_reg_reg_d);
13886 %}
13887
13888 // Math.min(DD)D
13889 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13890 match(Set dst (MinD src1 src2));
13891
13892 format %{ "fmind $dst, $src1, $src2" %}
13893 ins_encode %{
13894 __ fmind(as_FloatRegister($dst$$reg),
13895 as_FloatRegister($src1$$reg),
13896 as_FloatRegister($src2$$reg));
13897 %}
13898
13899 ins_pipe(fp_dop_reg_reg_d);
13900 %}
13901
13902
13903 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13904 match(Set dst (DivF src1 src2));
13905
13906 ins_cost(INSN_COST * 18);
13907 format %{ "fdivs $dst, $src1, $src2" %}
13908
13909 ins_encode %{
13910 __ fdivs(as_FloatRegister($dst$$reg),
13911 as_FloatRegister($src1$$reg),
13912 as_FloatRegister($src2$$reg));
13913 %}
13914
13915 ins_pipe(fp_div_s);
13916 %}
13917
13918 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13919 match(Set dst (DivD src1 src2));
13920
13921 ins_cost(INSN_COST * 32);
13922 format %{ "fdivd $dst, $src1, $src2" %}
13923
13924 ins_encode %{
13925 __ fdivd(as_FloatRegister($dst$$reg),
13926 as_FloatRegister($src1$$reg),
13927 as_FloatRegister($src2$$reg));
13928 %}
13929
13930 ins_pipe(fp_div_d);
13931 %}
13932
13933 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13934 match(Set dst (NegF src));
13935
13936 ins_cost(INSN_COST * 3);
13937 format %{ "fneg $dst, $src" %}
13938
13939 ins_encode %{
13940 __ fnegs(as_FloatRegister($dst$$reg),
13941 as_FloatRegister($src$$reg));
13942 %}
13943
13944 ins_pipe(fp_uop_s);
13945 %}
13946
13947 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13948 match(Set dst (NegD src));
13949
13950 ins_cost(INSN_COST * 3);
13951 format %{ "fnegd $dst, $src" %}
13952
13953 ins_encode %{
13954 __ fnegd(as_FloatRegister($dst$$reg),
13955 as_FloatRegister($src$$reg));
13956 %}
13957
13958 ins_pipe(fp_uop_d);
13959 %}
13960
13961 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13962 %{
13963 match(Set dst (AbsI src));
13964
13965 effect(KILL cr);
13966 ins_cost(INSN_COST * 2);
13967 format %{ "cmpw $src, zr\n\t"
13968 "cnegw $dst, $src, Assembler::LT\t# int abs"
13969 %}
13970
13971 ins_encode %{
13972 __ cmpw(as_Register($src$$reg), zr);
13973 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13974 %}
13975 ins_pipe(pipe_class_default);
13976 %}
13977
13978 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13979 %{
13980 match(Set dst (AbsL src));
13981
13982 effect(KILL cr);
13983 ins_cost(INSN_COST * 2);
13984 format %{ "cmp $src, zr\n\t"
13985 "cneg $dst, $src, Assembler::LT\t# long abs"
13986 %}
13987
13988 ins_encode %{
13989 __ cmp(as_Register($src$$reg), zr);
13990 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13991 %}
13992 ins_pipe(pipe_class_default);
13993 %}
13994
13995 instruct absF_reg(vRegF dst, vRegF src) %{
13996 match(Set dst (AbsF src));
13997
13998 ins_cost(INSN_COST * 3);
13999 format %{ "fabss $dst, $src" %}
14000 ins_encode %{
14001 __ fabss(as_FloatRegister($dst$$reg),
14002 as_FloatRegister($src$$reg));
14003 %}
14004
14005 ins_pipe(fp_uop_s);
14006 %}
14007
14008 instruct absD_reg(vRegD dst, vRegD src) %{
14009 match(Set dst (AbsD src));
14010
14011 ins_cost(INSN_COST * 3);
14012 format %{ "fabsd $dst, $src" %}
14013 ins_encode %{
14014 __ fabsd(as_FloatRegister($dst$$reg),
14015 as_FloatRegister($src$$reg));
14016 %}
14017
14018 ins_pipe(fp_uop_d);
14019 %}
14020
14021 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14022 match(Set dst (AbsF (SubF src1 src2)));
14023
14024 ins_cost(INSN_COST * 3);
14025 format %{ "fabds $dst, $src1, $src2" %}
14026 ins_encode %{
14027 __ fabds(as_FloatRegister($dst$$reg),
14028 as_FloatRegister($src1$$reg),
14029 as_FloatRegister($src2$$reg));
14030 %}
14031
14032 ins_pipe(fp_uop_s);
14033 %}
14034
14035 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14036 match(Set dst (AbsD (SubD src1 src2)));
14037
14038 ins_cost(INSN_COST * 3);
14039 format %{ "fabdd $dst, $src1, $src2" %}
14040 ins_encode %{
14041 __ fabdd(as_FloatRegister($dst$$reg),
14042 as_FloatRegister($src1$$reg),
14043 as_FloatRegister($src2$$reg));
14044 %}
14045
14046 ins_pipe(fp_uop_d);
14047 %}
14048
14049 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14050 match(Set dst (SqrtD src));
14051
14052 ins_cost(INSN_COST * 50);
14053 format %{ "fsqrtd $dst, $src" %}
14054 ins_encode %{
14055 __ fsqrtd(as_FloatRegister($dst$$reg),
14056 as_FloatRegister($src$$reg));
14057 %}
14058
14059 ins_pipe(fp_div_s);
14060 %}
14061
14062 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14063 match(Set dst (SqrtF src));
14064
14065 ins_cost(INSN_COST * 50);
14066 format %{ "fsqrts $dst, $src" %}
14067 ins_encode %{
14068 __ fsqrts(as_FloatRegister($dst$$reg),
14069 as_FloatRegister($src$$reg));
14070 %}
14071
14072 ins_pipe(fp_div_d);
14073 %}
14074
14075 // Math.rint, floor, ceil
14076 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14077 match(Set dst (RoundDoubleMode src rmode));
14078 format %{ "frint $dst, $src, $rmode" %}
14079 ins_encode %{
14080 switch ($rmode$$constant) {
14081 case RoundDoubleModeNode::rmode_rint:
14082 __ frintnd(as_FloatRegister($dst$$reg),
14083 as_FloatRegister($src$$reg));
14084 break;
14085 case RoundDoubleModeNode::rmode_floor:
14086 __ frintmd(as_FloatRegister($dst$$reg),
14087 as_FloatRegister($src$$reg));
14088 break;
14089 case RoundDoubleModeNode::rmode_ceil:
14090 __ frintpd(as_FloatRegister($dst$$reg),
14091 as_FloatRegister($src$$reg));
14092 break;
14093 }
14094 %}
14095 ins_pipe(fp_uop_d);
14096 %}
14097
14098 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14099 match(Set dst (CopySignD src1 (Binary src2 zero)));
14100 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14101 format %{ "CopySignD $dst $src1 $src2" %}
14102 ins_encode %{
14103 FloatRegister dst = as_FloatRegister($dst$$reg),
14104 src1 = as_FloatRegister($src1$$reg),
14105 src2 = as_FloatRegister($src2$$reg),
14106 zero = as_FloatRegister($zero$$reg);
14107 __ fnegd(dst, zero);
14108 __ bsl(dst, __ T8B, src2, src1);
14109 %}
14110 ins_pipe(fp_uop_d);
14111 %}
14112
14113 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14114 match(Set dst (CopySignF src1 src2));
14115 effect(TEMP_DEF dst, USE src1, USE src2);
14116 format %{ "CopySignF $dst $src1 $src2" %}
14117 ins_encode %{
14118 FloatRegister dst = as_FloatRegister($dst$$reg),
14119 src1 = as_FloatRegister($src1$$reg),
14120 src2 = as_FloatRegister($src2$$reg);
14121 __ movi(dst, __ T2S, 0x80, 24);
14122 __ bsl(dst, __ T8B, src2, src1);
14123 %}
14124 ins_pipe(fp_uop_d);
14125 %}
14126
14127 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14128 match(Set dst (SignumD src (Binary zero one)));
14129 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14130 format %{ "signumD $dst, $src" %}
14131 ins_encode %{
14132 FloatRegister src = as_FloatRegister($src$$reg),
14133 dst = as_FloatRegister($dst$$reg),
14134 zero = as_FloatRegister($zero$$reg),
14135 one = as_FloatRegister($one$$reg);
14136 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14137 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14138 // Bit selection instruction gets bit from "one" for each enabled bit in
14139 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14140 // NaN the whole "src" will be copied because "dst" is zero. For all other
14141 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14142 // from "src", and all other bits are copied from 1.0.
14143 __ bsl(dst, __ T8B, one, src);
14144 %}
14145 ins_pipe(fp_uop_d);
14146 %}
14147
14148 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14149 match(Set dst (SignumF src (Binary zero one)));
14150 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14151 format %{ "signumF $dst, $src" %}
14152 ins_encode %{
14153 FloatRegister src = as_FloatRegister($src$$reg),
14154 dst = as_FloatRegister($dst$$reg),
14155 zero = as_FloatRegister($zero$$reg),
14156 one = as_FloatRegister($one$$reg);
14157 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14158 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14159 // Bit selection instruction gets bit from "one" for each enabled bit in
14160 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14161 // NaN the whole "src" will be copied because "dst" is zero. For all other
14162 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14163 // from "src", and all other bits are copied from 1.0.
14164 __ bsl(dst, __ T8B, one, src);
14165 %}
14166 ins_pipe(fp_uop_d);
14167 %}
14168
14169 instruct onspinwait() %{
14170 match(OnSpinWait);
14171 ins_cost(INSN_COST);
14172
14173 format %{ "onspinwait" %}
14174
14175 ins_encode %{
14176 __ spin_wait();
14177 %}
14178 ins_pipe(pipe_class_empty);
14179 %}
14180
14181 // ============================================================================
14182 // Logical Instructions
14183
14184 // Integer Logical Instructions
14185
14186 // And Instructions
14187
14188
14189 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14190 match(Set dst (AndI src1 src2));
14191
14192 format %{ "andw $dst, $src1, $src2\t# int" %}
14193
14194 ins_cost(INSN_COST);
14195 ins_encode %{
14196 __ andw(as_Register($dst$$reg),
14197 as_Register($src1$$reg),
14198 as_Register($src2$$reg));
14199 %}
14200
14201 ins_pipe(ialu_reg_reg);
14202 %}
14203
14204 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14205 match(Set dst (AndI src1 src2));
14206
14207 format %{ "andsw $dst, $src1, $src2\t# int" %}
14208
14209 ins_cost(INSN_COST);
14210 ins_encode %{
14211 __ andw(as_Register($dst$$reg),
14212 as_Register($src1$$reg),
14213 (uint64_t)($src2$$constant));
14214 %}
14215
14216 ins_pipe(ialu_reg_imm);
14217 %}
14218
14219 // Or Instructions
14220
14221 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14222 match(Set dst (OrI src1 src2));
14223
14224 format %{ "orrw $dst, $src1, $src2\t# int" %}
14225
14226 ins_cost(INSN_COST);
14227 ins_encode %{
14228 __ orrw(as_Register($dst$$reg),
14229 as_Register($src1$$reg),
14230 as_Register($src2$$reg));
14231 %}
14232
14233 ins_pipe(ialu_reg_reg);
14234 %}
14235
14236 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14237 match(Set dst (OrI src1 src2));
14238
14239 format %{ "orrw $dst, $src1, $src2\t# int" %}
14240
14241 ins_cost(INSN_COST);
14242 ins_encode %{
14243 __ orrw(as_Register($dst$$reg),
14244 as_Register($src1$$reg),
14245 (uint64_t)($src2$$constant));
14246 %}
14247
14248 ins_pipe(ialu_reg_imm);
14249 %}
14250
14251 // Xor Instructions
14252
14253 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14254 match(Set dst (XorI src1 src2));
14255
14256 format %{ "eorw $dst, $src1, $src2\t# int" %}
14257
14258 ins_cost(INSN_COST);
14259 ins_encode %{
14260 __ eorw(as_Register($dst$$reg),
14261 as_Register($src1$$reg),
14262 as_Register($src2$$reg));
14263 %}
14264
14265 ins_pipe(ialu_reg_reg);
14266 %}
14267
14268 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14269 match(Set dst (XorI src1 src2));
14270
14271 format %{ "eorw $dst, $src1, $src2\t# int" %}
14272
14273 ins_cost(INSN_COST);
14274 ins_encode %{
14275 __ eorw(as_Register($dst$$reg),
14276 as_Register($src1$$reg),
14277 (uint64_t)($src2$$constant));
14278 %}
14279
14280 ins_pipe(ialu_reg_imm);
14281 %}
14282
14283 // Long Logical Instructions
14284 // TODO
14285
14286 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14287 match(Set dst (AndL src1 src2));
14288
14289 format %{ "and $dst, $src1, $src2\t# int" %}
14290
14291 ins_cost(INSN_COST);
14292 ins_encode %{
14293 __ andr(as_Register($dst$$reg),
14294 as_Register($src1$$reg),
14295 as_Register($src2$$reg));
14296 %}
14297
14298 ins_pipe(ialu_reg_reg);
14299 %}
14300
14301 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14302 match(Set dst (AndL src1 src2));
14303
14304 format %{ "and $dst, $src1, $src2\t# int" %}
14305
14306 ins_cost(INSN_COST);
14307 ins_encode %{
14308 __ andr(as_Register($dst$$reg),
14309 as_Register($src1$$reg),
14310 (uint64_t)($src2$$constant));
14311 %}
14312
14313 ins_pipe(ialu_reg_imm);
14314 %}
14315
14316 // Or Instructions
14317
14318 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14319 match(Set dst (OrL src1 src2));
14320
14321 format %{ "orr $dst, $src1, $src2\t# int" %}
14322
14323 ins_cost(INSN_COST);
14324 ins_encode %{
14325 __ orr(as_Register($dst$$reg),
14326 as_Register($src1$$reg),
14327 as_Register($src2$$reg));
14328 %}
14329
14330 ins_pipe(ialu_reg_reg);
14331 %}
14332
14333 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14334 match(Set dst (OrL src1 src2));
14335
14336 format %{ "orr $dst, $src1, $src2\t# int" %}
14337
14338 ins_cost(INSN_COST);
14339 ins_encode %{
14340 __ orr(as_Register($dst$$reg),
14341 as_Register($src1$$reg),
14342 (uint64_t)($src2$$constant));
14343 %}
14344
14345 ins_pipe(ialu_reg_imm);
14346 %}
14347
14348 // Xor Instructions
14349
14350 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14351 match(Set dst (XorL src1 src2));
14352
14353 format %{ "eor $dst, $src1, $src2\t# int" %}
14354
14355 ins_cost(INSN_COST);
14356 ins_encode %{
14357 __ eor(as_Register($dst$$reg),
14358 as_Register($src1$$reg),
14359 as_Register($src2$$reg));
14360 %}
14361
14362 ins_pipe(ialu_reg_reg);
14363 %}
14364
14365 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14366 match(Set dst (XorL src1 src2));
14367
14368 ins_cost(INSN_COST);
14369 format %{ "eor $dst, $src1, $src2\t# int" %}
14370
14371 ins_encode %{
14372 __ eor(as_Register($dst$$reg),
14373 as_Register($src1$$reg),
14374 (uint64_t)($src2$$constant));
14375 %}
14376
14377 ins_pipe(ialu_reg_imm);
14378 %}
14379
14380 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14381 %{
14382 match(Set dst (ConvI2L src));
14383
14384 ins_cost(INSN_COST);
14385 format %{ "sxtw $dst, $src\t# i2l" %}
14386 ins_encode %{
14387 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14388 %}
14389 ins_pipe(ialu_reg_shift);
14390 %}
14391
14392 // this pattern occurs in bigmath arithmetic
14393 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14394 %{
14395 match(Set dst (AndL (ConvI2L src) mask));
14396
14397 ins_cost(INSN_COST);
14398 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14399 ins_encode %{
14400 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14401 %}
14402
14403 ins_pipe(ialu_reg_shift);
14404 %}
14405
14406 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14407 match(Set dst (ConvL2I src));
14408
14409 ins_cost(INSN_COST);
14410 format %{ "movw $dst, $src \t// l2i" %}
14411
14412 ins_encode %{
14413 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14414 %}
14415
14416 ins_pipe(ialu_reg);
14417 %}
14418
14419 instruct convD2F_reg(vRegF dst, vRegD src) %{
14420 match(Set dst (ConvD2F src));
14421
14422 ins_cost(INSN_COST * 5);
14423 format %{ "fcvtd $dst, $src \t// d2f" %}
14424
14425 ins_encode %{
14426 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14427 %}
14428
14429 ins_pipe(fp_d2f);
14430 %}
14431
14432 instruct convF2D_reg(vRegD dst, vRegF src) %{
14433 match(Set dst (ConvF2D src));
14434
14435 ins_cost(INSN_COST * 5);
14436 format %{ "fcvts $dst, $src \t// f2d" %}
14437
14438 ins_encode %{
14439 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14440 %}
14441
14442 ins_pipe(fp_f2d);
14443 %}
14444
14445 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14446 match(Set dst (ConvF2I src));
14447
14448 ins_cost(INSN_COST * 5);
14449 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14450
14451 ins_encode %{
14452 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14453 %}
14454
14455 ins_pipe(fp_f2i);
14456 %}
14457
14458 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14459 match(Set dst (ConvF2L src));
14460
14461 ins_cost(INSN_COST * 5);
14462 format %{ "fcvtzs $dst, $src \t// f2l" %}
14463
14464 ins_encode %{
14465 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14466 %}
14467
14468 ins_pipe(fp_f2l);
14469 %}
14470
14471 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14472 match(Set dst (ConvF2HF src));
14473 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14474 "smov $dst, $tmp\t# move result from $tmp to $dst"
14475 %}
14476 effect(TEMP tmp);
14477 ins_encode %{
14478 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14479 %}
14480 ins_pipe(pipe_slow);
14481 %}
14482
14483 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14484 match(Set dst (ConvHF2F src));
14485 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14486 "fcvt $dst, $tmp\t# convert half to single precision"
14487 %}
14488 effect(TEMP tmp);
14489 ins_encode %{
14490 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14491 %}
14492 ins_pipe(pipe_slow);
14493 %}
14494
14495 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14496 match(Set dst (ConvI2F src));
14497
14498 ins_cost(INSN_COST * 5);
14499 format %{ "scvtfws $dst, $src \t// i2f" %}
14500
14501 ins_encode %{
14502 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14503 %}
14504
14505 ins_pipe(fp_i2f);
14506 %}
14507
14508 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14509 match(Set dst (ConvL2F src));
14510
14511 ins_cost(INSN_COST * 5);
14512 format %{ "scvtfs $dst, $src \t// l2f" %}
14513
14514 ins_encode %{
14515 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14516 %}
14517
14518 ins_pipe(fp_l2f);
14519 %}
14520
14521 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14522 match(Set dst (ConvD2I src));
14523
14524 ins_cost(INSN_COST * 5);
14525 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14526
14527 ins_encode %{
14528 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14529 %}
14530
14531 ins_pipe(fp_d2i);
14532 %}
14533
14534 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14535 match(Set dst (ConvD2L src));
14536
14537 ins_cost(INSN_COST * 5);
14538 format %{ "fcvtzd $dst, $src \t// d2l" %}
14539
14540 ins_encode %{
14541 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14542 %}
14543
14544 ins_pipe(fp_d2l);
14545 %}
14546
14547 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14548 match(Set dst (ConvI2D src));
14549
14550 ins_cost(INSN_COST * 5);
14551 format %{ "scvtfwd $dst, $src \t// i2d" %}
14552
14553 ins_encode %{
14554 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14555 %}
14556
14557 ins_pipe(fp_i2d);
14558 %}
14559
14560 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14561 match(Set dst (ConvL2D src));
14562
14563 ins_cost(INSN_COST * 5);
14564 format %{ "scvtfd $dst, $src \t// l2d" %}
14565
14566 ins_encode %{
14567 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14568 %}
14569
14570 ins_pipe(fp_l2d);
14571 %}
14572
14573 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14574 %{
14575 match(Set dst (RoundD src));
14576 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14577 format %{ "java_round_double $dst,$src"%}
14578 ins_encode %{
14579 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14580 as_FloatRegister($ftmp$$reg));
14581 %}
14582 ins_pipe(pipe_slow);
14583 %}
14584
14585 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14586 %{
14587 match(Set dst (RoundF src));
14588 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14589 format %{ "java_round_float $dst,$src"%}
14590 ins_encode %{
14591 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14592 as_FloatRegister($ftmp$$reg));
14593 %}
14594 ins_pipe(pipe_slow);
14595 %}
14596
14597 // stack <-> reg and reg <-> reg shuffles with no conversion
14598
14599 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14600
14601 match(Set dst (MoveF2I src));
14602
14603 effect(DEF dst, USE src);
14604
14605 ins_cost(4 * INSN_COST);
14606
14607 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14608
14609 ins_encode %{
14610 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14611 %}
14612
14613 ins_pipe(iload_reg_reg);
14614
14615 %}
14616
14617 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14618
14619 match(Set dst (MoveI2F src));
14620
14621 effect(DEF dst, USE src);
14622
14623 ins_cost(4 * INSN_COST);
14624
14625 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14626
14627 ins_encode %{
14628 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14629 %}
14630
14631 ins_pipe(pipe_class_memory);
14632
14633 %}
14634
14635 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14636
14637 match(Set dst (MoveD2L src));
14638
14639 effect(DEF dst, USE src);
14640
14641 ins_cost(4 * INSN_COST);
14642
14643 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14644
14645 ins_encode %{
14646 __ ldr($dst$$Register, Address(sp, $src$$disp));
14647 %}
14648
14649 ins_pipe(iload_reg_reg);
14650
14651 %}
14652
14653 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14654
14655 match(Set dst (MoveL2D src));
14656
14657 effect(DEF dst, USE src);
14658
14659 ins_cost(4 * INSN_COST);
14660
14661 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14662
14663 ins_encode %{
14664 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14665 %}
14666
14667 ins_pipe(pipe_class_memory);
14668
14669 %}
14670
14671 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14672
14673 match(Set dst (MoveF2I src));
14674
14675 effect(DEF dst, USE src);
14676
14677 ins_cost(INSN_COST);
14678
14679 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14680
14681 ins_encode %{
14682 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14683 %}
14684
14685 ins_pipe(pipe_class_memory);
14686
14687 %}
14688
14689 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14690
14691 match(Set dst (MoveI2F src));
14692
14693 effect(DEF dst, USE src);
14694
14695 ins_cost(INSN_COST);
14696
14697 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14698
14699 ins_encode %{
14700 __ strw($src$$Register, Address(sp, $dst$$disp));
14701 %}
14702
14703 ins_pipe(istore_reg_reg);
14704
14705 %}
14706
14707 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14708
14709 match(Set dst (MoveD2L src));
14710
14711 effect(DEF dst, USE src);
14712
14713 ins_cost(INSN_COST);
14714
14715 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14716
14717 ins_encode %{
14718 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14719 %}
14720
14721 ins_pipe(pipe_class_memory);
14722
14723 %}
14724
14725 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14726
14727 match(Set dst (MoveL2D src));
14728
14729 effect(DEF dst, USE src);
14730
14731 ins_cost(INSN_COST);
14732
14733 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14734
14735 ins_encode %{
14736 __ str($src$$Register, Address(sp, $dst$$disp));
14737 %}
14738
14739 ins_pipe(istore_reg_reg);
14740
14741 %}
14742
14743 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14744
14745 match(Set dst (MoveF2I src));
14746
14747 effect(DEF dst, USE src);
14748
14749 ins_cost(INSN_COST);
14750
14751 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14752
14753 ins_encode %{
14754 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14755 %}
14756
14757 ins_pipe(fp_f2i);
14758
14759 %}
14760
14761 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14762
14763 match(Set dst (MoveI2F src));
14764
14765 effect(DEF dst, USE src);
14766
14767 ins_cost(INSN_COST);
14768
14769 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14770
14771 ins_encode %{
14772 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14773 %}
14774
14775 ins_pipe(fp_i2f);
14776
14777 %}
14778
14779 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14780
14781 match(Set dst (MoveD2L src));
14782
14783 effect(DEF dst, USE src);
14784
14785 ins_cost(INSN_COST);
14786
14787 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14788
14789 ins_encode %{
14790 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14791 %}
14792
14793 ins_pipe(fp_d2l);
14794
14795 %}
14796
14797 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14798
14799 match(Set dst (MoveL2D src));
14800
14801 effect(DEF dst, USE src);
14802
14803 ins_cost(INSN_COST);
14804
14805 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14806
14807 ins_encode %{
14808 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14809 %}
14810
14811 ins_pipe(fp_l2d);
14812
14813 %}
14814
14815 // ============================================================================
14816 // clearing of an array
14817
14818 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14819 %{
14820 match(Set dummy (ClearArray cnt base));
14821 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14822
14823 ins_cost(4 * INSN_COST);
14824 format %{ "ClearArray $cnt, $base" %}
14825
14826 ins_encode %{
14827 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14828 if (tpc == nullptr) {
14829 ciEnv::current()->record_failure("CodeCache is full");
14830 return;
14831 }
14832 %}
14833
14834 ins_pipe(pipe_class_memory);
14835 %}
14836
14837 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14838 %{
14839 predicate((uint64_t)n->in(2)->get_long()
14840 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14841 match(Set dummy (ClearArray cnt base));
14842 effect(TEMP temp, USE_KILL base, KILL cr);
14843
14844 ins_cost(4 * INSN_COST);
14845 format %{ "ClearArray $cnt, $base" %}
14846
14847 ins_encode %{
14848 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14849 if (tpc == nullptr) {
14850 ciEnv::current()->record_failure("CodeCache is full");
14851 return;
14852 }
14853 %}
14854
14855 ins_pipe(pipe_class_memory);
14856 %}
14857
14858 // ============================================================================
14859 // Overflow Math Instructions
14860
14861 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14862 %{
14863 match(Set cr (OverflowAddI op1 op2));
14864
14865 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14866 ins_cost(INSN_COST);
14867 ins_encode %{
14868 __ cmnw($op1$$Register, $op2$$Register);
14869 %}
14870
14871 ins_pipe(icmp_reg_reg);
14872 %}
14873
14874 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14875 %{
14876 match(Set cr (OverflowAddI op1 op2));
14877
14878 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14879 ins_cost(INSN_COST);
14880 ins_encode %{
14881 __ cmnw($op1$$Register, $op2$$constant);
14882 %}
14883
14884 ins_pipe(icmp_reg_imm);
14885 %}
14886
14887 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14888 %{
14889 match(Set cr (OverflowAddL op1 op2));
14890
14891 format %{ "cmn $op1, $op2\t# overflow check long" %}
14892 ins_cost(INSN_COST);
14893 ins_encode %{
14894 __ cmn($op1$$Register, $op2$$Register);
14895 %}
14896
14897 ins_pipe(icmp_reg_reg);
14898 %}
14899
14900 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14901 %{
14902 match(Set cr (OverflowAddL op1 op2));
14903
14904 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14905 ins_cost(INSN_COST);
14906 ins_encode %{
14907 __ adds(zr, $op1$$Register, $op2$$constant);
14908 %}
14909
14910 ins_pipe(icmp_reg_imm);
14911 %}
14912
14913 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14914 %{
14915 match(Set cr (OverflowSubI op1 op2));
14916
14917 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14918 ins_cost(INSN_COST);
14919 ins_encode %{
14920 __ cmpw($op1$$Register, $op2$$Register);
14921 %}
14922
14923 ins_pipe(icmp_reg_reg);
14924 %}
14925
14926 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14927 %{
14928 match(Set cr (OverflowSubI op1 op2));
14929
14930 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14931 ins_cost(INSN_COST);
14932 ins_encode %{
14933 __ cmpw($op1$$Register, $op2$$constant);
14934 %}
14935
14936 ins_pipe(icmp_reg_imm);
14937 %}
14938
14939 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14940 %{
14941 match(Set cr (OverflowSubL op1 op2));
14942
14943 format %{ "cmp $op1, $op2\t# overflow check long" %}
14944 ins_cost(INSN_COST);
14945 ins_encode %{
14946 __ cmp($op1$$Register, $op2$$Register);
14947 %}
14948
14949 ins_pipe(icmp_reg_reg);
14950 %}
14951
14952 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14953 %{
14954 match(Set cr (OverflowSubL op1 op2));
14955
14956 format %{ "cmp $op1, $op2\t# overflow check long" %}
14957 ins_cost(INSN_COST);
14958 ins_encode %{
14959 __ subs(zr, $op1$$Register, $op2$$constant);
14960 %}
14961
14962 ins_pipe(icmp_reg_imm);
14963 %}
14964
14965 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14966 %{
14967 match(Set cr (OverflowSubI zero op1));
14968
14969 format %{ "cmpw zr, $op1\t# overflow check int" %}
14970 ins_cost(INSN_COST);
14971 ins_encode %{
14972 __ cmpw(zr, $op1$$Register);
14973 %}
14974
14975 ins_pipe(icmp_reg_imm);
14976 %}
14977
14978 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14979 %{
14980 match(Set cr (OverflowSubL zero op1));
14981
14982 format %{ "cmp zr, $op1\t# overflow check long" %}
14983 ins_cost(INSN_COST);
14984 ins_encode %{
14985 __ cmp(zr, $op1$$Register);
14986 %}
14987
14988 ins_pipe(icmp_reg_imm);
14989 %}
14990
14991 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14992 %{
14993 match(Set cr (OverflowMulI op1 op2));
14994
14995 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14996 "cmp rscratch1, rscratch1, sxtw\n\t"
14997 "movw rscratch1, #0x80000000\n\t"
14998 "cselw rscratch1, rscratch1, zr, NE\n\t"
14999 "cmpw rscratch1, #1" %}
15000 ins_cost(5 * INSN_COST);
15001 ins_encode %{
15002 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15003 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15004 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15005 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15006 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15007 %}
15008
15009 ins_pipe(pipe_slow);
15010 %}
15011
15012 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15013 %{
15014 match(If cmp (OverflowMulI op1 op2));
15015 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15016 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15017 effect(USE labl, KILL cr);
15018
15019 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15020 "cmp rscratch1, rscratch1, sxtw\n\t"
15021 "b$cmp $labl" %}
15022 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15023 ins_encode %{
15024 Label* L = $labl$$label;
15025 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15026 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15027 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15028 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15029 %}
15030
15031 ins_pipe(pipe_serial);
15032 %}
15033
15034 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15035 %{
15036 match(Set cr (OverflowMulL op1 op2));
15037
15038 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15039 "smulh rscratch2, $op1, $op2\n\t"
15040 "cmp rscratch2, rscratch1, ASR #63\n\t"
15041 "movw rscratch1, #0x80000000\n\t"
15042 "cselw rscratch1, rscratch1, zr, NE\n\t"
15043 "cmpw rscratch1, #1" %}
15044 ins_cost(6 * INSN_COST);
15045 ins_encode %{
15046 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15047 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15048 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15049 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15050 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15051 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15052 %}
15053
15054 ins_pipe(pipe_slow);
15055 %}
15056
15057 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15058 %{
15059 match(If cmp (OverflowMulL op1 op2));
15060 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15061 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15062 effect(USE labl, KILL cr);
15063
15064 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15065 "smulh rscratch2, $op1, $op2\n\t"
15066 "cmp rscratch2, rscratch1, ASR #63\n\t"
15067 "b$cmp $labl" %}
15068 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15069 ins_encode %{
15070 Label* L = $labl$$label;
15071 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15072 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15073 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15074 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15075 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15076 %}
15077
15078 ins_pipe(pipe_serial);
15079 %}
15080
15081 // ============================================================================
15082 // Compare Instructions
15083
15084 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15085 %{
15086 match(Set cr (CmpI op1 op2));
15087
15088 effect(DEF cr, USE op1, USE op2);
15089
15090 ins_cost(INSN_COST);
15091 format %{ "cmpw $op1, $op2" %}
15092
15093 ins_encode(aarch64_enc_cmpw(op1, op2));
15094
15095 ins_pipe(icmp_reg_reg);
15096 %}
15097
15098 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15099 %{
15100 match(Set cr (CmpI op1 zero));
15101
15102 effect(DEF cr, USE op1);
15103
15104 ins_cost(INSN_COST);
15105 format %{ "cmpw $op1, 0" %}
15106
15107 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15108
15109 ins_pipe(icmp_reg_imm);
15110 %}
15111
15112 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15113 %{
15114 match(Set cr (CmpI op1 op2));
15115
15116 effect(DEF cr, USE op1);
15117
15118 ins_cost(INSN_COST);
15119 format %{ "cmpw $op1, $op2" %}
15120
15121 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15122
15123 ins_pipe(icmp_reg_imm);
15124 %}
15125
15126 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15127 %{
15128 match(Set cr (CmpI op1 op2));
15129
15130 effect(DEF cr, USE op1);
15131
15132 ins_cost(INSN_COST * 2);
15133 format %{ "cmpw $op1, $op2" %}
15134
15135 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15136
15137 ins_pipe(icmp_reg_imm);
15138 %}
15139
15140 // Unsigned compare Instructions; really, same as signed compare
15141 // except it should only be used to feed an If or a CMovI which takes a
15142 // cmpOpU.
15143
15144 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15145 %{
15146 match(Set cr (CmpU op1 op2));
15147
15148 effect(DEF cr, USE op1, USE op2);
15149
15150 ins_cost(INSN_COST);
15151 format %{ "cmpw $op1, $op2\t# unsigned" %}
15152
15153 ins_encode(aarch64_enc_cmpw(op1, op2));
15154
15155 ins_pipe(icmp_reg_reg);
15156 %}
15157
15158 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15159 %{
15160 match(Set cr (CmpU op1 zero));
15161
15162 effect(DEF cr, USE op1);
15163
15164 ins_cost(INSN_COST);
15165 format %{ "cmpw $op1, #0\t# unsigned" %}
15166
15167 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15168
15169 ins_pipe(icmp_reg_imm);
15170 %}
15171
15172 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15173 %{
15174 match(Set cr (CmpU op1 op2));
15175
15176 effect(DEF cr, USE op1);
15177
15178 ins_cost(INSN_COST);
15179 format %{ "cmpw $op1, $op2\t# unsigned" %}
15180
15181 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15182
15183 ins_pipe(icmp_reg_imm);
15184 %}
15185
15186 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15187 %{
15188 match(Set cr (CmpU op1 op2));
15189
15190 effect(DEF cr, USE op1);
15191
15192 ins_cost(INSN_COST * 2);
15193 format %{ "cmpw $op1, $op2\t# unsigned" %}
15194
15195 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15196
15197 ins_pipe(icmp_reg_imm);
15198 %}
15199
15200 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15201 %{
15202 match(Set cr (CmpL op1 op2));
15203
15204 effect(DEF cr, USE op1, USE op2);
15205
15206 ins_cost(INSN_COST);
15207 format %{ "cmp $op1, $op2" %}
15208
15209 ins_encode(aarch64_enc_cmp(op1, op2));
15210
15211 ins_pipe(icmp_reg_reg);
15212 %}
15213
15214 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15215 %{
15216 match(Set cr (CmpL op1 zero));
15217
15218 effect(DEF cr, USE op1);
15219
15220 ins_cost(INSN_COST);
15221 format %{ "tst $op1" %}
15222
15223 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15224
15225 ins_pipe(icmp_reg_imm);
15226 %}
15227
15228 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15229 %{
15230 match(Set cr (CmpL op1 op2));
15231
15232 effect(DEF cr, USE op1);
15233
15234 ins_cost(INSN_COST);
15235 format %{ "cmp $op1, $op2" %}
15236
15237 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15238
15239 ins_pipe(icmp_reg_imm);
15240 %}
15241
15242 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15243 %{
15244 match(Set cr (CmpL op1 op2));
15245
15246 effect(DEF cr, USE op1);
15247
15248 ins_cost(INSN_COST * 2);
15249 format %{ "cmp $op1, $op2" %}
15250
15251 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15252
15253 ins_pipe(icmp_reg_imm);
15254 %}
15255
15256 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15257 %{
15258 match(Set cr (CmpUL op1 op2));
15259
15260 effect(DEF cr, USE op1, USE op2);
15261
15262 ins_cost(INSN_COST);
15263 format %{ "cmp $op1, $op2" %}
15264
15265 ins_encode(aarch64_enc_cmp(op1, op2));
15266
15267 ins_pipe(icmp_reg_reg);
15268 %}
15269
15270 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15271 %{
15272 match(Set cr (CmpUL op1 zero));
15273
15274 effect(DEF cr, USE op1);
15275
15276 ins_cost(INSN_COST);
15277 format %{ "tst $op1" %}
15278
15279 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15280
15281 ins_pipe(icmp_reg_imm);
15282 %}
15283
15284 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15285 %{
15286 match(Set cr (CmpUL op1 op2));
15287
15288 effect(DEF cr, USE op1);
15289
15290 ins_cost(INSN_COST);
15291 format %{ "cmp $op1, $op2" %}
15292
15293 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15294
15295 ins_pipe(icmp_reg_imm);
15296 %}
15297
15298 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15299 %{
15300 match(Set cr (CmpUL op1 op2));
15301
15302 effect(DEF cr, USE op1);
15303
15304 ins_cost(INSN_COST * 2);
15305 format %{ "cmp $op1, $op2" %}
15306
15307 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15308
15309 ins_pipe(icmp_reg_imm);
15310 %}
15311
15312 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15313 %{
15314 match(Set cr (CmpP op1 op2));
15315
15316 effect(DEF cr, USE op1, USE op2);
15317
15318 ins_cost(INSN_COST);
15319 format %{ "cmp $op1, $op2\t // ptr" %}
15320
15321 ins_encode(aarch64_enc_cmpp(op1, op2));
15322
15323 ins_pipe(icmp_reg_reg);
15324 %}
15325
15326 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15327 %{
15328 match(Set cr (CmpN op1 op2));
15329
15330 effect(DEF cr, USE op1, USE op2);
15331
15332 ins_cost(INSN_COST);
15333 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15334
15335 ins_encode(aarch64_enc_cmpn(op1, op2));
15336
15337 ins_pipe(icmp_reg_reg);
15338 %}
15339
15340 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15341 %{
15342 match(Set cr (CmpP op1 zero));
15343
15344 effect(DEF cr, USE op1, USE zero);
15345
15346 ins_cost(INSN_COST);
15347 format %{ "cmp $op1, 0\t // ptr" %}
15348
15349 ins_encode(aarch64_enc_testp(op1));
15350
15351 ins_pipe(icmp_reg_imm);
15352 %}
15353
15354 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15355 %{
15356 match(Set cr (CmpN op1 zero));
15357
15358 effect(DEF cr, USE op1, USE zero);
15359
15360 ins_cost(INSN_COST);
15361 format %{ "cmp $op1, 0\t // compressed ptr" %}
15362
15363 ins_encode(aarch64_enc_testn(op1));
15364
15365 ins_pipe(icmp_reg_imm);
15366 %}
15367
15368 // FP comparisons
15369 //
15370 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15371 // using normal cmpOp. See declaration of rFlagsReg for details.
15372
15373 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15374 %{
15375 match(Set cr (CmpF src1 src2));
15376
15377 ins_cost(3 * INSN_COST);
15378 format %{ "fcmps $src1, $src2" %}
15379
15380 ins_encode %{
15381 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15382 %}
15383
15384 ins_pipe(pipe_class_compare);
15385 %}
15386
15387 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15388 %{
15389 match(Set cr (CmpF src1 src2));
15390
15391 ins_cost(3 * INSN_COST);
15392 format %{ "fcmps $src1, 0.0" %}
15393
15394 ins_encode %{
15395 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15396 %}
15397
15398 ins_pipe(pipe_class_compare);
15399 %}
15400 // FROM HERE
15401
15402 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15403 %{
15404 match(Set cr (CmpD src1 src2));
15405
15406 ins_cost(3 * INSN_COST);
15407 format %{ "fcmpd $src1, $src2" %}
15408
15409 ins_encode %{
15410 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15411 %}
15412
15413 ins_pipe(pipe_class_compare);
15414 %}
15415
15416 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15417 %{
15418 match(Set cr (CmpD src1 src2));
15419
15420 ins_cost(3 * INSN_COST);
15421 format %{ "fcmpd $src1, 0.0" %}
15422
15423 ins_encode %{
15424 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15425 %}
15426
15427 ins_pipe(pipe_class_compare);
15428 %}
15429
15430 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15431 %{
15432 match(Set dst (CmpF3 src1 src2));
15433 effect(KILL cr);
15434
15435 ins_cost(5 * INSN_COST);
15436 format %{ "fcmps $src1, $src2\n\t"
15437 "csinvw($dst, zr, zr, eq\n\t"
15438 "csnegw($dst, $dst, $dst, lt)"
15439 %}
15440
15441 ins_encode %{
15442 Label done;
15443 FloatRegister s1 = as_FloatRegister($src1$$reg);
15444 FloatRegister s2 = as_FloatRegister($src2$$reg);
15445 Register d = as_Register($dst$$reg);
15446 __ fcmps(s1, s2);
15447 // installs 0 if EQ else -1
15448 __ csinvw(d, zr, zr, Assembler::EQ);
15449 // keeps -1 if less or unordered else installs 1
15450 __ csnegw(d, d, d, Assembler::LT);
15451 __ bind(done);
15452 %}
15453
15454 ins_pipe(pipe_class_default);
15455
15456 %}
15457
15458 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15459 %{
15460 match(Set dst (CmpD3 src1 src2));
15461 effect(KILL cr);
15462
15463 ins_cost(5 * INSN_COST);
15464 format %{ "fcmpd $src1, $src2\n\t"
15465 "csinvw($dst, zr, zr, eq\n\t"
15466 "csnegw($dst, $dst, $dst, lt)"
15467 %}
15468
15469 ins_encode %{
15470 Label done;
15471 FloatRegister s1 = as_FloatRegister($src1$$reg);
15472 FloatRegister s2 = as_FloatRegister($src2$$reg);
15473 Register d = as_Register($dst$$reg);
15474 __ fcmpd(s1, s2);
15475 // installs 0 if EQ else -1
15476 __ csinvw(d, zr, zr, Assembler::EQ);
15477 // keeps -1 if less or unordered else installs 1
15478 __ csnegw(d, d, d, Assembler::LT);
15479 __ bind(done);
15480 %}
15481 ins_pipe(pipe_class_default);
15482
15483 %}
15484
15485 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15486 %{
15487 match(Set dst (CmpF3 src1 zero));
15488 effect(KILL cr);
15489
15490 ins_cost(5 * INSN_COST);
15491 format %{ "fcmps $src1, 0.0\n\t"
15492 "csinvw($dst, zr, zr, eq\n\t"
15493 "csnegw($dst, $dst, $dst, lt)"
15494 %}
15495
15496 ins_encode %{
15497 Label done;
15498 FloatRegister s1 = as_FloatRegister($src1$$reg);
15499 Register d = as_Register($dst$$reg);
15500 __ fcmps(s1, 0.0);
15501 // installs 0 if EQ else -1
15502 __ csinvw(d, zr, zr, Assembler::EQ);
15503 // keeps -1 if less or unordered else installs 1
15504 __ csnegw(d, d, d, Assembler::LT);
15505 __ bind(done);
15506 %}
15507
15508 ins_pipe(pipe_class_default);
15509
15510 %}
15511
15512 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15513 %{
15514 match(Set dst (CmpD3 src1 zero));
15515 effect(KILL cr);
15516
15517 ins_cost(5 * INSN_COST);
15518 format %{ "fcmpd $src1, 0.0\n\t"
15519 "csinvw($dst, zr, zr, eq\n\t"
15520 "csnegw($dst, $dst, $dst, lt)"
15521 %}
15522
15523 ins_encode %{
15524 Label done;
15525 FloatRegister s1 = as_FloatRegister($src1$$reg);
15526 Register d = as_Register($dst$$reg);
15527 __ fcmpd(s1, 0.0);
15528 // installs 0 if EQ else -1
15529 __ csinvw(d, zr, zr, Assembler::EQ);
15530 // keeps -1 if less or unordered else installs 1
15531 __ csnegw(d, d, d, Assembler::LT);
15532 __ bind(done);
15533 %}
15534 ins_pipe(pipe_class_default);
15535
15536 %}
15537
15538 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15539 %{
15540 match(Set dst (CmpLTMask p q));
15541 effect(KILL cr);
15542
15543 ins_cost(3 * INSN_COST);
15544
15545 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15546 "csetw $dst, lt\n\t"
15547 "subw $dst, zr, $dst"
15548 %}
15549
15550 ins_encode %{
15551 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15552 __ csetw(as_Register($dst$$reg), Assembler::LT);
15553 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15554 %}
15555
15556 ins_pipe(ialu_reg_reg);
15557 %}
15558
15559 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15560 %{
15561 match(Set dst (CmpLTMask src zero));
15562 effect(KILL cr);
15563
15564 ins_cost(INSN_COST);
15565
15566 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15567
15568 ins_encode %{
15569 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15570 %}
15571
15572 ins_pipe(ialu_reg_shift);
15573 %}
15574
15575 // ============================================================================
15576 // Max and Min
15577
15578 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15579
15580 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15581 %{
15582 effect(DEF cr, USE src);
15583 ins_cost(INSN_COST);
15584 format %{ "cmpw $src, 0" %}
15585
15586 ins_encode %{
15587 __ cmpw($src$$Register, 0);
15588 %}
15589 ins_pipe(icmp_reg_imm);
15590 %}
15591
15592 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15593 %{
15594 match(Set dst (MinI src1 src2));
15595 ins_cost(INSN_COST * 3);
15596
15597 expand %{
15598 rFlagsReg cr;
15599 compI_reg_reg(cr, src1, src2);
15600 cmovI_reg_reg_lt(dst, src1, src2, cr);
15601 %}
15602 %}
15603
15604 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15605 %{
15606 match(Set dst (MaxI src1 src2));
15607 ins_cost(INSN_COST * 3);
15608
15609 expand %{
15610 rFlagsReg cr;
15611 compI_reg_reg(cr, src1, src2);
15612 cmovI_reg_reg_gt(dst, src1, src2, cr);
15613 %}
15614 %}
15615
15616
15617 // ============================================================================
15618 // Branch Instructions
15619
15620 // Direct Branch.
15621 instruct branch(label lbl)
15622 %{
15623 match(Goto);
15624
15625 effect(USE lbl);
15626
15627 ins_cost(BRANCH_COST);
15628 format %{ "b $lbl" %}
15629
15630 ins_encode(aarch64_enc_b(lbl));
15631
15632 ins_pipe(pipe_branch);
15633 %}
15634
15635 // Conditional Near Branch
15636 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15637 %{
15638 // Same match rule as `branchConFar'.
15639 match(If cmp cr);
15640
15641 effect(USE lbl);
15642
15643 ins_cost(BRANCH_COST);
15644 // If set to 1 this indicates that the current instruction is a
15645 // short variant of a long branch. This avoids using this
15646 // instruction in first-pass matching. It will then only be used in
15647 // the `Shorten_branches' pass.
15648 // ins_short_branch(1);
15649 format %{ "b$cmp $lbl" %}
15650
15651 ins_encode(aarch64_enc_br_con(cmp, lbl));
15652
15653 ins_pipe(pipe_branch_cond);
15654 %}
15655
15656 // Conditional Near Branch Unsigned
15657 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15658 %{
15659 // Same match rule as `branchConFar'.
15660 match(If cmp cr);
15661
15662 effect(USE lbl);
15663
15664 ins_cost(BRANCH_COST);
15665 // If set to 1 this indicates that the current instruction is a
15666 // short variant of a long branch. This avoids using this
15667 // instruction in first-pass matching. It will then only be used in
15668 // the `Shorten_branches' pass.
15669 // ins_short_branch(1);
15670 format %{ "b$cmp $lbl\t# unsigned" %}
15671
15672 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15673
15674 ins_pipe(pipe_branch_cond);
15675 %}
15676
15677 // Make use of CBZ and CBNZ. These instructions, as well as being
15678 // shorter than (cmp; branch), have the additional benefit of not
15679 // killing the flags.
15680
15681 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15682 match(If cmp (CmpI op1 op2));
15683 effect(USE labl);
15684
15685 ins_cost(BRANCH_COST);
15686 format %{ "cbw$cmp $op1, $labl" %}
15687 ins_encode %{
15688 Label* L = $labl$$label;
15689 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15690 if (cond == Assembler::EQ)
15691 __ cbzw($op1$$Register, *L);
15692 else
15693 __ cbnzw($op1$$Register, *L);
15694 %}
15695 ins_pipe(pipe_cmp_branch);
15696 %}
15697
15698 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15699 match(If cmp (CmpL op1 op2));
15700 effect(USE labl);
15701
15702 ins_cost(BRANCH_COST);
15703 format %{ "cb$cmp $op1, $labl" %}
15704 ins_encode %{
15705 Label* L = $labl$$label;
15706 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15707 if (cond == Assembler::EQ)
15708 __ cbz($op1$$Register, *L);
15709 else
15710 __ cbnz($op1$$Register, *L);
15711 %}
15712 ins_pipe(pipe_cmp_branch);
15713 %}
15714
15715 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15716 match(If cmp (CmpP op1 op2));
15717 effect(USE labl);
15718
15719 ins_cost(BRANCH_COST);
15720 format %{ "cb$cmp $op1, $labl" %}
15721 ins_encode %{
15722 Label* L = $labl$$label;
15723 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15724 if (cond == Assembler::EQ)
15725 __ cbz($op1$$Register, *L);
15726 else
15727 __ cbnz($op1$$Register, *L);
15728 %}
15729 ins_pipe(pipe_cmp_branch);
15730 %}
15731
15732 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15733 match(If cmp (CmpN op1 op2));
15734 effect(USE labl);
15735
15736 ins_cost(BRANCH_COST);
15737 format %{ "cbw$cmp $op1, $labl" %}
15738 ins_encode %{
15739 Label* L = $labl$$label;
15740 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15741 if (cond == Assembler::EQ)
15742 __ cbzw($op1$$Register, *L);
15743 else
15744 __ cbnzw($op1$$Register, *L);
15745 %}
15746 ins_pipe(pipe_cmp_branch);
15747 %}
15748
15749 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15750 match(If cmp (CmpP (DecodeN oop) zero));
15751 effect(USE labl);
15752
15753 ins_cost(BRANCH_COST);
15754 format %{ "cb$cmp $oop, $labl" %}
15755 ins_encode %{
15756 Label* L = $labl$$label;
15757 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15758 if (cond == Assembler::EQ)
15759 __ cbzw($oop$$Register, *L);
15760 else
15761 __ cbnzw($oop$$Register, *L);
15762 %}
15763 ins_pipe(pipe_cmp_branch);
15764 %}
15765
15766 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15767 match(If cmp (CmpU op1 op2));
15768 effect(USE labl);
15769
15770 ins_cost(BRANCH_COST);
15771 format %{ "cbw$cmp $op1, $labl" %}
15772 ins_encode %{
15773 Label* L = $labl$$label;
15774 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15775 if (cond == Assembler::EQ || cond == Assembler::LS) {
15776 __ cbzw($op1$$Register, *L);
15777 } else {
15778 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15779 __ cbnzw($op1$$Register, *L);
15780 }
15781 %}
15782 ins_pipe(pipe_cmp_branch);
15783 %}
15784
15785 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15786 match(If cmp (CmpUL op1 op2));
15787 effect(USE labl);
15788
15789 ins_cost(BRANCH_COST);
15790 format %{ "cb$cmp $op1, $labl" %}
15791 ins_encode %{
15792 Label* L = $labl$$label;
15793 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15794 if (cond == Assembler::EQ || cond == Assembler::LS) {
15795 __ cbz($op1$$Register, *L);
15796 } else {
15797 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15798 __ cbnz($op1$$Register, *L);
15799 }
15800 %}
15801 ins_pipe(pipe_cmp_branch);
15802 %}
15803
15804 // Test bit and Branch
15805
15806 // Patterns for short (< 32KiB) variants
15807 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15808 match(If cmp (CmpL op1 op2));
15809 effect(USE labl);
15810
15811 ins_cost(BRANCH_COST);
15812 format %{ "cb$cmp $op1, $labl # long" %}
15813 ins_encode %{
15814 Label* L = $labl$$label;
15815 Assembler::Condition cond =
15816 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15817 __ tbr(cond, $op1$$Register, 63, *L);
15818 %}
15819 ins_pipe(pipe_cmp_branch);
15820 ins_short_branch(1);
15821 %}
15822
15823 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15824 match(If cmp (CmpI op1 op2));
15825 effect(USE labl);
15826
15827 ins_cost(BRANCH_COST);
15828 format %{ "cb$cmp $op1, $labl # int" %}
15829 ins_encode %{
15830 Label* L = $labl$$label;
15831 Assembler::Condition cond =
15832 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15833 __ tbr(cond, $op1$$Register, 31, *L);
15834 %}
15835 ins_pipe(pipe_cmp_branch);
15836 ins_short_branch(1);
15837 %}
15838
15839 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15840 match(If cmp (CmpL (AndL op1 op2) op3));
15841 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15842 effect(USE labl);
15843
15844 ins_cost(BRANCH_COST);
15845 format %{ "tb$cmp $op1, $op2, $labl" %}
15846 ins_encode %{
15847 Label* L = $labl$$label;
15848 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15849 int bit = exact_log2_long($op2$$constant);
15850 __ tbr(cond, $op1$$Register, bit, *L);
15851 %}
15852 ins_pipe(pipe_cmp_branch);
15853 ins_short_branch(1);
15854 %}
15855
15856 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15857 match(If cmp (CmpI (AndI op1 op2) op3));
15858 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15859 effect(USE labl);
15860
15861 ins_cost(BRANCH_COST);
15862 format %{ "tb$cmp $op1, $op2, $labl" %}
15863 ins_encode %{
15864 Label* L = $labl$$label;
15865 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15866 int bit = exact_log2((juint)$op2$$constant);
15867 __ tbr(cond, $op1$$Register, bit, *L);
15868 %}
15869 ins_pipe(pipe_cmp_branch);
15870 ins_short_branch(1);
15871 %}
15872
15873 // And far variants
15874 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15875 match(If cmp (CmpL op1 op2));
15876 effect(USE labl);
15877
15878 ins_cost(BRANCH_COST);
15879 format %{ "cb$cmp $op1, $labl # long" %}
15880 ins_encode %{
15881 Label* L = $labl$$label;
15882 Assembler::Condition cond =
15883 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15884 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15885 %}
15886 ins_pipe(pipe_cmp_branch);
15887 %}
15888
15889 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15890 match(If cmp (CmpI op1 op2));
15891 effect(USE labl);
15892
15893 ins_cost(BRANCH_COST);
15894 format %{ "cb$cmp $op1, $labl # int" %}
15895 ins_encode %{
15896 Label* L = $labl$$label;
15897 Assembler::Condition cond =
15898 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15899 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15900 %}
15901 ins_pipe(pipe_cmp_branch);
15902 %}
15903
15904 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15905 match(If cmp (CmpL (AndL op1 op2) op3));
15906 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15907 effect(USE labl);
15908
15909 ins_cost(BRANCH_COST);
15910 format %{ "tb$cmp $op1, $op2, $labl" %}
15911 ins_encode %{
15912 Label* L = $labl$$label;
15913 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15914 int bit = exact_log2_long($op2$$constant);
15915 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15916 %}
15917 ins_pipe(pipe_cmp_branch);
15918 %}
15919
15920 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15921 match(If cmp (CmpI (AndI op1 op2) op3));
15922 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15923 effect(USE labl);
15924
15925 ins_cost(BRANCH_COST);
15926 format %{ "tb$cmp $op1, $op2, $labl" %}
15927 ins_encode %{
15928 Label* L = $labl$$label;
15929 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15930 int bit = exact_log2((juint)$op2$$constant);
15931 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15932 %}
15933 ins_pipe(pipe_cmp_branch);
15934 %}
15935
15936 // Test bits
15937
15938 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15939 match(Set cr (CmpL (AndL op1 op2) op3));
15940 predicate(Assembler::operand_valid_for_logical_immediate
15941 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15942
15943 ins_cost(INSN_COST);
15944 format %{ "tst $op1, $op2 # long" %}
15945 ins_encode %{
15946 __ tst($op1$$Register, $op2$$constant);
15947 %}
15948 ins_pipe(ialu_reg_reg);
15949 %}
15950
15951 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15952 match(Set cr (CmpI (AndI op1 op2) op3));
15953 predicate(Assembler::operand_valid_for_logical_immediate
15954 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15955
15956 ins_cost(INSN_COST);
15957 format %{ "tst $op1, $op2 # int" %}
15958 ins_encode %{
15959 __ tstw($op1$$Register, $op2$$constant);
15960 %}
15961 ins_pipe(ialu_reg_reg);
15962 %}
15963
15964 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15965 match(Set cr (CmpL (AndL op1 op2) op3));
15966
15967 ins_cost(INSN_COST);
15968 format %{ "tst $op1, $op2 # long" %}
15969 ins_encode %{
15970 __ tst($op1$$Register, $op2$$Register);
15971 %}
15972 ins_pipe(ialu_reg_reg);
15973 %}
15974
15975 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15976 match(Set cr (CmpI (AndI op1 op2) op3));
15977
15978 ins_cost(INSN_COST);
15979 format %{ "tstw $op1, $op2 # int" %}
15980 ins_encode %{
15981 __ tstw($op1$$Register, $op2$$Register);
15982 %}
15983 ins_pipe(ialu_reg_reg);
15984 %}
15985
15986
15987 // Conditional Far Branch
15988 // Conditional Far Branch Unsigned
15989 // TODO: fixme
15990
15991 // counted loop end branch near
15992 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15993 %{
15994 match(CountedLoopEnd cmp cr);
15995
15996 effect(USE lbl);
15997
15998 ins_cost(BRANCH_COST);
15999 // short variant.
16000 // ins_short_branch(1);
16001 format %{ "b$cmp $lbl \t// counted loop end" %}
16002
16003 ins_encode(aarch64_enc_br_con(cmp, lbl));
16004
16005 ins_pipe(pipe_branch);
16006 %}
16007
16008 // counted loop end branch far
16009 // TODO: fixme
16010
16011 // ============================================================================
16012 // inlined locking and unlocking
16013
16014 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16015 %{
16016 predicate(LockingMode != LM_LIGHTWEIGHT);
16017 match(Set cr (FastLock object box));
16018 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16019
16020 ins_cost(5 * INSN_COST);
16021 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16022
16023 ins_encode %{
16024 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16025 %}
16026
16027 ins_pipe(pipe_serial);
16028 %}
16029
16030 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16031 %{
16032 predicate(LockingMode != LM_LIGHTWEIGHT);
16033 match(Set cr (FastUnlock object box));
16034 effect(TEMP tmp, TEMP tmp2);
16035
16036 ins_cost(5 * INSN_COST);
16037 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16038
16039 ins_encode %{
16040 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16041 %}
16042
16043 ins_pipe(pipe_serial);
16044 %}
16045
16046 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16047 %{
16048 predicate(LockingMode == LM_LIGHTWEIGHT);
16049 match(Set cr (FastLock object box));
16050 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16051
16052 ins_cost(5 * INSN_COST);
16053 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16054
16055 ins_encode %{
16056 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16057 %}
16058
16059 ins_pipe(pipe_serial);
16060 %}
16061
16062 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16063 %{
16064 predicate(LockingMode == LM_LIGHTWEIGHT);
16065 match(Set cr (FastUnlock object box));
16066 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16067
16068 ins_cost(5 * INSN_COST);
16069 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16070
16071 ins_encode %{
16072 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16073 %}
16074
16075 ins_pipe(pipe_serial);
16076 %}
16077
16078 // ============================================================================
16079 // Safepoint Instructions
16080
16081 // TODO
16082 // provide a near and far version of this code
16083
16084 instruct safePoint(rFlagsReg cr, iRegP poll)
16085 %{
16086 match(SafePoint poll);
16087 effect(KILL cr);
16088
16089 format %{
16090 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16091 %}
16092 ins_encode %{
16093 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16094 %}
16095 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16096 %}
16097
16098
16099 // ============================================================================
16100 // Procedure Call/Return Instructions
16101
16102 // Call Java Static Instruction
16103
16104 instruct CallStaticJavaDirect(method meth)
16105 %{
16106 match(CallStaticJava);
16107
16108 effect(USE meth);
16109
16110 ins_cost(CALL_COST);
16111
16112 format %{ "call,static $meth \t// ==> " %}
16113
16114 ins_encode(aarch64_enc_java_static_call(meth),
16115 aarch64_enc_call_epilog);
16116
16117 ins_pipe(pipe_class_call);
16118 %}
16119
16120 // TO HERE
16121
16122 // Call Java Dynamic Instruction
16123 instruct CallDynamicJavaDirect(method meth)
16124 %{
16125 match(CallDynamicJava);
16126
16127 effect(USE meth);
16128
16129 ins_cost(CALL_COST);
16130
16131 format %{ "CALL,dynamic $meth \t// ==> " %}
16132
16133 ins_encode(aarch64_enc_java_dynamic_call(meth),
16134 aarch64_enc_call_epilog);
16135
16136 ins_pipe(pipe_class_call);
16137 %}
16138
16139 // Call Runtime Instruction
16140
16141 instruct CallRuntimeDirect(method meth)
16142 %{
16143 match(CallRuntime);
16144
16145 effect(USE meth);
16146
16147 ins_cost(CALL_COST);
16148
16149 format %{ "CALL, runtime $meth" %}
16150
16151 ins_encode( aarch64_enc_java_to_runtime(meth) );
16152
16153 ins_pipe(pipe_class_call);
16154 %}
16155
16156 // Call Runtime Instruction
16157
16158 instruct CallLeafDirect(method meth)
16159 %{
16160 match(CallLeaf);
16161
16162 effect(USE meth);
16163
16164 ins_cost(CALL_COST);
16165
16166 format %{ "CALL, runtime leaf $meth" %}
16167
16168 ins_encode( aarch64_enc_java_to_runtime(meth) );
16169
16170 ins_pipe(pipe_class_call);
16171 %}
16172
16173 // Call Runtime Instruction without safepoint and with vector arguments
16174 instruct CallLeafDirectVector(method meth)
16175 %{
16176 match(CallLeafVector);
16177
16178 effect(USE meth);
16179
16180 ins_cost(CALL_COST);
16181
16182 format %{ "CALL, runtime leaf vector $meth" %}
16183
16184 ins_encode(aarch64_enc_java_to_runtime(meth));
16185
16186 ins_pipe(pipe_class_call);
16187 %}
16188
16189 // Call Runtime Instruction
16190
16191 instruct CallLeafNoFPDirect(method meth)
16192 %{
16193 match(CallLeafNoFP);
16194
16195 effect(USE meth);
16196
16197 ins_cost(CALL_COST);
16198
16199 format %{ "CALL, runtime leaf nofp $meth" %}
16200
16201 ins_encode( aarch64_enc_java_to_runtime(meth) );
16202
16203 ins_pipe(pipe_class_call);
16204 %}
16205
16206 // Tail Call; Jump from runtime stub to Java code.
16207 // Also known as an 'interprocedural jump'.
16208 // Target of jump will eventually return to caller.
16209 // TailJump below removes the return address.
16210 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16211 // emitted just above the TailCall which has reset rfp to the caller state.
16212 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16213 %{
16214 match(TailCall jump_target method_ptr);
16215
16216 ins_cost(CALL_COST);
16217
16218 format %{ "br $jump_target\t# $method_ptr holds method" %}
16219
16220 ins_encode(aarch64_enc_tail_call(jump_target));
16221
16222 ins_pipe(pipe_class_call);
16223 %}
16224
16225 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16226 %{
16227 match(TailJump jump_target ex_oop);
16228
16229 ins_cost(CALL_COST);
16230
16231 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16232
16233 ins_encode(aarch64_enc_tail_jmp(jump_target));
16234
16235 ins_pipe(pipe_class_call);
16236 %}
16237
16238 // Forward exception.
16239 instruct ForwardExceptionjmp()
16240 %{
16241 match(ForwardException);
16242 ins_cost(CALL_COST);
16243
16244 format %{ "b forward_exception_stub" %}
16245 ins_encode %{
16246 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16247 %}
16248 ins_pipe(pipe_class_call);
16249 %}
16250
16251 // Create exception oop: created by stack-crawling runtime code.
16252 // Created exception is now available to this handler, and is setup
16253 // just prior to jumping to this handler. No code emitted.
16254 // TODO check
16255 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16256 instruct CreateException(iRegP_R0 ex_oop)
16257 %{
16258 match(Set ex_oop (CreateEx));
16259
16260 format %{ " -- \t// exception oop; no code emitted" %}
16261
16262 size(0);
16263
16264 ins_encode( /*empty*/ );
16265
16266 ins_pipe(pipe_class_empty);
16267 %}
16268
16269 // Rethrow exception: The exception oop will come in the first
16270 // argument position. Then JUMP (not call) to the rethrow stub code.
16271 instruct RethrowException() %{
16272 match(Rethrow);
16273 ins_cost(CALL_COST);
16274
16275 format %{ "b rethrow_stub" %}
16276
16277 ins_encode( aarch64_enc_rethrow() );
16278
16279 ins_pipe(pipe_class_call);
16280 %}
16281
16282
16283 // Return Instruction
16284 // epilog node loads ret address into lr as part of frame pop
16285 instruct Ret()
16286 %{
16287 match(Return);
16288
16289 format %{ "ret\t// return register" %}
16290
16291 ins_encode( aarch64_enc_ret() );
16292
16293 ins_pipe(pipe_branch);
16294 %}
16295
16296 // Die now.
16297 instruct ShouldNotReachHere() %{
16298 match(Halt);
16299
16300 ins_cost(CALL_COST);
16301 format %{ "ShouldNotReachHere" %}
16302
16303 ins_encode %{
16304 if (is_reachable()) {
16305 __ stop(_halt_reason);
16306 }
16307 %}
16308
16309 ins_pipe(pipe_class_default);
16310 %}
16311
16312 // ============================================================================
16313 // Partial Subtype Check
16314 //
16315 // superklass array for an instance of the superklass. Set a hidden
16316 // internal cache on a hit (cache is checked with exposed code in
16317 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16318 // encoding ALSO sets flags.
16319
16320 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16321 %{
16322 match(Set result (PartialSubtypeCheck sub super));
16323 predicate(!UseSecondarySupersTable);
16324 effect(KILL cr, KILL temp);
16325
16326 ins_cost(20 * INSN_COST); // slightly larger than the next version
16327 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16328
16329 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16330
16331 opcode(0x1); // Force zero of result reg on hit
16332
16333 ins_pipe(pipe_class_memory);
16334 %}
16335
16336 // Two versions of partialSubtypeCheck, both used when we need to
16337 // search for a super class in the secondary supers array. The first
16338 // is used when we don't know _a priori_ the class being searched
16339 // for. The second, far more common, is used when we do know: this is
16340 // used for instanceof, checkcast, and any case where C2 can determine
16341 // it by constant propagation.
16342
16343 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16344 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16345 rFlagsReg cr)
16346 %{
16347 match(Set result (PartialSubtypeCheck sub super));
16348 predicate(UseSecondarySupersTable);
16349 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16350
16351 ins_cost(10 * INSN_COST); // slightly larger than the next version
16352 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16353
16354 ins_encode %{
16355 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16356 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16357 $vtemp$$FloatRegister,
16358 $result$$Register, /*L_success*/nullptr);
16359 %}
16360
16361 ins_pipe(pipe_class_memory);
16362 %}
16363
16364 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16365 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16366 rFlagsReg cr)
16367 %{
16368 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16369 predicate(UseSecondarySupersTable);
16370 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16371
16372 ins_cost(5 * INSN_COST); // smaller than the next version
16373 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16374
16375 ins_encode %{
16376 bool success = false;
16377 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16378 if (InlineSecondarySupersTest) {
16379 success =
16380 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16381 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16382 $vtemp$$FloatRegister,
16383 $result$$Register,
16384 super_klass_slot);
16385 } else {
16386 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16387 success = (call != nullptr);
16388 }
16389 if (!success) {
16390 ciEnv::current()->record_failure("CodeCache is full");
16391 return;
16392 }
16393 %}
16394
16395 ins_pipe(pipe_class_memory);
16396 %}
16397
16398 // Intrisics for String.compareTo()
16399
16400 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16401 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16402 %{
16403 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16404 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16405 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16406
16407 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16408 ins_encode %{
16409 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16410 __ string_compare($str1$$Register, $str2$$Register,
16411 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16412 $tmp1$$Register, $tmp2$$Register,
16413 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16414 %}
16415 ins_pipe(pipe_class_memory);
16416 %}
16417
16418 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16419 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16420 %{
16421 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16422 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16423 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16424
16425 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16426 ins_encode %{
16427 __ string_compare($str1$$Register, $str2$$Register,
16428 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16429 $tmp1$$Register, $tmp2$$Register,
16430 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16431 %}
16432 ins_pipe(pipe_class_memory);
16433 %}
16434
16435 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16436 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16437 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16438 %{
16439 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16440 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16441 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16442 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16443
16444 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16445 ins_encode %{
16446 __ string_compare($str1$$Register, $str2$$Register,
16447 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16448 $tmp1$$Register, $tmp2$$Register,
16449 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16450 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16451 %}
16452 ins_pipe(pipe_class_memory);
16453 %}
16454
16455 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16456 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16457 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16458 %{
16459 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16460 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16461 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16462 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16463
16464 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16465 ins_encode %{
16466 __ string_compare($str1$$Register, $str2$$Register,
16467 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16468 $tmp1$$Register, $tmp2$$Register,
16469 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16470 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16471 %}
16472 ins_pipe(pipe_class_memory);
16473 %}
16474
16475 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16476 // these string_compare variants as NEON register type for convenience so that the prototype of
16477 // string_compare can be shared with all variants.
16478
16479 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16480 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16481 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16482 pRegGov_P1 pgtmp2, rFlagsReg cr)
16483 %{
16484 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16485 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16486 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16487 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16488
16489 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16490 ins_encode %{
16491 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16492 __ string_compare($str1$$Register, $str2$$Register,
16493 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16494 $tmp1$$Register, $tmp2$$Register,
16495 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16496 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16497 StrIntrinsicNode::LL);
16498 %}
16499 ins_pipe(pipe_class_memory);
16500 %}
16501
16502 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16503 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16504 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16505 pRegGov_P1 pgtmp2, rFlagsReg cr)
16506 %{
16507 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16508 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16509 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16510 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16511
16512 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16513 ins_encode %{
16514 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16515 __ string_compare($str1$$Register, $str2$$Register,
16516 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16517 $tmp1$$Register, $tmp2$$Register,
16518 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16519 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16520 StrIntrinsicNode::LU);
16521 %}
16522 ins_pipe(pipe_class_memory);
16523 %}
16524
16525 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16526 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16527 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16528 pRegGov_P1 pgtmp2, rFlagsReg cr)
16529 %{
16530 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16531 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16532 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16533 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16534
16535 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16536 ins_encode %{
16537 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16538 __ string_compare($str1$$Register, $str2$$Register,
16539 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16540 $tmp1$$Register, $tmp2$$Register,
16541 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16542 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16543 StrIntrinsicNode::UL);
16544 %}
16545 ins_pipe(pipe_class_memory);
16546 %}
16547
16548 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16549 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16550 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16551 pRegGov_P1 pgtmp2, rFlagsReg cr)
16552 %{
16553 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16554 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16555 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16556 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16557
16558 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16559 ins_encode %{
16560 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16561 __ string_compare($str1$$Register, $str2$$Register,
16562 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16563 $tmp1$$Register, $tmp2$$Register,
16564 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16565 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16566 StrIntrinsicNode::UU);
16567 %}
16568 ins_pipe(pipe_class_memory);
16569 %}
16570
16571 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16572 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16573 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16574 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16575 %{
16576 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16577 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16578 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16579 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16580 TEMP vtmp0, TEMP vtmp1, KILL cr);
16581 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16582 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16583
16584 ins_encode %{
16585 __ string_indexof($str1$$Register, $str2$$Register,
16586 $cnt1$$Register, $cnt2$$Register,
16587 $tmp1$$Register, $tmp2$$Register,
16588 $tmp3$$Register, $tmp4$$Register,
16589 $tmp5$$Register, $tmp6$$Register,
16590 -1, $result$$Register, StrIntrinsicNode::UU);
16591 %}
16592 ins_pipe(pipe_class_memory);
16593 %}
16594
16595 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16596 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16597 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16598 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16599 %{
16600 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16601 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16602 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16603 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16604 TEMP vtmp0, TEMP vtmp1, KILL cr);
16605 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16606 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16607
16608 ins_encode %{
16609 __ string_indexof($str1$$Register, $str2$$Register,
16610 $cnt1$$Register, $cnt2$$Register,
16611 $tmp1$$Register, $tmp2$$Register,
16612 $tmp3$$Register, $tmp4$$Register,
16613 $tmp5$$Register, $tmp6$$Register,
16614 -1, $result$$Register, StrIntrinsicNode::LL);
16615 %}
16616 ins_pipe(pipe_class_memory);
16617 %}
16618
16619 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16620 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16621 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16622 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16623 %{
16624 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16625 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16626 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16627 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16628 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16629 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16630 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16631
16632 ins_encode %{
16633 __ string_indexof($str1$$Register, $str2$$Register,
16634 $cnt1$$Register, $cnt2$$Register,
16635 $tmp1$$Register, $tmp2$$Register,
16636 $tmp3$$Register, $tmp4$$Register,
16637 $tmp5$$Register, $tmp6$$Register,
16638 -1, $result$$Register, StrIntrinsicNode::UL);
16639 %}
16640 ins_pipe(pipe_class_memory);
16641 %}
16642
16643 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16644 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16645 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16646 %{
16647 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16648 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16649 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16650 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16651 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16652 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16653
16654 ins_encode %{
16655 int icnt2 = (int)$int_cnt2$$constant;
16656 __ string_indexof($str1$$Register, $str2$$Register,
16657 $cnt1$$Register, zr,
16658 $tmp1$$Register, $tmp2$$Register,
16659 $tmp3$$Register, $tmp4$$Register, zr, zr,
16660 icnt2, $result$$Register, StrIntrinsicNode::UU);
16661 %}
16662 ins_pipe(pipe_class_memory);
16663 %}
16664
16665 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16666 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16667 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16668 %{
16669 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16670 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16671 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16672 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16673 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16674 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16675
16676 ins_encode %{
16677 int icnt2 = (int)$int_cnt2$$constant;
16678 __ string_indexof($str1$$Register, $str2$$Register,
16679 $cnt1$$Register, zr,
16680 $tmp1$$Register, $tmp2$$Register,
16681 $tmp3$$Register, $tmp4$$Register, zr, zr,
16682 icnt2, $result$$Register, StrIntrinsicNode::LL);
16683 %}
16684 ins_pipe(pipe_class_memory);
16685 %}
16686
16687 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16688 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16689 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16690 %{
16691 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16692 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16693 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16694 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16695 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16696 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16697
16698 ins_encode %{
16699 int icnt2 = (int)$int_cnt2$$constant;
16700 __ string_indexof($str1$$Register, $str2$$Register,
16701 $cnt1$$Register, zr,
16702 $tmp1$$Register, $tmp2$$Register,
16703 $tmp3$$Register, $tmp4$$Register, zr, zr,
16704 icnt2, $result$$Register, StrIntrinsicNode::UL);
16705 %}
16706 ins_pipe(pipe_class_memory);
16707 %}
16708
16709 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16710 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16711 iRegINoSp tmp3, rFlagsReg cr)
16712 %{
16713 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16714 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16715 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16716 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16717
16718 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16719
16720 ins_encode %{
16721 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16722 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16723 $tmp3$$Register);
16724 %}
16725 ins_pipe(pipe_class_memory);
16726 %}
16727
16728 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16729 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16730 iRegINoSp tmp3, rFlagsReg cr)
16731 %{
16732 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16733 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16734 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16735 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16736
16737 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16738
16739 ins_encode %{
16740 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16741 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16742 $tmp3$$Register);
16743 %}
16744 ins_pipe(pipe_class_memory);
16745 %}
16746
16747 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16748 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16749 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16750 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16751 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16752 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16753 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16754 ins_encode %{
16755 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16756 $result$$Register, $ztmp1$$FloatRegister,
16757 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16758 $ptmp$$PRegister, true /* isL */);
16759 %}
16760 ins_pipe(pipe_class_memory);
16761 %}
16762
16763 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16764 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16765 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16766 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16767 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16768 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16769 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16770 ins_encode %{
16771 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16772 $result$$Register, $ztmp1$$FloatRegister,
16773 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16774 $ptmp$$PRegister, false /* isL */);
16775 %}
16776 ins_pipe(pipe_class_memory);
16777 %}
16778
16779 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16780 iRegI_R0 result, rFlagsReg cr)
16781 %{
16782 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16783 match(Set result (StrEquals (Binary str1 str2) cnt));
16784 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16785
16786 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16787 ins_encode %{
16788 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16789 __ string_equals($str1$$Register, $str2$$Register,
16790 $result$$Register, $cnt$$Register);
16791 %}
16792 ins_pipe(pipe_class_memory);
16793 %}
16794
16795 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16796 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16797 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16798 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16799 iRegP_R10 tmp, rFlagsReg cr)
16800 %{
16801 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16802 match(Set result (AryEq ary1 ary2));
16803 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16804 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16805 TEMP vtmp6, TEMP vtmp7, KILL cr);
16806
16807 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16808 ins_encode %{
16809 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16810 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16811 $result$$Register, $tmp$$Register, 1);
16812 if (tpc == nullptr) {
16813 ciEnv::current()->record_failure("CodeCache is full");
16814 return;
16815 }
16816 %}
16817 ins_pipe(pipe_class_memory);
16818 %}
16819
16820 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16821 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16822 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16823 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16824 iRegP_R10 tmp, rFlagsReg cr)
16825 %{
16826 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16827 match(Set result (AryEq ary1 ary2));
16828 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16829 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16830 TEMP vtmp6, TEMP vtmp7, KILL cr);
16831
16832 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16833 ins_encode %{
16834 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16835 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16836 $result$$Register, $tmp$$Register, 2);
16837 if (tpc == nullptr) {
16838 ciEnv::current()->record_failure("CodeCache is full");
16839 return;
16840 }
16841 %}
16842 ins_pipe(pipe_class_memory);
16843 %}
16844
16845 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16846 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16847 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16848 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16849 %{
16850 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16851 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16852 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16853
16854 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16855 ins_encode %{
16856 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16857 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16858 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16859 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16860 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16861 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16862 (BasicType)$basic_type$$constant);
16863 if (tpc == nullptr) {
16864 ciEnv::current()->record_failure("CodeCache is full");
16865 return;
16866 }
16867 %}
16868 ins_pipe(pipe_class_memory);
16869 %}
16870
16871 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16872 %{
16873 match(Set result (CountPositives ary1 len));
16874 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16875 format %{ "count positives byte[] $ary1,$len -> $result" %}
16876 ins_encode %{
16877 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16878 if (tpc == nullptr) {
16879 ciEnv::current()->record_failure("CodeCache is full");
16880 return;
16881 }
16882 %}
16883 ins_pipe( pipe_slow );
16884 %}
16885
16886 // fast char[] to byte[] compression
16887 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16888 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16889 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16890 iRegI_R0 result, rFlagsReg cr)
16891 %{
16892 match(Set result (StrCompressedCopy src (Binary dst len)));
16893 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16894 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16895
16896 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16897 ins_encode %{
16898 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16899 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16900 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16901 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16902 %}
16903 ins_pipe(pipe_slow);
16904 %}
16905
16906 // fast byte[] to char[] inflation
16907 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16908 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16909 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16910 %{
16911 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16912 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16913 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16914 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16915
16916 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16917 ins_encode %{
16918 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16919 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16920 $vtmp2$$FloatRegister, $tmp$$Register);
16921 if (tpc == nullptr) {
16922 ciEnv::current()->record_failure("CodeCache is full");
16923 return;
16924 }
16925 %}
16926 ins_pipe(pipe_class_memory);
16927 %}
16928
16929 // encode char[] to byte[] in ISO_8859_1
16930 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16931 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16932 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16933 iRegI_R0 result, rFlagsReg cr)
16934 %{
16935 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16936 match(Set result (EncodeISOArray src (Binary dst len)));
16937 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16938 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16939
16940 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16941 ins_encode %{
16942 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16943 $result$$Register, false,
16944 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16945 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16946 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16947 %}
16948 ins_pipe(pipe_class_memory);
16949 %}
16950
16951 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16952 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16953 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16954 iRegI_R0 result, rFlagsReg cr)
16955 %{
16956 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16957 match(Set result (EncodeISOArray src (Binary dst len)));
16958 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16959 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16960
16961 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16962 ins_encode %{
16963 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16964 $result$$Register, true,
16965 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16966 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16967 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16968 %}
16969 ins_pipe(pipe_class_memory);
16970 %}
16971
16972 //----------------------------- CompressBits/ExpandBits ------------------------
16973
16974 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16975 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16976 match(Set dst (CompressBits src mask));
16977 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16978 format %{ "mov $tsrc, $src\n\t"
16979 "mov $tmask, $mask\n\t"
16980 "bext $tdst, $tsrc, $tmask\n\t"
16981 "mov $dst, $tdst"
16982 %}
16983 ins_encode %{
16984 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16985 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16986 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16987 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16988 %}
16989 ins_pipe(pipe_slow);
16990 %}
16991
16992 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16993 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16994 match(Set dst (CompressBits (LoadI mem) mask));
16995 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16996 format %{ "ldrs $tsrc, $mem\n\t"
16997 "ldrs $tmask, $mask\n\t"
16998 "bext $tdst, $tsrc, $tmask\n\t"
16999 "mov $dst, $tdst"
17000 %}
17001 ins_encode %{
17002 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17003 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17004 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17005 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17006 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17007 %}
17008 ins_pipe(pipe_slow);
17009 %}
17010
17011 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17012 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17013 match(Set dst (CompressBits src mask));
17014 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17015 format %{ "mov $tsrc, $src\n\t"
17016 "mov $tmask, $mask\n\t"
17017 "bext $tdst, $tsrc, $tmask\n\t"
17018 "mov $dst, $tdst"
17019 %}
17020 ins_encode %{
17021 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17022 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17023 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17024 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17025 %}
17026 ins_pipe(pipe_slow);
17027 %}
17028
17029 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17030 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17031 match(Set dst (CompressBits (LoadL mem) mask));
17032 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17033 format %{ "ldrd $tsrc, $mem\n\t"
17034 "ldrd $tmask, $mask\n\t"
17035 "bext $tdst, $tsrc, $tmask\n\t"
17036 "mov $dst, $tdst"
17037 %}
17038 ins_encode %{
17039 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17040 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17041 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17042 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17043 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17044 %}
17045 ins_pipe(pipe_slow);
17046 %}
17047
17048 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17049 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17050 match(Set dst (ExpandBits src mask));
17051 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17052 format %{ "mov $tsrc, $src\n\t"
17053 "mov $tmask, $mask\n\t"
17054 "bdep $tdst, $tsrc, $tmask\n\t"
17055 "mov $dst, $tdst"
17056 %}
17057 ins_encode %{
17058 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17059 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17060 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17061 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17062 %}
17063 ins_pipe(pipe_slow);
17064 %}
17065
17066 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17067 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17068 match(Set dst (ExpandBits (LoadI mem) mask));
17069 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17070 format %{ "ldrs $tsrc, $mem\n\t"
17071 "ldrs $tmask, $mask\n\t"
17072 "bdep $tdst, $tsrc, $tmask\n\t"
17073 "mov $dst, $tdst"
17074 %}
17075 ins_encode %{
17076 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17077 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17078 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17079 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17080 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17081 %}
17082 ins_pipe(pipe_slow);
17083 %}
17084
17085 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17086 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17087 match(Set dst (ExpandBits src mask));
17088 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17089 format %{ "mov $tsrc, $src\n\t"
17090 "mov $tmask, $mask\n\t"
17091 "bdep $tdst, $tsrc, $tmask\n\t"
17092 "mov $dst, $tdst"
17093 %}
17094 ins_encode %{
17095 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17096 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17097 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17098 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17099 %}
17100 ins_pipe(pipe_slow);
17101 %}
17102
17103
17104 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17105 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17106 match(Set dst (ExpandBits (LoadL mem) mask));
17107 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17108 format %{ "ldrd $tsrc, $mem\n\t"
17109 "ldrd $tmask, $mask\n\t"
17110 "bdep $tdst, $tsrc, $tmask\n\t"
17111 "mov $dst, $tdst"
17112 %}
17113 ins_encode %{
17114 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17115 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17116 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17117 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17118 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17119 %}
17120 ins_pipe(pipe_slow);
17121 %}
17122
17123 // ============================================================================
17124 // This name is KNOWN by the ADLC and cannot be changed.
17125 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17126 // for this guy.
17127 instruct tlsLoadP(thread_RegP dst)
17128 %{
17129 match(Set dst (ThreadLocal));
17130
17131 ins_cost(0);
17132
17133 format %{ " -- \t// $dst=Thread::current(), empty" %}
17134
17135 size(0);
17136
17137 ins_encode( /*empty*/ );
17138
17139 ins_pipe(pipe_class_empty);
17140 %}
17141
17142 //----------PEEPHOLE RULES-----------------------------------------------------
17143 // These must follow all instruction definitions as they use the names
17144 // defined in the instructions definitions.
17145 //
17146 // peepmatch ( root_instr_name [preceding_instruction]* );
17147 //
17148 // peepconstraint %{
17149 // (instruction_number.operand_name relational_op instruction_number.operand_name
17150 // [, ...] );
17151 // // instruction numbers are zero-based using left to right order in peepmatch
17152 //
17153 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17154 // // provide an instruction_number.operand_name for each operand that appears
17155 // // in the replacement instruction's match rule
17156 //
17157 // ---------VM FLAGS---------------------------------------------------------
17158 //
17159 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17160 //
17161 // Each peephole rule is given an identifying number starting with zero and
17162 // increasing by one in the order seen by the parser. An individual peephole
17163 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17164 // on the command-line.
17165 //
17166 // ---------CURRENT LIMITATIONS----------------------------------------------
17167 //
17168 // Only match adjacent instructions in same basic block
17169 // Only equality constraints
17170 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17171 // Only one replacement instruction
17172 //
17173 // ---------EXAMPLE----------------------------------------------------------
17174 //
17175 // // pertinent parts of existing instructions in architecture description
17176 // instruct movI(iRegINoSp dst, iRegI src)
17177 // %{
17178 // match(Set dst (CopyI src));
17179 // %}
17180 //
17181 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17182 // %{
17183 // match(Set dst (AddI dst src));
17184 // effect(KILL cr);
17185 // %}
17186 //
17187 // // Change (inc mov) to lea
17188 // peephole %{
17189 // // increment preceded by register-register move
17190 // peepmatch ( incI_iReg movI );
17191 // // require that the destination register of the increment
17192 // // match the destination register of the move
17193 // peepconstraint ( 0.dst == 1.dst );
17194 // // construct a replacement instruction that sets
17195 // // the destination to ( move's source register + one )
17196 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17197 // %}
17198 //
17199
17200 // Implementation no longer uses movX instructions since
17201 // machine-independent system no longer uses CopyX nodes.
17202 //
17203 // peephole
17204 // %{
17205 // peepmatch (incI_iReg movI);
17206 // peepconstraint (0.dst == 1.dst);
17207 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17208 // %}
17209
17210 // peephole
17211 // %{
17212 // peepmatch (decI_iReg movI);
17213 // peepconstraint (0.dst == 1.dst);
17214 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17215 // %}
17216
17217 // peephole
17218 // %{
17219 // peepmatch (addI_iReg_imm movI);
17220 // peepconstraint (0.dst == 1.dst);
17221 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17222 // %}
17223
17224 // peephole
17225 // %{
17226 // peepmatch (incL_iReg movL);
17227 // peepconstraint (0.dst == 1.dst);
17228 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17229 // %}
17230
17231 // peephole
17232 // %{
17233 // peepmatch (decL_iReg movL);
17234 // peepconstraint (0.dst == 1.dst);
17235 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17236 // %}
17237
17238 // peephole
17239 // %{
17240 // peepmatch (addL_iReg_imm movL);
17241 // peepconstraint (0.dst == 1.dst);
17242 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17243 // %}
17244
17245 // peephole
17246 // %{
17247 // peepmatch (addP_iReg_imm movP);
17248 // peepconstraint (0.dst == 1.dst);
17249 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17250 // %}
17251
17252 // // Change load of spilled value to only a spill
17253 // instruct storeI(memory mem, iRegI src)
17254 // %{
17255 // match(Set mem (StoreI mem src));
17256 // %}
17257 //
17258 // instruct loadI(iRegINoSp dst, memory mem)
17259 // %{
17260 // match(Set dst (LoadI mem));
17261 // %}
17262 //
17263
17264 //----------SMARTSPILL RULES---------------------------------------------------
17265 // These must follow all instruction definitions as they use the names
17266 // defined in the instructions definitions.
17267
17268 // Local Variables:
17269 // mode: c++
17270 // End: