1 //
2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return MacroAssembler::max_trampoline_stub_size(); // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1652 // lea(rscratch1, RuntimeAddress(addr)
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else {
1658 return 6 * NativeInstruction::instruction_size;
1659 }
1660 }
1661
1662 //=============================================================================
1663
1664 #ifndef PRODUCT
1665 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1666 st->print("BREAKPOINT");
1667 }
1668 #endif
1669
1670 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1671 __ brk(0);
1672 }
1673
1674 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1675 return MachNode::size(ra_);
1676 }
1677
1678 //=============================================================================
1679
1680 #ifndef PRODUCT
1681 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1682 st->print("nop \t# %d bytes pad for loops and calls", _count);
1683 }
1684 #endif
1685
1686 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1687 for (int i = 0; i < _count; i++) {
1688 __ nop();
1689 }
1690 }
1691
1692 uint MachNopNode::size(PhaseRegAlloc*) const {
1693 return _count * NativeInstruction::instruction_size;
1694 }
1695
1696 //=============================================================================
1697 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1698
1699 int ConstantTable::calculate_table_base_offset() const {
1700 return 0; // absolute addressing, no offset
1701 }
1702
1703 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1704 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1705 ShouldNotReachHere();
1706 }
1707
1708 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1709 // Empty encoding
1710 }
1711
1712 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1713 return 0;
1714 }
1715
1716 #ifndef PRODUCT
1717 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1718 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1719 }
1720 #endif
1721
1722 #ifndef PRODUCT
1723 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1724 Compile* C = ra_->C;
1725
1726 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1727
1728 if (C->output()->need_stack_bang(framesize))
1729 st->print("# stack bang size=%d\n\t", framesize);
1730
1731 if (VM_Version::use_rop_protection()) {
1732 st->print("ldr zr, [lr]\n\t");
1733 st->print("paciaz\n\t");
1734 }
1735 if (framesize < ((1 << 9) + 2 * wordSize)) {
1736 st->print("sub sp, sp, #%d\n\t", framesize);
1737 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1738 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1739 } else {
1740 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1741 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1742 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1743 st->print("sub sp, sp, rscratch1");
1744 }
1745 if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1746 st->print("\n\t");
1747 st->print("ldr rscratch1, [guard]\n\t");
1748 st->print("dmb ishld\n\t");
1749 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1750 st->print("cmp rscratch1, rscratch2\n\t");
1751 st->print("b.eq skip");
1752 st->print("\n\t");
1753 st->print("blr #nmethod_entry_barrier_stub\n\t");
1754 st->print("b skip\n\t");
1755 st->print("guard: int\n\t");
1756 st->print("\n\t");
1757 st->print("skip:\n\t");
1758 }
1759 }
1760 #endif
1761
1762 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1763 Compile* C = ra_->C;
1764
1765 // n.b. frame size includes space for return pc and rfp
1766 const int framesize = C->output()->frame_size_in_bytes();
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 if (C->clinit_barrier_on_entry()) {
1773 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1774
1775 Label L_skip_barrier;
1776
1777 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1778 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1779 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1780 __ bind(L_skip_barrier);
1781 }
1782
1783 if (C->max_vector_size() > 0) {
1784 __ reinitialize_ptrue();
1785 }
1786
1787 int bangsize = C->output()->bang_size_in_bytes();
1788 if (C->output()->need_stack_bang(bangsize))
1789 __ generate_stack_overflow_check(bangsize);
1790
1791 __ build_frame(framesize);
1792
1793 if (C->stub_function() == nullptr) {
1794 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1795 if (BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1796 // Dummy labels for just measuring the code size
1797 Label dummy_slow_path;
1798 Label dummy_continuation;
1799 Label dummy_guard;
1800 Label* slow_path = &dummy_slow_path;
1801 Label* continuation = &dummy_continuation;
1802 Label* guard = &dummy_guard;
1803 if (!Compile::current()->output()->in_scratch_emit_size()) {
1804 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1805 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1806 Compile::current()->output()->add_stub(stub);
1807 slow_path = &stub->entry();
1808 continuation = &stub->continuation();
1809 guard = &stub->guard();
1810 }
1811 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1812 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1813 }
1814 }
1815
1816 if (VerifyStackAtCalls) {
1817 Unimplemented();
1818 }
1819
1820 C->output()->set_frame_complete(__ offset());
1821
1822 if (C->has_mach_constant_base_node()) {
1823 // NOTE: We set the table base offset here because users might be
1824 // emitted before MachConstantBaseNode.
1825 ConstantTable& constant_table = C->output()->constant_table();
1826 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1827 }
1828 }
1829
1830 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1831 {
1832 return MachNode::size(ra_); // too many variables; just compute it
1833 // the hard way
1834 }
1835
1836 int MachPrologNode::reloc() const
1837 {
1838 return 0;
1839 }
1840
1841 //=============================================================================
1842
1843 #ifndef PRODUCT
1844 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1845 Compile* C = ra_->C;
1846 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1847
1848 st->print("# pop frame %d\n\t",framesize);
1849
1850 if (framesize == 0) {
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1853 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1854 st->print("add sp, sp, #%d\n\t", framesize);
1855 } else {
1856 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1857 st->print("add sp, sp, rscratch1\n\t");
1858 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1859 }
1860 if (VM_Version::use_rop_protection()) {
1861 st->print("autiaz\n\t");
1862 st->print("ldr zr, [lr]\n\t");
1863 }
1864
1865 if (do_polling() && C->is_method_compilation()) {
1866 st->print("# test polling word\n\t");
1867 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1868 st->print("cmp sp, rscratch1\n\t");
1869 st->print("bhi #slow_path");
1870 }
1871 }
1872 #endif
1873
1874 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 __ remove_frame(framesize);
1879
1880 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1881 __ reserved_stack_check();
1882 }
1883
1884 if (do_polling() && C->is_method_compilation()) {
1885 Label dummy_label;
1886 Label* code_stub = &dummy_label;
1887 if (!C->output()->in_scratch_emit_size()) {
1888 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1889 C->output()->add_stub(stub);
1890 code_stub = &stub->entry();
1891 }
1892 __ relocate(relocInfo::poll_return_type);
1893 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1894 }
1895 }
1896
1897 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1898 // Variable size. Determine dynamically.
1899 return MachNode::size(ra_);
1900 }
1901
1902 int MachEpilogNode::reloc() const {
1903 // Return number of relocatable values contained in this instruction.
1904 return 1; // 1 for polling page.
1905 }
1906
1907 const Pipeline * MachEpilogNode::pipeline() const {
1908 return MachNode::pipeline_class();
1909 }
1910
1911 //=============================================================================
1912
1913 static enum RC rc_class(OptoReg::Name reg) {
1914
1915 if (reg == OptoReg::Bad) {
1916 return rc_bad;
1917 }
1918
1919 // we have 32 int registers * 2 halves
1920 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1921
1922 if (reg < slots_of_int_registers) {
1923 return rc_int;
1924 }
1925
1926 // we have 32 float register * 8 halves
1927 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1928 if (reg < slots_of_int_registers + slots_of_float_registers) {
1929 return rc_float;
1930 }
1931
1932 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1933 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1934 return rc_predicate;
1935 }
1936
1937 // Between predicate regs & stack is the flags.
1938 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1939
1940 return rc_stack;
1941 }
1942
1943 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1944 Compile* C = ra_->C;
1945
1946 // Get registers to move.
1947 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1948 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1949 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1950 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1951
1952 enum RC src_hi_rc = rc_class(src_hi);
1953 enum RC src_lo_rc = rc_class(src_lo);
1954 enum RC dst_hi_rc = rc_class(dst_hi);
1955 enum RC dst_lo_rc = rc_class(dst_lo);
1956
1957 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1958
1959 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1960 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1961 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1962 "expected aligned-adjacent pairs");
1963 }
1964
1965 if (src_lo == dst_lo && src_hi == dst_hi) {
1966 return 0; // Self copy, no move.
1967 }
1968
1969 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1970 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1971 int src_offset = ra_->reg2offset(src_lo);
1972 int dst_offset = ra_->reg2offset(dst_lo);
1973
1974 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1975 uint ireg = ideal_reg();
1976 if (ireg == Op_VecA && masm) {
1977 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1978 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1979 // stack->stack
1980 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1981 sve_vector_reg_size_in_bytes);
1982 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1983 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1984 sve_vector_reg_size_in_bytes);
1985 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1986 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1987 sve_vector_reg_size_in_bytes);
1988 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1989 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1990 as_FloatRegister(Matcher::_regEncode[src_lo]),
1991 as_FloatRegister(Matcher::_regEncode[src_lo]));
1992 } else {
1993 ShouldNotReachHere();
1994 }
1995 } else if (masm) {
1996 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1997 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1998 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1999 // stack->stack
2000 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2001 if (ireg == Op_VecD) {
2002 __ unspill(rscratch1, true, src_offset);
2003 __ spill(rscratch1, true, dst_offset);
2004 } else {
2005 __ spill_copy128(src_offset, dst_offset);
2006 }
2007 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2008 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2009 ireg == Op_VecD ? __ T8B : __ T16B,
2010 as_FloatRegister(Matcher::_regEncode[src_lo]));
2011 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2012 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2013 ireg == Op_VecD ? __ D : __ Q,
2014 ra_->reg2offset(dst_lo));
2015 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2016 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 ireg == Op_VecD ? __ D : __ Q,
2018 ra_->reg2offset(src_lo));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 }
2023 } else if (masm) {
2024 switch (src_lo_rc) {
2025 case rc_int:
2026 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2027 if (is64) {
2028 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2029 as_Register(Matcher::_regEncode[src_lo]));
2030 } else {
2031 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2032 as_Register(Matcher::_regEncode[src_lo]));
2033 }
2034 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2035 if (is64) {
2036 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2037 as_Register(Matcher::_regEncode[src_lo]));
2038 } else {
2039 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2040 as_Register(Matcher::_regEncode[src_lo]));
2041 }
2042 } else { // gpr --> stack spill
2043 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2044 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2045 }
2046 break;
2047 case rc_float:
2048 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2049 if (is64) {
2050 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2051 as_FloatRegister(Matcher::_regEncode[src_lo]));
2052 } else {
2053 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2054 as_FloatRegister(Matcher::_regEncode[src_lo]));
2055 }
2056 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2057 if (is64) {
2058 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2059 as_FloatRegister(Matcher::_regEncode[src_lo]));
2060 } else {
2061 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2062 as_FloatRegister(Matcher::_regEncode[src_lo]));
2063 }
2064 } else { // fpr --> stack spill
2065 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2066 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2067 is64 ? __ D : __ S, dst_offset);
2068 }
2069 break;
2070 case rc_stack:
2071 if (dst_lo_rc == rc_int) { // stack --> gpr load
2072 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2073 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2074 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2075 is64 ? __ D : __ S, src_offset);
2076 } else if (dst_lo_rc == rc_predicate) {
2077 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2078 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2079 } else { // stack --> stack copy
2080 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2081 if (ideal_reg() == Op_RegVectMask) {
2082 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2083 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2084 } else {
2085 __ unspill(rscratch1, is64, src_offset);
2086 __ spill(rscratch1, is64, dst_offset);
2087 }
2088 }
2089 break;
2090 case rc_predicate:
2091 if (dst_lo_rc == rc_predicate) {
2092 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2093 } else if (dst_lo_rc == rc_stack) {
2094 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2095 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2096 } else {
2097 assert(false, "bad src and dst rc_class combination.");
2098 ShouldNotReachHere();
2099 }
2100 break;
2101 default:
2102 assert(false, "bad rc_class for spill");
2103 ShouldNotReachHere();
2104 }
2105 }
2106
2107 if (st) {
2108 st->print("spill ");
2109 if (src_lo_rc == rc_stack) {
2110 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2111 } else {
2112 st->print("%s -> ", Matcher::regName[src_lo]);
2113 }
2114 if (dst_lo_rc == rc_stack) {
2115 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2116 } else {
2117 st->print("%s", Matcher::regName[dst_lo]);
2118 }
2119 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2120 int vsize = 0;
2121 switch (ideal_reg()) {
2122 case Op_VecD:
2123 vsize = 64;
2124 break;
2125 case Op_VecX:
2126 vsize = 128;
2127 break;
2128 case Op_VecA:
2129 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2130 break;
2131 default:
2132 assert(false, "bad register type for spill");
2133 ShouldNotReachHere();
2134 }
2135 st->print("\t# vector spill size = %d", vsize);
2136 } else if (ideal_reg() == Op_RegVectMask) {
2137 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2138 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2139 st->print("\t# predicate spill size = %d", vsize);
2140 } else {
2141 st->print("\t# spill size = %d", is64 ? 64 : 32);
2142 }
2143 }
2144
2145 return 0;
2146
2147 }
2148
2149 #ifndef PRODUCT
2150 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2151 if (!ra_)
2152 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2153 else
2154 implementation(nullptr, ra_, false, st);
2155 }
2156 #endif
2157
2158 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2159 implementation(masm, ra_, false, nullptr);
2160 }
2161
2162 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2163 return MachNode::size(ra_);
2164 }
2165
2166 //=============================================================================
2167
2168 #ifndef PRODUCT
2169 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2170 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2171 int reg = ra_->get_reg_first(this);
2172 st->print("add %s, rsp, #%d]\t# box lock",
2173 Matcher::regName[reg], offset);
2174 }
2175 #endif
2176
2177 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2178 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2179 int reg = ra_->get_encode(this);
2180
2181 // This add will handle any 24-bit signed offset. 24 bits allows an
2182 // 8 megabyte stack frame.
2183 __ add(as_Register(reg), sp, offset);
2184 }
2185
2186 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2187 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2188 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2189
2190 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2191 return NativeInstruction::instruction_size;
2192 } else {
2193 return 2 * NativeInstruction::instruction_size;
2194 }
2195 }
2196
2197 //=============================================================================
2198
2199 #ifndef PRODUCT
2200 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2201 {
2202 st->print_cr("# MachUEPNode");
2203 if (UseCompressedClassPointers) {
2204 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2205 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2206 st->print_cr("\tcmpw rscratch1, r10");
2207 } else {
2208 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2209 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2210 st->print_cr("\tcmp rscratch1, r10");
2211 }
2212 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2213 }
2214 #endif
2215
2216 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2217 {
2218 __ ic_check(InteriorEntryAlignment);
2219 }
2220
2221 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2222 {
2223 return MachNode::size(ra_);
2224 }
2225
2226 // REQUIRED EMIT CODE
2227
2228 //=============================================================================
2229
2230 // Emit exception handler code.
2231 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2232 {
2233 // mov rscratch1 #exception_blob_entry_point
2234 // br rscratch1
2235 // Note that the code buffer's insts_mark is always relative to insts.
2236 // That's why we must use the macroassembler to generate a handler.
2237 address base = __ start_a_stub(size_exception_handler());
2238 if (base == nullptr) {
2239 ciEnv::current()->record_failure("CodeCache is full");
2240 return 0; // CodeBuffer::expand failed
2241 }
2242 int offset = __ offset();
2243 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2244 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2245 __ end_a_stub();
2246 return offset;
2247 }
2248
2249 // Emit deopt handler code.
2250 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2251 {
2252 // Note that the code buffer's insts_mark is always relative to insts.
2253 // That's why we must use the macroassembler to generate a handler.
2254 address base = __ start_a_stub(size_deopt_handler());
2255 if (base == nullptr) {
2256 ciEnv::current()->record_failure("CodeCache is full");
2257 return 0; // CodeBuffer::expand failed
2258 }
2259 int offset = __ offset();
2260
2261 __ adr(lr, __ pc());
2262 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2263
2264 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2265 __ end_a_stub();
2266 return offset;
2267 }
2268
2269 // REQUIRED MATCHER CODE
2270
2271 //=============================================================================
2272
2273 bool Matcher::match_rule_supported(int opcode) {
2274 if (!has_match_rule(opcode))
2275 return false;
2276
2277 switch (opcode) {
2278 case Op_OnSpinWait:
2279 return VM_Version::supports_on_spin_wait();
2280 case Op_CacheWB:
2281 case Op_CacheWBPreSync:
2282 case Op_CacheWBPostSync:
2283 if (!VM_Version::supports_data_cache_line_flush()) {
2284 return false;
2285 }
2286 break;
2287 case Op_ExpandBits:
2288 case Op_CompressBits:
2289 if (!VM_Version::supports_svebitperm()) {
2290 return false;
2291 }
2292 break;
2293 case Op_FmaF:
2294 case Op_FmaD:
2295 case Op_FmaVF:
2296 case Op_FmaVD:
2297 if (!UseFMA) {
2298 return false;
2299 }
2300 break;
2301 }
2302
2303 return true; // Per default match rules are supported.
2304 }
2305
2306 const RegMask* Matcher::predicate_reg_mask(void) {
2307 return &_PR_REG_mask;
2308 }
2309
2310 bool Matcher::supports_vector_calling_convention(void) {
2311 return EnableVectorSupport && UseVectorStubs;
2312 }
2313
2314 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2315 assert(EnableVectorSupport && UseVectorStubs, "sanity");
2316 int lo = V0_num;
2317 int hi = V0_H_num;
2318 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2319 hi = V0_K_num;
2320 }
2321 return OptoRegPair(hi, lo);
2322 }
2323
2324 // Is this branch offset short enough that a short branch can be used?
2325 //
2326 // NOTE: If the platform does not provide any short branch variants, then
2327 // this method should return false for offset 0.
2328 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2329 // The passed offset is relative to address of the branch.
2330
2331 return (-32768 <= offset && offset < 32768);
2332 }
2333
2334 // Vector width in bytes.
2335 int Matcher::vector_width_in_bytes(BasicType bt) {
2336 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2337 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2338 // Minimum 2 values in vector
2339 if (size < 2*type2aelembytes(bt)) size = 0;
2340 // But never < 4
2341 if (size < 4) size = 0;
2342 return size;
2343 }
2344
2345 // Limits on vector size (number of elements) loaded into vector.
2346 int Matcher::max_vector_size(const BasicType bt) {
2347 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2348 }
2349
2350 int Matcher::min_vector_size(const BasicType bt) {
2351 int max_size = max_vector_size(bt);
2352 // Limit the min vector size to 8 bytes.
2353 int size = 8 / type2aelembytes(bt);
2354 if (bt == T_BYTE) {
2355 // To support vector api shuffle/rearrange.
2356 size = 4;
2357 } else if (bt == T_BOOLEAN) {
2358 // To support vector api load/store mask.
2359 size = 2;
2360 }
2361 if (size < 2) size = 2;
2362 return MIN2(size, max_size);
2363 }
2364
2365 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2366 return Matcher::max_vector_size(bt);
2367 }
2368
2369 // Actual max scalable vector register length.
2370 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2371 return Matcher::max_vector_size(bt);
2372 }
2373
2374 // Vector ideal reg.
2375 uint Matcher::vector_ideal_reg(int len) {
2376 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2377 return Op_VecA;
2378 }
2379 switch(len) {
2380 // For 16-bit/32-bit mask vector, reuse VecD.
2381 case 2:
2382 case 4:
2383 case 8: return Op_VecD;
2384 case 16: return Op_VecX;
2385 }
2386 ShouldNotReachHere();
2387 return 0;
2388 }
2389
2390 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2391 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2392 switch (ideal_reg) {
2393 case Op_VecA: return new vecAOper();
2394 case Op_VecD: return new vecDOper();
2395 case Op_VecX: return new vecXOper();
2396 }
2397 ShouldNotReachHere();
2398 return nullptr;
2399 }
2400
2401 bool Matcher::is_reg2reg_move(MachNode* m) {
2402 return false;
2403 }
2404
2405 bool Matcher::is_generic_vector(MachOper* opnd) {
2406 return opnd->opcode() == VREG;
2407 }
2408
2409 // Return whether or not this register is ever used as an argument.
2410 // This function is used on startup to build the trampoline stubs in
2411 // generateOptoStub. Registers not mentioned will be killed by the VM
2412 // call in the trampoline, and arguments in those registers not be
2413 // available to the callee.
2414 bool Matcher::can_be_java_arg(int reg)
2415 {
2416 return
2417 reg == R0_num || reg == R0_H_num ||
2418 reg == R1_num || reg == R1_H_num ||
2419 reg == R2_num || reg == R2_H_num ||
2420 reg == R3_num || reg == R3_H_num ||
2421 reg == R4_num || reg == R4_H_num ||
2422 reg == R5_num || reg == R5_H_num ||
2423 reg == R6_num || reg == R6_H_num ||
2424 reg == R7_num || reg == R7_H_num ||
2425 reg == V0_num || reg == V0_H_num ||
2426 reg == V1_num || reg == V1_H_num ||
2427 reg == V2_num || reg == V2_H_num ||
2428 reg == V3_num || reg == V3_H_num ||
2429 reg == V4_num || reg == V4_H_num ||
2430 reg == V5_num || reg == V5_H_num ||
2431 reg == V6_num || reg == V6_H_num ||
2432 reg == V7_num || reg == V7_H_num;
2433 }
2434
2435 bool Matcher::is_spillable_arg(int reg)
2436 {
2437 return can_be_java_arg(reg);
2438 }
2439
2440 uint Matcher::int_pressure_limit()
2441 {
2442 // JDK-8183543: When taking the number of available registers as int
2443 // register pressure threshold, the jtreg test:
2444 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2445 // failed due to C2 compilation failure with
2446 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2447 //
2448 // A derived pointer is live at CallNode and then is flagged by RA
2449 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2450 // derived pointers and lastly fail to spill after reaching maximum
2451 // number of iterations. Lowering the default pressure threshold to
2452 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2453 // a high register pressure area of the code so that split_DEF can
2454 // generate DefinitionSpillCopy for the derived pointer.
2455 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2456 if (!PreserveFramePointer) {
2457 // When PreserveFramePointer is off, frame pointer is allocatable,
2458 // but different from other SOC registers, it is excluded from
2459 // fatproj's mask because its save type is No-Save. Decrease 1 to
2460 // ensure high pressure at fatproj when PreserveFramePointer is off.
2461 // See check_pressure_at_fatproj().
2462 default_int_pressure_threshold--;
2463 }
2464 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2465 }
2466
2467 uint Matcher::float_pressure_limit()
2468 {
2469 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2470 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2471 }
2472
2473 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2474 return false;
2475 }
2476
2477 RegMask Matcher::divI_proj_mask() {
2478 ShouldNotReachHere();
2479 return RegMask();
2480 }
2481
2482 // Register for MODI projection of divmodI.
2483 RegMask Matcher::modI_proj_mask() {
2484 ShouldNotReachHere();
2485 return RegMask();
2486 }
2487
2488 // Register for DIVL projection of divmodL.
2489 RegMask Matcher::divL_proj_mask() {
2490 ShouldNotReachHere();
2491 return RegMask();
2492 }
2493
2494 // Register for MODL projection of divmodL.
2495 RegMask Matcher::modL_proj_mask() {
2496 ShouldNotReachHere();
2497 return RegMask();
2498 }
2499
2500 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2501 return FP_REG_mask();
2502 }
2503
2504 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2505 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2506 Node* u = addp->fast_out(i);
2507 if (u->is_LoadStore()) {
2508 // On AArch64, LoadStoreNodes (i.e. compare and swap
2509 // instructions) only take register indirect as an operand, so
2510 // any attempt to use an AddPNode as an input to a LoadStoreNode
2511 // must fail.
2512 return false;
2513 }
2514 if (u->is_Mem()) {
2515 int opsize = u->as_Mem()->memory_size();
2516 assert(opsize > 0, "unexpected memory operand size");
2517 if (u->as_Mem()->memory_size() != (1<<shift)) {
2518 return false;
2519 }
2520 }
2521 }
2522 return true;
2523 }
2524
2525 // Convert BootTest condition to Assembler condition.
2526 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2527 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2528 Assembler::Condition result;
2529 switch(cond) {
2530 case BoolTest::eq:
2531 result = Assembler::EQ; break;
2532 case BoolTest::ne:
2533 result = Assembler::NE; break;
2534 case BoolTest::le:
2535 result = Assembler::LE; break;
2536 case BoolTest::ge:
2537 result = Assembler::GE; break;
2538 case BoolTest::lt:
2539 result = Assembler::LT; break;
2540 case BoolTest::gt:
2541 result = Assembler::GT; break;
2542 case BoolTest::ule:
2543 result = Assembler::LS; break;
2544 case BoolTest::uge:
2545 result = Assembler::HS; break;
2546 case BoolTest::ult:
2547 result = Assembler::LO; break;
2548 case BoolTest::ugt:
2549 result = Assembler::HI; break;
2550 case BoolTest::overflow:
2551 result = Assembler::VS; break;
2552 case BoolTest::no_overflow:
2553 result = Assembler::VC; break;
2554 default:
2555 ShouldNotReachHere();
2556 return Assembler::Condition(-1);
2557 }
2558
2559 // Check conversion
2560 if (cond & BoolTest::unsigned_compare) {
2561 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2562 } else {
2563 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2564 }
2565
2566 return result;
2567 }
2568
2569 // Binary src (Replicate con)
2570 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2571 if (n == nullptr || m == nullptr) {
2572 return false;
2573 }
2574
2575 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2576 return false;
2577 }
2578
2579 Node* imm_node = m->in(1);
2580 if (!imm_node->is_Con()) {
2581 return false;
2582 }
2583
2584 const Type* t = imm_node->bottom_type();
2585 if (!(t->isa_int() || t->isa_long())) {
2586 return false;
2587 }
2588
2589 switch (n->Opcode()) {
2590 case Op_AndV:
2591 case Op_OrV:
2592 case Op_XorV: {
2593 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2594 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2595 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2596 }
2597 case Op_AddVB:
2598 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2599 case Op_AddVS:
2600 case Op_AddVI:
2601 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2602 case Op_AddVL:
2603 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2604 default:
2605 return false;
2606 }
2607 }
2608
2609 // (XorV src (Replicate m1))
2610 // (XorVMask src (MaskAll m1))
2611 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2612 if (n != nullptr && m != nullptr) {
2613 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2614 VectorNode::is_all_ones_vector(m);
2615 }
2616 return false;
2617 }
2618
2619 // Should the matcher clone input 'm' of node 'n'?
2620 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2621 if (is_vshift_con_pattern(n, m) ||
2622 is_vector_bitwise_not_pattern(n, m) ||
2623 is_valid_sve_arith_imm_pattern(n, m) ||
2624 is_encode_and_store_pattern(n, m)) {
2625 mstack.push(m, Visit);
2626 return true;
2627 }
2628 return false;
2629 }
2630
2631 // Should the Matcher clone shifts on addressing modes, expecting them
2632 // to be subsumed into complex addressing expressions or compute them
2633 // into registers?
2634 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2635 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2636 return true;
2637 }
2638
2639 Node *off = m->in(AddPNode::Offset);
2640 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2641 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2642 // Are there other uses besides address expressions?
2643 !is_visited(off)) {
2644 address_visited.set(off->_idx); // Flag as address_visited
2645 mstack.push(off->in(2), Visit);
2646 Node *conv = off->in(1);
2647 if (conv->Opcode() == Op_ConvI2L &&
2648 // Are there other uses besides address expressions?
2649 !is_visited(conv)) {
2650 address_visited.set(conv->_idx); // Flag as address_visited
2651 mstack.push(conv->in(1), Pre_Visit);
2652 } else {
2653 mstack.push(conv, Pre_Visit);
2654 }
2655 address_visited.test_set(m->_idx); // Flag as address_visited
2656 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2657 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2658 return true;
2659 } else if (off->Opcode() == Op_ConvI2L &&
2660 // Are there other uses besides address expressions?
2661 !is_visited(off)) {
2662 address_visited.test_set(m->_idx); // Flag as address_visited
2663 address_visited.set(off->_idx); // Flag as address_visited
2664 mstack.push(off->in(1), Pre_Visit);
2665 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2666 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2667 return true;
2668 }
2669 return false;
2670 }
2671
2672 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2673 { \
2674 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2675 guarantee(DISP == 0, "mode not permitted for volatile"); \
2676 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2677 __ INSN(REG, as_Register(BASE)); \
2678 }
2679
2680
2681 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2682 {
2683 Address::extend scale;
2684
2685 // Hooboy, this is fugly. We need a way to communicate to the
2686 // encoder that the index needs to be sign extended, so we have to
2687 // enumerate all the cases.
2688 switch (opcode) {
2689 case INDINDEXSCALEDI2L:
2690 case INDINDEXSCALEDI2LN:
2691 case INDINDEXI2L:
2692 case INDINDEXI2LN:
2693 scale = Address::sxtw(size);
2694 break;
2695 default:
2696 scale = Address::lsl(size);
2697 }
2698
2699 if (index == -1) {
2700 return Address(base, disp);
2701 } else {
2702 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2703 return Address(base, as_Register(index), scale);
2704 }
2705 }
2706
2707
2708 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2709 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2710 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2711 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2712 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2713
2714 // Used for all non-volatile memory accesses. The use of
2715 // $mem->opcode() to discover whether this pattern uses sign-extended
2716 // offsets is something of a kludge.
2717 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2718 Register reg, int opcode,
2719 Register base, int index, int scale, int disp,
2720 int size_in_memory)
2721 {
2722 Address addr = mem2address(opcode, base, index, scale, disp);
2723 if (addr.getMode() == Address::base_plus_offset) {
2724 /* Fix up any out-of-range offsets. */
2725 assert_different_registers(rscratch1, base);
2726 assert_different_registers(rscratch1, reg);
2727 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2728 }
2729 (masm->*insn)(reg, addr);
2730 }
2731
2732 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2733 FloatRegister reg, int opcode,
2734 Register base, int index, int size, int disp,
2735 int size_in_memory)
2736 {
2737 Address::extend scale;
2738
2739 switch (opcode) {
2740 case INDINDEXSCALEDI2L:
2741 case INDINDEXSCALEDI2LN:
2742 scale = Address::sxtw(size);
2743 break;
2744 default:
2745 scale = Address::lsl(size);
2746 }
2747
2748 if (index == -1) {
2749 // Fix up any out-of-range offsets.
2750 assert_different_registers(rscratch1, base);
2751 Address addr = Address(base, disp);
2752 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2753 (masm->*insn)(reg, addr);
2754 } else {
2755 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2756 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2757 }
2758 }
2759
2760 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2761 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2762 int opcode, Register base, int index, int size, int disp)
2763 {
2764 if (index == -1) {
2765 (masm->*insn)(reg, T, Address(base, disp));
2766 } else {
2767 assert(disp == 0, "unsupported address mode");
2768 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2769 }
2770 }
2771
2772 %}
2773
2774
2775
2776 //----------ENCODING BLOCK-----------------------------------------------------
2777 // This block specifies the encoding classes used by the compiler to
2778 // output byte streams. Encoding classes are parameterized macros
2779 // used by Machine Instruction Nodes in order to generate the bit
2780 // encoding of the instruction. Operands specify their base encoding
2781 // interface with the interface keyword. There are currently
2782 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2783 // COND_INTER. REG_INTER causes an operand to generate a function
2784 // which returns its register number when queried. CONST_INTER causes
2785 // an operand to generate a function which returns the value of the
2786 // constant when queried. MEMORY_INTER causes an operand to generate
2787 // four functions which return the Base Register, the Index Register,
2788 // the Scale Value, and the Offset Value of the operand when queried.
2789 // COND_INTER causes an operand to generate six functions which return
2790 // the encoding code (ie - encoding bits for the instruction)
2791 // associated with each basic boolean condition for a conditional
2792 // instruction.
2793 //
2794 // Instructions specify two basic values for encoding. Again, a
2795 // function is available to check if the constant displacement is an
2796 // oop. They use the ins_encode keyword to specify their encoding
2797 // classes (which must be a sequence of enc_class names, and their
2798 // parameters, specified in the encoding block), and they use the
2799 // opcode keyword to specify, in order, their primary, secondary, and
2800 // tertiary opcode. Only the opcode sections which a particular
2801 // instruction needs for encoding need to be specified.
2802 encode %{
2803 // Build emit functions for each basic byte or larger field in the
2804 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2805 // from C++ code in the enc_class source block. Emit functions will
2806 // live in the main source block for now. In future, we can
2807 // generalize this by adding a syntax that specifies the sizes of
2808 // fields in an order, so that the adlc can build the emit functions
2809 // automagically
2810
2811 // catch all for unimplemented encodings
2812 enc_class enc_unimplemented %{
2813 __ unimplemented("C2 catch all");
2814 %}
2815
2816 // BEGIN Non-volatile memory access
2817
2818 // This encoding class is generated automatically from ad_encode.m4.
2819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2820 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2821 Register dst_reg = as_Register($dst$$reg);
2822 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2823 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2824 %}
2825
2826 // This encoding class is generated automatically from ad_encode.m4.
2827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2828 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2829 Register dst_reg = as_Register($dst$$reg);
2830 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2831 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2832 %}
2833
2834 // This encoding class is generated automatically from ad_encode.m4.
2835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2836 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2837 Register dst_reg = as_Register($dst$$reg);
2838 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2839 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2840 %}
2841
2842 // This encoding class is generated automatically from ad_encode.m4.
2843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2844 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2845 Register dst_reg = as_Register($dst$$reg);
2846 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2847 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2848 %}
2849
2850 // This encoding class is generated automatically from ad_encode.m4.
2851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2852 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2853 Register dst_reg = as_Register($dst$$reg);
2854 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2855 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2856 %}
2857
2858 // This encoding class is generated automatically from ad_encode.m4.
2859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2860 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2861 Register dst_reg = as_Register($dst$$reg);
2862 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2863 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2864 %}
2865
2866 // This encoding class is generated automatically from ad_encode.m4.
2867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2868 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2869 Register dst_reg = as_Register($dst$$reg);
2870 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2871 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2872 %}
2873
2874 // This encoding class is generated automatically from ad_encode.m4.
2875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2876 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2877 Register dst_reg = as_Register($dst$$reg);
2878 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2879 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2880 %}
2881
2882 // This encoding class is generated automatically from ad_encode.m4.
2883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2884 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2885 Register dst_reg = as_Register($dst$$reg);
2886 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2887 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2888 %}
2889
2890 // This encoding class is generated automatically from ad_encode.m4.
2891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2892 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2893 Register dst_reg = as_Register($dst$$reg);
2894 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2895 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2896 %}
2897
2898 // This encoding class is generated automatically from ad_encode.m4.
2899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2900 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2901 Register dst_reg = as_Register($dst$$reg);
2902 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2903 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2904 %}
2905
2906 // This encoding class is generated automatically from ad_encode.m4.
2907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2908 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2909 Register dst_reg = as_Register($dst$$reg);
2910 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2911 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2912 %}
2913
2914 // This encoding class is generated automatically from ad_encode.m4.
2915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2916 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2917 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2918 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2919 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2920 %}
2921
2922 // This encoding class is generated automatically from ad_encode.m4.
2923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2924 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2925 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2926 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2927 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2928 %}
2929
2930 // This encoding class is generated automatically from ad_encode.m4.
2931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2932 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2933 Register src_reg = as_Register($src$$reg);
2934 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2935 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2936 %}
2937
2938 // This encoding class is generated automatically from ad_encode.m4.
2939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2940 enc_class aarch64_enc_strb0(memory1 mem) %{
2941 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2942 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2943 %}
2944
2945 // This encoding class is generated automatically from ad_encode.m4.
2946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2947 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2948 Register src_reg = as_Register($src$$reg);
2949 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2950 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2951 %}
2952
2953 // This encoding class is generated automatically from ad_encode.m4.
2954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2955 enc_class aarch64_enc_strh0(memory2 mem) %{
2956 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2957 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2958 %}
2959
2960 // This encoding class is generated automatically from ad_encode.m4.
2961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2962 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2963 Register src_reg = as_Register($src$$reg);
2964 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
2965 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2966 %}
2967
2968 // This encoding class is generated automatically from ad_encode.m4.
2969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2970 enc_class aarch64_enc_strw0(memory4 mem) %{
2971 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
2972 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2973 %}
2974
2975 // This encoding class is generated automatically from ad_encode.m4.
2976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2977 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2978 Register src_reg = as_Register($src$$reg);
2979 // we sometimes get asked to store the stack pointer into the
2980 // current thread -- we cannot do that directly on AArch64
2981 if (src_reg == r31_sp) {
2982 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2983 __ mov(rscratch2, sp);
2984 src_reg = rscratch2;
2985 }
2986 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
2987 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2988 %}
2989
2990 // This encoding class is generated automatically from ad_encode.m4.
2991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2992 enc_class aarch64_enc_str0(memory8 mem) %{
2993 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
2994 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2995 %}
2996
2997 // This encoding class is generated automatically from ad_encode.m4.
2998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2999 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3000 FloatRegister src_reg = as_FloatRegister($src$$reg);
3001 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3002 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3003 %}
3004
3005 // This encoding class is generated automatically from ad_encode.m4.
3006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3007 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3008 FloatRegister src_reg = as_FloatRegister($src$$reg);
3009 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3010 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3011 %}
3012
3013 // This encoding class is generated automatically from ad_encode.m4.
3014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3015 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3016 __ membar(Assembler::StoreStore);
3017 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3018 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3019 %}
3020
3021 // END Non-volatile memory access
3022
3023 // Vector loads and stores
3024 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3025 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3026 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3027 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3028 %}
3029
3030 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3031 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3032 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3033 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3034 %}
3035
3036 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3037 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3038 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3039 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3040 %}
3041
3042 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3043 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3044 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3045 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3046 %}
3047
3048 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3049 FloatRegister src_reg = as_FloatRegister($src$$reg);
3050 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3051 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3052 %}
3053
3054 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3055 FloatRegister src_reg = as_FloatRegister($src$$reg);
3056 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3057 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3058 %}
3059
3060 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3061 FloatRegister src_reg = as_FloatRegister($src$$reg);
3062 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3063 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3064 %}
3065
3066 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3067 FloatRegister src_reg = as_FloatRegister($src$$reg);
3068 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3069 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3070 %}
3071
3072 // volatile loads and stores
3073
3074 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3075 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3076 rscratch1, stlrb);
3077 %}
3078
3079 enc_class aarch64_enc_stlrb0(memory mem) %{
3080 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3081 rscratch1, stlrb);
3082 %}
3083
3084 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3085 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3086 rscratch1, stlrh);
3087 %}
3088
3089 enc_class aarch64_enc_stlrh0(memory mem) %{
3090 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3091 rscratch1, stlrh);
3092 %}
3093
3094 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3095 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3096 rscratch1, stlrw);
3097 %}
3098
3099 enc_class aarch64_enc_stlrw0(memory mem) %{
3100 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3101 rscratch1, stlrw);
3102 %}
3103
3104 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3105 Register dst_reg = as_Register($dst$$reg);
3106 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3107 rscratch1, ldarb);
3108 __ sxtbw(dst_reg, dst_reg);
3109 %}
3110
3111 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3112 Register dst_reg = as_Register($dst$$reg);
3113 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3114 rscratch1, ldarb);
3115 __ sxtb(dst_reg, dst_reg);
3116 %}
3117
3118 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3119 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3120 rscratch1, ldarb);
3121 %}
3122
3123 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3124 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3125 rscratch1, ldarb);
3126 %}
3127
3128 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3129 Register dst_reg = as_Register($dst$$reg);
3130 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3131 rscratch1, ldarh);
3132 __ sxthw(dst_reg, dst_reg);
3133 %}
3134
3135 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3136 Register dst_reg = as_Register($dst$$reg);
3137 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3138 rscratch1, ldarh);
3139 __ sxth(dst_reg, dst_reg);
3140 %}
3141
3142 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3143 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3144 rscratch1, ldarh);
3145 %}
3146
3147 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3148 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3149 rscratch1, ldarh);
3150 %}
3151
3152 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3153 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3154 rscratch1, ldarw);
3155 %}
3156
3157 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3158 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3159 rscratch1, ldarw);
3160 %}
3161
3162 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3163 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3164 rscratch1, ldar);
3165 %}
3166
3167 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3168 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3169 rscratch1, ldarw);
3170 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3171 %}
3172
3173 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3174 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3175 rscratch1, ldar);
3176 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3177 %}
3178
3179 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3180 Register src_reg = as_Register($src$$reg);
3181 // we sometimes get asked to store the stack pointer into the
3182 // current thread -- we cannot do that directly on AArch64
3183 if (src_reg == r31_sp) {
3184 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3185 __ mov(rscratch2, sp);
3186 src_reg = rscratch2;
3187 }
3188 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3189 rscratch1, stlr);
3190 %}
3191
3192 enc_class aarch64_enc_stlr0(memory mem) %{
3193 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3194 rscratch1, stlr);
3195 %}
3196
3197 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3198 {
3199 FloatRegister src_reg = as_FloatRegister($src$$reg);
3200 __ fmovs(rscratch2, src_reg);
3201 }
3202 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3203 rscratch1, stlrw);
3204 %}
3205
3206 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3207 {
3208 FloatRegister src_reg = as_FloatRegister($src$$reg);
3209 __ fmovd(rscratch2, src_reg);
3210 }
3211 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3212 rscratch1, stlr);
3213 %}
3214
3215 // synchronized read/update encodings
3216
3217 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3218 Register dst_reg = as_Register($dst$$reg);
3219 Register base = as_Register($mem$$base);
3220 int index = $mem$$index;
3221 int scale = $mem$$scale;
3222 int disp = $mem$$disp;
3223 if (index == -1) {
3224 if (disp != 0) {
3225 __ lea(rscratch1, Address(base, disp));
3226 __ ldaxr(dst_reg, rscratch1);
3227 } else {
3228 // TODO
3229 // should we ever get anything other than this case?
3230 __ ldaxr(dst_reg, base);
3231 }
3232 } else {
3233 Register index_reg = as_Register(index);
3234 if (disp == 0) {
3235 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3236 __ ldaxr(dst_reg, rscratch1);
3237 } else {
3238 __ lea(rscratch1, Address(base, disp));
3239 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3240 __ ldaxr(dst_reg, rscratch1);
3241 }
3242 }
3243 %}
3244
3245 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3246 Register src_reg = as_Register($src$$reg);
3247 Register base = as_Register($mem$$base);
3248 int index = $mem$$index;
3249 int scale = $mem$$scale;
3250 int disp = $mem$$disp;
3251 if (index == -1) {
3252 if (disp != 0) {
3253 __ lea(rscratch2, Address(base, disp));
3254 __ stlxr(rscratch1, src_reg, rscratch2);
3255 } else {
3256 // TODO
3257 // should we ever get anything other than this case?
3258 __ stlxr(rscratch1, src_reg, base);
3259 }
3260 } else {
3261 Register index_reg = as_Register(index);
3262 if (disp == 0) {
3263 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3264 __ stlxr(rscratch1, src_reg, rscratch2);
3265 } else {
3266 __ lea(rscratch2, Address(base, disp));
3267 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3268 __ stlxr(rscratch1, src_reg, rscratch2);
3269 }
3270 }
3271 __ cmpw(rscratch1, zr);
3272 %}
3273
3274 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3275 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3276 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3277 Assembler::xword, /*acquire*/ false, /*release*/ true,
3278 /*weak*/ false, noreg);
3279 %}
3280
3281 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3282 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3283 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3284 Assembler::word, /*acquire*/ false, /*release*/ true,
3285 /*weak*/ false, noreg);
3286 %}
3287
3288 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3289 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3290 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3291 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3292 /*weak*/ false, noreg);
3293 %}
3294
3295 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3296 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3297 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3298 Assembler::byte, /*acquire*/ false, /*release*/ true,
3299 /*weak*/ false, noreg);
3300 %}
3301
3302
3303 // The only difference between aarch64_enc_cmpxchg and
3304 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3305 // CompareAndSwap sequence to serve as a barrier on acquiring a
3306 // lock.
3307 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3308 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3309 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3310 Assembler::xword, /*acquire*/ true, /*release*/ true,
3311 /*weak*/ false, noreg);
3312 %}
3313
3314 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3315 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3316 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3317 Assembler::word, /*acquire*/ true, /*release*/ true,
3318 /*weak*/ false, noreg);
3319 %}
3320
3321 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3322 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3323 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3324 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3325 /*weak*/ false, noreg);
3326 %}
3327
3328 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3329 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3330 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3331 Assembler::byte, /*acquire*/ true, /*release*/ true,
3332 /*weak*/ false, noreg);
3333 %}
3334
3335 // auxiliary used for CompareAndSwapX to set result register
3336 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3337 Register res_reg = as_Register($res$$reg);
3338 __ cset(res_reg, Assembler::EQ);
3339 %}
3340
3341 // prefetch encodings
3342
3343 enc_class aarch64_enc_prefetchw(memory mem) %{
3344 Register base = as_Register($mem$$base);
3345 int index = $mem$$index;
3346 int scale = $mem$$scale;
3347 int disp = $mem$$disp;
3348 if (index == -1) {
3349 // Fix up any out-of-range offsets.
3350 assert_different_registers(rscratch1, base);
3351 Address addr = Address(base, disp);
3352 addr = __ legitimize_address(addr, 8, rscratch1);
3353 __ prfm(addr, PSTL1KEEP);
3354 } else {
3355 Register index_reg = as_Register(index);
3356 if (disp == 0) {
3357 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3358 } else {
3359 __ lea(rscratch1, Address(base, disp));
3360 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3361 }
3362 }
3363 %}
3364
3365 // mov encodings
3366
3367 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3368 uint32_t con = (uint32_t)$src$$constant;
3369 Register dst_reg = as_Register($dst$$reg);
3370 if (con == 0) {
3371 __ movw(dst_reg, zr);
3372 } else {
3373 __ movw(dst_reg, con);
3374 }
3375 %}
3376
3377 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3378 Register dst_reg = as_Register($dst$$reg);
3379 uint64_t con = (uint64_t)$src$$constant;
3380 if (con == 0) {
3381 __ mov(dst_reg, zr);
3382 } else {
3383 __ mov(dst_reg, con);
3384 }
3385 %}
3386
3387 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3388 Register dst_reg = as_Register($dst$$reg);
3389 address con = (address)$src$$constant;
3390 if (con == nullptr || con == (address)1) {
3391 ShouldNotReachHere();
3392 } else {
3393 relocInfo::relocType rtype = $src->constant_reloc();
3394 if (rtype == relocInfo::oop_type) {
3395 __ movoop(dst_reg, (jobject)con);
3396 } else if (rtype == relocInfo::metadata_type) {
3397 __ mov_metadata(dst_reg, (Metadata*)con);
3398 } else {
3399 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3400 if (! __ is_valid_AArch64_address(con) ||
3401 con < (address)(uintptr_t)os::vm_page_size()) {
3402 __ mov(dst_reg, con);
3403 } else {
3404 uint64_t offset;
3405 __ adrp(dst_reg, con, offset);
3406 __ add(dst_reg, dst_reg, offset);
3407 }
3408 }
3409 }
3410 %}
3411
3412 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3413 Register dst_reg = as_Register($dst$$reg);
3414 __ mov(dst_reg, zr);
3415 %}
3416
3417 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3418 Register dst_reg = as_Register($dst$$reg);
3419 __ mov(dst_reg, (uint64_t)1);
3420 %}
3421
3422 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3423 Register dst_reg = as_Register($dst$$reg);
3424 address con = (address)$src$$constant;
3425 if (con == nullptr) {
3426 ShouldNotReachHere();
3427 } else {
3428 relocInfo::relocType rtype = $src->constant_reloc();
3429 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3430 __ set_narrow_oop(dst_reg, (jobject)con);
3431 }
3432 %}
3433
3434 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3435 Register dst_reg = as_Register($dst$$reg);
3436 __ mov(dst_reg, zr);
3437 %}
3438
3439 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3440 Register dst_reg = as_Register($dst$$reg);
3441 address con = (address)$src$$constant;
3442 if (con == nullptr) {
3443 ShouldNotReachHere();
3444 } else {
3445 relocInfo::relocType rtype = $src->constant_reloc();
3446 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3447 __ set_narrow_klass(dst_reg, (Klass *)con);
3448 }
3449 %}
3450
3451 // arithmetic encodings
3452
3453 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3454 Register dst_reg = as_Register($dst$$reg);
3455 Register src_reg = as_Register($src1$$reg);
3456 int32_t con = (int32_t)$src2$$constant;
3457 // add has primary == 0, subtract has primary == 1
3458 if ($primary) { con = -con; }
3459 if (con < 0) {
3460 __ subw(dst_reg, src_reg, -con);
3461 } else {
3462 __ addw(dst_reg, src_reg, con);
3463 }
3464 %}
3465
3466 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3467 Register dst_reg = as_Register($dst$$reg);
3468 Register src_reg = as_Register($src1$$reg);
3469 int32_t con = (int32_t)$src2$$constant;
3470 // add has primary == 0, subtract has primary == 1
3471 if ($primary) { con = -con; }
3472 if (con < 0) {
3473 __ sub(dst_reg, src_reg, -con);
3474 } else {
3475 __ add(dst_reg, src_reg, con);
3476 }
3477 %}
3478
3479 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3480 Register dst_reg = as_Register($dst$$reg);
3481 Register src1_reg = as_Register($src1$$reg);
3482 Register src2_reg = as_Register($src2$$reg);
3483 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3484 %}
3485
3486 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3487 Register dst_reg = as_Register($dst$$reg);
3488 Register src1_reg = as_Register($src1$$reg);
3489 Register src2_reg = as_Register($src2$$reg);
3490 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3491 %}
3492
3493 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3494 Register dst_reg = as_Register($dst$$reg);
3495 Register src1_reg = as_Register($src1$$reg);
3496 Register src2_reg = as_Register($src2$$reg);
3497 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3498 %}
3499
3500 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3501 Register dst_reg = as_Register($dst$$reg);
3502 Register src1_reg = as_Register($src1$$reg);
3503 Register src2_reg = as_Register($src2$$reg);
3504 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3505 %}
3506
3507 // compare instruction encodings
3508
3509 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3510 Register reg1 = as_Register($src1$$reg);
3511 Register reg2 = as_Register($src2$$reg);
3512 __ cmpw(reg1, reg2);
3513 %}
3514
3515 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3516 Register reg = as_Register($src1$$reg);
3517 int32_t val = $src2$$constant;
3518 if (val >= 0) {
3519 __ subsw(zr, reg, val);
3520 } else {
3521 __ addsw(zr, reg, -val);
3522 }
3523 %}
3524
3525 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3526 Register reg1 = as_Register($src1$$reg);
3527 uint32_t val = (uint32_t)$src2$$constant;
3528 __ movw(rscratch1, val);
3529 __ cmpw(reg1, rscratch1);
3530 %}
3531
3532 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3533 Register reg1 = as_Register($src1$$reg);
3534 Register reg2 = as_Register($src2$$reg);
3535 __ cmp(reg1, reg2);
3536 %}
3537
3538 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3539 Register reg = as_Register($src1$$reg);
3540 int64_t val = $src2$$constant;
3541 if (val >= 0) {
3542 __ subs(zr, reg, val);
3543 } else if (val != -val) {
3544 __ adds(zr, reg, -val);
3545 } else {
3546 // aargh, Long.MIN_VALUE is a special case
3547 __ orr(rscratch1, zr, (uint64_t)val);
3548 __ subs(zr, reg, rscratch1);
3549 }
3550 %}
3551
3552 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3553 Register reg1 = as_Register($src1$$reg);
3554 uint64_t val = (uint64_t)$src2$$constant;
3555 __ mov(rscratch1, val);
3556 __ cmp(reg1, rscratch1);
3557 %}
3558
3559 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3560 Register reg1 = as_Register($src1$$reg);
3561 Register reg2 = as_Register($src2$$reg);
3562 __ cmp(reg1, reg2);
3563 %}
3564
3565 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3566 Register reg1 = as_Register($src1$$reg);
3567 Register reg2 = as_Register($src2$$reg);
3568 __ cmpw(reg1, reg2);
3569 %}
3570
3571 enc_class aarch64_enc_testp(iRegP src) %{
3572 Register reg = as_Register($src$$reg);
3573 __ cmp(reg, zr);
3574 %}
3575
3576 enc_class aarch64_enc_testn(iRegN src) %{
3577 Register reg = as_Register($src$$reg);
3578 __ cmpw(reg, zr);
3579 %}
3580
3581 enc_class aarch64_enc_b(label lbl) %{
3582 Label *L = $lbl$$label;
3583 __ b(*L);
3584 %}
3585
3586 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3587 Label *L = $lbl$$label;
3588 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3589 %}
3590
3591 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3592 Label *L = $lbl$$label;
3593 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3594 %}
3595
3596 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3597 %{
3598 Register sub_reg = as_Register($sub$$reg);
3599 Register super_reg = as_Register($super$$reg);
3600 Register temp_reg = as_Register($temp$$reg);
3601 Register result_reg = as_Register($result$$reg);
3602
3603 Label miss;
3604 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3605 nullptr, &miss,
3606 /*set_cond_codes:*/ true);
3607 if ($primary) {
3608 __ mov(result_reg, zr);
3609 }
3610 __ bind(miss);
3611 %}
3612
3613 enc_class aarch64_enc_java_static_call(method meth) %{
3614 address addr = (address)$meth$$method;
3615 address call;
3616 if (!_method) {
3617 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3618 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3619 if (call == nullptr) {
3620 ciEnv::current()->record_failure("CodeCache is full");
3621 return;
3622 }
3623 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3624 // The NOP here is purely to ensure that eliding a call to
3625 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3626 __ nop();
3627 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3628 } else {
3629 int method_index = resolved_method_index(masm);
3630 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3631 : static_call_Relocation::spec(method_index);
3632 call = __ trampoline_call(Address(addr, rspec));
3633 if (call == nullptr) {
3634 ciEnv::current()->record_failure("CodeCache is full");
3635 return;
3636 }
3637 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3638 // Calls of the same statically bound method can share
3639 // a stub to the interpreter.
3640 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3641 } else {
3642 // Emit stub for static call
3643 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3644 if (stub == nullptr) {
3645 ciEnv::current()->record_failure("CodeCache is full");
3646 return;
3647 }
3648 }
3649 }
3650
3651 __ post_call_nop();
3652
3653 // Only non uncommon_trap calls need to reinitialize ptrue.
3654 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3655 __ reinitialize_ptrue();
3656 }
3657 %}
3658
3659 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3660 int method_index = resolved_method_index(masm);
3661 address call = __ ic_call((address)$meth$$method, method_index);
3662 if (call == nullptr) {
3663 ciEnv::current()->record_failure("CodeCache is full");
3664 return;
3665 }
3666 __ post_call_nop();
3667 if (Compile::current()->max_vector_size() > 0) {
3668 __ reinitialize_ptrue();
3669 }
3670 %}
3671
3672 enc_class aarch64_enc_call_epilog() %{
3673 if (VerifyStackAtCalls) {
3674 // Check that stack depth is unchanged: find majik cookie on stack
3675 __ call_Unimplemented();
3676 }
3677 %}
3678
3679 enc_class aarch64_enc_java_to_runtime(method meth) %{
3680 // some calls to generated routines (arraycopy code) are scheduled
3681 // by C2 as runtime calls. if so we can call them using a br (they
3682 // will be in a reachable segment) otherwise we have to use a blr
3683 // which loads the absolute address into a register.
3684 address entry = (address)$meth$$method;
3685 CodeBlob *cb = CodeCache::find_blob(entry);
3686 if (cb) {
3687 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3688 if (call == nullptr) {
3689 ciEnv::current()->record_failure("CodeCache is full");
3690 return;
3691 }
3692 __ post_call_nop();
3693 } else {
3694 Label retaddr;
3695 // Make the anchor frame walkable
3696 __ adr(rscratch2, retaddr);
3697 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3698 __ lea(rscratch1, RuntimeAddress(entry));
3699 __ blr(rscratch1);
3700 __ bind(retaddr);
3701 __ post_call_nop();
3702 }
3703 if (Compile::current()->max_vector_size() > 0) {
3704 __ reinitialize_ptrue();
3705 }
3706 %}
3707
3708 enc_class aarch64_enc_rethrow() %{
3709 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3710 %}
3711
3712 enc_class aarch64_enc_ret() %{
3713 #ifdef ASSERT
3714 if (Compile::current()->max_vector_size() > 0) {
3715 __ verify_ptrue();
3716 }
3717 #endif
3718 __ ret(lr);
3719 %}
3720
3721 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3722 Register target_reg = as_Register($jump_target$$reg);
3723 __ br(target_reg);
3724 %}
3725
3726 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3727 Register target_reg = as_Register($jump_target$$reg);
3728 // exception oop should be in r0
3729 // ret addr has been popped into lr
3730 // callee expects it in r3
3731 __ mov(r3, lr);
3732 __ br(target_reg);
3733 %}
3734
3735 %}
3736
3737 //----------FRAME--------------------------------------------------------------
3738 // Definition of frame structure and management information.
3739 //
3740 // S T A C K L A Y O U T Allocators stack-slot number
3741 // | (to get allocators register number
3742 // G Owned by | | v add OptoReg::stack0())
3743 // r CALLER | |
3744 // o | +--------+ pad to even-align allocators stack-slot
3745 // w V | pad0 | numbers; owned by CALLER
3746 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3747 // h ^ | in | 5
3748 // | | args | 4 Holes in incoming args owned by SELF
3749 // | | | | 3
3750 // | | +--------+
3751 // V | | old out| Empty on Intel, window on Sparc
3752 // | old |preserve| Must be even aligned.
3753 // | SP-+--------+----> Matcher::_old_SP, even aligned
3754 // | | in | 3 area for Intel ret address
3755 // Owned by |preserve| Empty on Sparc.
3756 // SELF +--------+
3757 // | | pad2 | 2 pad to align old SP
3758 // | +--------+ 1
3759 // | | locks | 0
3760 // | +--------+----> OptoReg::stack0(), even aligned
3761 // | | pad1 | 11 pad to align new SP
3762 // | +--------+
3763 // | | | 10
3764 // | | spills | 9 spills
3765 // V | | 8 (pad0 slot for callee)
3766 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3767 // ^ | out | 7
3768 // | | args | 6 Holes in outgoing args owned by CALLEE
3769 // Owned by +--------+
3770 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3771 // | new |preserve| Must be even-aligned.
3772 // | SP-+--------+----> Matcher::_new_SP, even aligned
3773 // | | |
3774 //
3775 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3776 // known from SELF's arguments and the Java calling convention.
3777 // Region 6-7 is determined per call site.
3778 // Note 2: If the calling convention leaves holes in the incoming argument
3779 // area, those holes are owned by SELF. Holes in the outgoing area
3780 // are owned by the CALLEE. Holes should not be necessary in the
3781 // incoming area, as the Java calling convention is completely under
3782 // the control of the AD file. Doubles can be sorted and packed to
3783 // avoid holes. Holes in the outgoing arguments may be necessary for
3784 // varargs C calling conventions.
3785 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3786 // even aligned with pad0 as needed.
3787 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3788 // (the latter is true on Intel but is it false on AArch64?)
3789 // region 6-11 is even aligned; it may be padded out more so that
3790 // the region from SP to FP meets the minimum stack alignment.
3791 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3792 // alignment. Region 11, pad1, may be dynamically extended so that
3793 // SP meets the minimum alignment.
3794
3795 frame %{
3796 // These three registers define part of the calling convention
3797 // between compiled code and the interpreter.
3798
3799 // Inline Cache Register or Method for I2C.
3800 inline_cache_reg(R12);
3801
3802 // Number of stack slots consumed by locking an object
3803 sync_stack_slots(2);
3804
3805 // Compiled code's Frame Pointer
3806 frame_pointer(R31);
3807
3808 // Interpreter stores its frame pointer in a register which is
3809 // stored to the stack by I2CAdaptors.
3810 // I2CAdaptors convert from interpreted java to compiled java.
3811 interpreter_frame_pointer(R29);
3812
3813 // Stack alignment requirement
3814 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3815
3816 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3817 // for calls to C. Supports the var-args backing area for register parms.
3818 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3819
3820 // The after-PROLOG location of the return address. Location of
3821 // return address specifies a type (REG or STACK) and a number
3822 // representing the register number (i.e. - use a register name) or
3823 // stack slot.
3824 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3825 // Otherwise, it is above the locks and verification slot and alignment word
3826 // TODO this may well be correct but need to check why that - 2 is there
3827 // ppc port uses 0 but we definitely need to allow for fixed_slots
3828 // which folds in the space used for monitors
3829 return_addr(STACK - 2 +
3830 align_up((Compile::current()->in_preserve_stack_slots() +
3831 Compile::current()->fixed_slots()),
3832 stack_alignment_in_slots()));
3833
3834 // Location of compiled Java return values. Same as C for now.
3835 return_value
3836 %{
3837 // TODO do we allow ideal_reg == Op_RegN???
3838 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3839 "only return normal values");
3840
3841 static const int lo[Op_RegL + 1] = { // enum name
3842 0, // Op_Node
3843 0, // Op_Set
3844 R0_num, // Op_RegN
3845 R0_num, // Op_RegI
3846 R0_num, // Op_RegP
3847 V0_num, // Op_RegF
3848 V0_num, // Op_RegD
3849 R0_num // Op_RegL
3850 };
3851
3852 static const int hi[Op_RegL + 1] = { // enum name
3853 0, // Op_Node
3854 0, // Op_Set
3855 OptoReg::Bad, // Op_RegN
3856 OptoReg::Bad, // Op_RegI
3857 R0_H_num, // Op_RegP
3858 OptoReg::Bad, // Op_RegF
3859 V0_H_num, // Op_RegD
3860 R0_H_num // Op_RegL
3861 };
3862
3863 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3864 %}
3865 %}
3866
3867 //----------ATTRIBUTES---------------------------------------------------------
3868 //----------Operand Attributes-------------------------------------------------
3869 op_attrib op_cost(1); // Required cost attribute
3870
3871 //----------Instruction Attributes---------------------------------------------
3872 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3873 ins_attrib ins_size(32); // Required size attribute (in bits)
3874 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3875 // a non-matching short branch variant
3876 // of some long branch?
3877 ins_attrib ins_alignment(4); // Required alignment attribute (must
3878 // be a power of 2) specifies the
3879 // alignment that some part of the
3880 // instruction (not necessarily the
3881 // start) requires. If > 1, a
3882 // compute_padding() function must be
3883 // provided for the instruction
3884
3885 //----------OPERANDS-----------------------------------------------------------
3886 // Operand definitions must precede instruction definitions for correct parsing
3887 // in the ADLC because operands constitute user defined types which are used in
3888 // instruction definitions.
3889
3890 //----------Simple Operands----------------------------------------------------
3891
3892 // Integer operands 32 bit
3893 // 32 bit immediate
3894 operand immI()
3895 %{
3896 match(ConI);
3897
3898 op_cost(0);
3899 format %{ %}
3900 interface(CONST_INTER);
3901 %}
3902
3903 // 32 bit zero
3904 operand immI0()
3905 %{
3906 predicate(n->get_int() == 0);
3907 match(ConI);
3908
3909 op_cost(0);
3910 format %{ %}
3911 interface(CONST_INTER);
3912 %}
3913
3914 // 32 bit unit increment
3915 operand immI_1()
3916 %{
3917 predicate(n->get_int() == 1);
3918 match(ConI);
3919
3920 op_cost(0);
3921 format %{ %}
3922 interface(CONST_INTER);
3923 %}
3924
3925 // 32 bit unit decrement
3926 operand immI_M1()
3927 %{
3928 predicate(n->get_int() == -1);
3929 match(ConI);
3930
3931 op_cost(0);
3932 format %{ %}
3933 interface(CONST_INTER);
3934 %}
3935
3936 // Shift values for add/sub extension shift
3937 operand immIExt()
3938 %{
3939 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3940 match(ConI);
3941
3942 op_cost(0);
3943 format %{ %}
3944 interface(CONST_INTER);
3945 %}
3946
3947 operand immI_gt_1()
3948 %{
3949 predicate(n->get_int() > 1);
3950 match(ConI);
3951
3952 op_cost(0);
3953 format %{ %}
3954 interface(CONST_INTER);
3955 %}
3956
3957 operand immI_le_4()
3958 %{
3959 predicate(n->get_int() <= 4);
3960 match(ConI);
3961
3962 op_cost(0);
3963 format %{ %}
3964 interface(CONST_INTER);
3965 %}
3966
3967 operand immI_16()
3968 %{
3969 predicate(n->get_int() == 16);
3970 match(ConI);
3971
3972 op_cost(0);
3973 format %{ %}
3974 interface(CONST_INTER);
3975 %}
3976
3977 operand immI_24()
3978 %{
3979 predicate(n->get_int() == 24);
3980 match(ConI);
3981
3982 op_cost(0);
3983 format %{ %}
3984 interface(CONST_INTER);
3985 %}
3986
3987 operand immI_32()
3988 %{
3989 predicate(n->get_int() == 32);
3990 match(ConI);
3991
3992 op_cost(0);
3993 format %{ %}
3994 interface(CONST_INTER);
3995 %}
3996
3997 operand immI_48()
3998 %{
3999 predicate(n->get_int() == 48);
4000 match(ConI);
4001
4002 op_cost(0);
4003 format %{ %}
4004 interface(CONST_INTER);
4005 %}
4006
4007 operand immI_56()
4008 %{
4009 predicate(n->get_int() == 56);
4010 match(ConI);
4011
4012 op_cost(0);
4013 format %{ %}
4014 interface(CONST_INTER);
4015 %}
4016
4017 operand immI_255()
4018 %{
4019 predicate(n->get_int() == 255);
4020 match(ConI);
4021
4022 op_cost(0);
4023 format %{ %}
4024 interface(CONST_INTER);
4025 %}
4026
4027 operand immI_65535()
4028 %{
4029 predicate(n->get_int() == 65535);
4030 match(ConI);
4031
4032 op_cost(0);
4033 format %{ %}
4034 interface(CONST_INTER);
4035 %}
4036
4037 operand immI_positive()
4038 %{
4039 predicate(n->get_int() > 0);
4040 match(ConI);
4041
4042 op_cost(0);
4043 format %{ %}
4044 interface(CONST_INTER);
4045 %}
4046
4047 // BoolTest condition for signed compare
4048 operand immI_cmp_cond()
4049 %{
4050 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4051 match(ConI);
4052
4053 op_cost(0);
4054 format %{ %}
4055 interface(CONST_INTER);
4056 %}
4057
4058 // BoolTest condition for unsigned compare
4059 operand immI_cmpU_cond()
4060 %{
4061 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4062 match(ConI);
4063
4064 op_cost(0);
4065 format %{ %}
4066 interface(CONST_INTER);
4067 %}
4068
4069 operand immL_255()
4070 %{
4071 predicate(n->get_long() == 255L);
4072 match(ConL);
4073
4074 op_cost(0);
4075 format %{ %}
4076 interface(CONST_INTER);
4077 %}
4078
4079 operand immL_65535()
4080 %{
4081 predicate(n->get_long() == 65535L);
4082 match(ConL);
4083
4084 op_cost(0);
4085 format %{ %}
4086 interface(CONST_INTER);
4087 %}
4088
4089 operand immL_4294967295()
4090 %{
4091 predicate(n->get_long() == 4294967295L);
4092 match(ConL);
4093
4094 op_cost(0);
4095 format %{ %}
4096 interface(CONST_INTER);
4097 %}
4098
4099 operand immL_bitmask()
4100 %{
4101 predicate((n->get_long() != 0)
4102 && ((n->get_long() & 0xc000000000000000l) == 0)
4103 && is_power_of_2(n->get_long() + 1));
4104 match(ConL);
4105
4106 op_cost(0);
4107 format %{ %}
4108 interface(CONST_INTER);
4109 %}
4110
4111 operand immI_bitmask()
4112 %{
4113 predicate((n->get_int() != 0)
4114 && ((n->get_int() & 0xc0000000) == 0)
4115 && is_power_of_2(n->get_int() + 1));
4116 match(ConI);
4117
4118 op_cost(0);
4119 format %{ %}
4120 interface(CONST_INTER);
4121 %}
4122
4123 operand immL_positive_bitmaskI()
4124 %{
4125 predicate((n->get_long() != 0)
4126 && ((julong)n->get_long() < 0x80000000ULL)
4127 && is_power_of_2(n->get_long() + 1));
4128 match(ConL);
4129
4130 op_cost(0);
4131 format %{ %}
4132 interface(CONST_INTER);
4133 %}
4134
4135 // Scale values for scaled offset addressing modes (up to long but not quad)
4136 operand immIScale()
4137 %{
4138 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4139 match(ConI);
4140
4141 op_cost(0);
4142 format %{ %}
4143 interface(CONST_INTER);
4144 %}
4145
4146 // 5 bit signed integer
4147 operand immI5()
4148 %{
4149 predicate(Assembler::is_simm(n->get_int(), 5));
4150 match(ConI);
4151
4152 op_cost(0);
4153 format %{ %}
4154 interface(CONST_INTER);
4155 %}
4156
4157 // 7 bit unsigned integer
4158 operand immIU7()
4159 %{
4160 predicate(Assembler::is_uimm(n->get_int(), 7));
4161 match(ConI);
4162
4163 op_cost(0);
4164 format %{ %}
4165 interface(CONST_INTER);
4166 %}
4167
4168 // Offset for scaled or unscaled immediate loads and stores
4169 operand immIOffset()
4170 %{
4171 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4172 match(ConI);
4173
4174 op_cost(0);
4175 format %{ %}
4176 interface(CONST_INTER);
4177 %}
4178
4179 operand immIOffset1()
4180 %{
4181 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4182 match(ConI);
4183
4184 op_cost(0);
4185 format %{ %}
4186 interface(CONST_INTER);
4187 %}
4188
4189 operand immIOffset2()
4190 %{
4191 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4192 match(ConI);
4193
4194 op_cost(0);
4195 format %{ %}
4196 interface(CONST_INTER);
4197 %}
4198
4199 operand immIOffset4()
4200 %{
4201 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4202 match(ConI);
4203
4204 op_cost(0);
4205 format %{ %}
4206 interface(CONST_INTER);
4207 %}
4208
4209 operand immIOffset8()
4210 %{
4211 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4212 match(ConI);
4213
4214 op_cost(0);
4215 format %{ %}
4216 interface(CONST_INTER);
4217 %}
4218
4219 operand immIOffset16()
4220 %{
4221 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4222 match(ConI);
4223
4224 op_cost(0);
4225 format %{ %}
4226 interface(CONST_INTER);
4227 %}
4228
4229 operand immLOffset()
4230 %{
4231 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4232 match(ConL);
4233
4234 op_cost(0);
4235 format %{ %}
4236 interface(CONST_INTER);
4237 %}
4238
4239 operand immLoffset1()
4240 %{
4241 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4242 match(ConL);
4243
4244 op_cost(0);
4245 format %{ %}
4246 interface(CONST_INTER);
4247 %}
4248
4249 operand immLoffset2()
4250 %{
4251 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4252 match(ConL);
4253
4254 op_cost(0);
4255 format %{ %}
4256 interface(CONST_INTER);
4257 %}
4258
4259 operand immLoffset4()
4260 %{
4261 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4262 match(ConL);
4263
4264 op_cost(0);
4265 format %{ %}
4266 interface(CONST_INTER);
4267 %}
4268
4269 operand immLoffset8()
4270 %{
4271 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4272 match(ConL);
4273
4274 op_cost(0);
4275 format %{ %}
4276 interface(CONST_INTER);
4277 %}
4278
4279 operand immLoffset16()
4280 %{
4281 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4282 match(ConL);
4283
4284 op_cost(0);
4285 format %{ %}
4286 interface(CONST_INTER);
4287 %}
4288
4289 // 5 bit signed long integer
4290 operand immL5()
4291 %{
4292 predicate(Assembler::is_simm(n->get_long(), 5));
4293 match(ConL);
4294
4295 op_cost(0);
4296 format %{ %}
4297 interface(CONST_INTER);
4298 %}
4299
4300 // 7 bit unsigned long integer
4301 operand immLU7()
4302 %{
4303 predicate(Assembler::is_uimm(n->get_long(), 7));
4304 match(ConL);
4305
4306 op_cost(0);
4307 format %{ %}
4308 interface(CONST_INTER);
4309 %}
4310
4311 // 8 bit signed value.
4312 operand immI8()
4313 %{
4314 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4315 match(ConI);
4316
4317 op_cost(0);
4318 format %{ %}
4319 interface(CONST_INTER);
4320 %}
4321
4322 // 8 bit signed value (simm8), or #simm8 LSL 8.
4323 operand immI8_shift8()
4324 %{
4325 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4326 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4327 match(ConI);
4328
4329 op_cost(0);
4330 format %{ %}
4331 interface(CONST_INTER);
4332 %}
4333
4334 // 8 bit signed value (simm8), or #simm8 LSL 8.
4335 operand immL8_shift8()
4336 %{
4337 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4338 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4339 match(ConL);
4340
4341 op_cost(0);
4342 format %{ %}
4343 interface(CONST_INTER);
4344 %}
4345
4346 // 8 bit integer valid for vector add sub immediate
4347 operand immBAddSubV()
4348 %{
4349 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4350 match(ConI);
4351
4352 op_cost(0);
4353 format %{ %}
4354 interface(CONST_INTER);
4355 %}
4356
4357 // 32 bit integer valid for add sub immediate
4358 operand immIAddSub()
4359 %{
4360 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4361 match(ConI);
4362 op_cost(0);
4363 format %{ %}
4364 interface(CONST_INTER);
4365 %}
4366
4367 // 32 bit integer valid for vector add sub immediate
4368 operand immIAddSubV()
4369 %{
4370 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4371 match(ConI);
4372
4373 op_cost(0);
4374 format %{ %}
4375 interface(CONST_INTER);
4376 %}
4377
4378 // 32 bit unsigned integer valid for logical immediate
4379
4380 operand immBLog()
4381 %{
4382 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4383 match(ConI);
4384
4385 op_cost(0);
4386 format %{ %}
4387 interface(CONST_INTER);
4388 %}
4389
4390 operand immSLog()
4391 %{
4392 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4393 match(ConI);
4394
4395 op_cost(0);
4396 format %{ %}
4397 interface(CONST_INTER);
4398 %}
4399
4400 operand immILog()
4401 %{
4402 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4403 match(ConI);
4404
4405 op_cost(0);
4406 format %{ %}
4407 interface(CONST_INTER);
4408 %}
4409
4410 // Integer operands 64 bit
4411 // 64 bit immediate
4412 operand immL()
4413 %{
4414 match(ConL);
4415
4416 op_cost(0);
4417 format %{ %}
4418 interface(CONST_INTER);
4419 %}
4420
4421 // 64 bit zero
4422 operand immL0()
4423 %{
4424 predicate(n->get_long() == 0);
4425 match(ConL);
4426
4427 op_cost(0);
4428 format %{ %}
4429 interface(CONST_INTER);
4430 %}
4431
4432 // 64 bit unit decrement
4433 operand immL_M1()
4434 %{
4435 predicate(n->get_long() == -1);
4436 match(ConL);
4437
4438 op_cost(0);
4439 format %{ %}
4440 interface(CONST_INTER);
4441 %}
4442
4443 // 64 bit integer valid for add sub immediate
4444 operand immLAddSub()
4445 %{
4446 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4447 match(ConL);
4448 op_cost(0);
4449 format %{ %}
4450 interface(CONST_INTER);
4451 %}
4452
4453 // 64 bit integer valid for addv subv immediate
4454 operand immLAddSubV()
4455 %{
4456 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4457 match(ConL);
4458
4459 op_cost(0);
4460 format %{ %}
4461 interface(CONST_INTER);
4462 %}
4463
4464 // 64 bit integer valid for logical immediate
4465 operand immLLog()
4466 %{
4467 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4468 match(ConL);
4469 op_cost(0);
4470 format %{ %}
4471 interface(CONST_INTER);
4472 %}
4473
4474 // Long Immediate: low 32-bit mask
4475 operand immL_32bits()
4476 %{
4477 predicate(n->get_long() == 0xFFFFFFFFL);
4478 match(ConL);
4479 op_cost(0);
4480 format %{ %}
4481 interface(CONST_INTER);
4482 %}
4483
4484 // Pointer operands
4485 // Pointer Immediate
4486 operand immP()
4487 %{
4488 match(ConP);
4489
4490 op_cost(0);
4491 format %{ %}
4492 interface(CONST_INTER);
4493 %}
4494
4495 // nullptr Pointer Immediate
4496 operand immP0()
4497 %{
4498 predicate(n->get_ptr() == 0);
4499 match(ConP);
4500
4501 op_cost(0);
4502 format %{ %}
4503 interface(CONST_INTER);
4504 %}
4505
4506 // Pointer Immediate One
4507 // this is used in object initialization (initial object header)
4508 operand immP_1()
4509 %{
4510 predicate(n->get_ptr() == 1);
4511 match(ConP);
4512
4513 op_cost(0);
4514 format %{ %}
4515 interface(CONST_INTER);
4516 %}
4517
4518 // Card Table Byte Map Base
4519 operand immByteMapBase()
4520 %{
4521 // Get base of card map
4522 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4523 is_card_table_address((address)(n->get_ptr())));
4524 match(ConP);
4525
4526 op_cost(0);
4527 format %{ %}
4528 interface(CONST_INTER);
4529 %}
4530
4531 // AOT Runtime Constants Address
4532 operand immAOTRuntimeConstantsAddress()
4533 %{
4534 // Check if the address is in the range of AOT Runtime Constants
4535 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4536 match(ConP);
4537
4538 op_cost(0);
4539 format %{ %}
4540 interface(CONST_INTER);
4541 %}
4542
4543 // Float and Double operands
4544 // Double Immediate
4545 operand immD()
4546 %{
4547 match(ConD);
4548 op_cost(0);
4549 format %{ %}
4550 interface(CONST_INTER);
4551 %}
4552
4553 // Double Immediate: +0.0d
4554 operand immD0()
4555 %{
4556 predicate(jlong_cast(n->getd()) == 0);
4557 match(ConD);
4558
4559 op_cost(0);
4560 format %{ %}
4561 interface(CONST_INTER);
4562 %}
4563
4564 // constant 'double +0.0'.
4565 operand immDPacked()
4566 %{
4567 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4568 match(ConD);
4569 op_cost(0);
4570 format %{ %}
4571 interface(CONST_INTER);
4572 %}
4573
4574 // Float Immediate
4575 operand immF()
4576 %{
4577 match(ConF);
4578 op_cost(0);
4579 format %{ %}
4580 interface(CONST_INTER);
4581 %}
4582
4583 // Float Immediate: +0.0f.
4584 operand immF0()
4585 %{
4586 predicate(jint_cast(n->getf()) == 0);
4587 match(ConF);
4588
4589 op_cost(0);
4590 format %{ %}
4591 interface(CONST_INTER);
4592 %}
4593
4594 //
4595 operand immFPacked()
4596 %{
4597 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4598 match(ConF);
4599 op_cost(0);
4600 format %{ %}
4601 interface(CONST_INTER);
4602 %}
4603
4604 // Narrow pointer operands
4605 // Narrow Pointer Immediate
4606 operand immN()
4607 %{
4608 match(ConN);
4609
4610 op_cost(0);
4611 format %{ %}
4612 interface(CONST_INTER);
4613 %}
4614
4615 // Narrow nullptr Pointer Immediate
4616 operand immN0()
4617 %{
4618 predicate(n->get_narrowcon() == 0);
4619 match(ConN);
4620
4621 op_cost(0);
4622 format %{ %}
4623 interface(CONST_INTER);
4624 %}
4625
4626 operand immNKlass()
4627 %{
4628 match(ConNKlass);
4629
4630 op_cost(0);
4631 format %{ %}
4632 interface(CONST_INTER);
4633 %}
4634
4635 // Integer 32 bit Register Operands
4636 // Integer 32 bitRegister (excludes SP)
4637 operand iRegI()
4638 %{
4639 constraint(ALLOC_IN_RC(any_reg32));
4640 match(RegI);
4641 match(iRegINoSp);
4642 op_cost(0);
4643 format %{ %}
4644 interface(REG_INTER);
4645 %}
4646
4647 // Integer 32 bit Register not Special
4648 operand iRegINoSp()
4649 %{
4650 constraint(ALLOC_IN_RC(no_special_reg32));
4651 match(RegI);
4652 op_cost(0);
4653 format %{ %}
4654 interface(REG_INTER);
4655 %}
4656
4657 // Integer 64 bit Register Operands
4658 // Integer 64 bit Register (includes SP)
4659 operand iRegL()
4660 %{
4661 constraint(ALLOC_IN_RC(any_reg));
4662 match(RegL);
4663 match(iRegLNoSp);
4664 op_cost(0);
4665 format %{ %}
4666 interface(REG_INTER);
4667 %}
4668
4669 // Integer 64 bit Register not Special
4670 operand iRegLNoSp()
4671 %{
4672 constraint(ALLOC_IN_RC(no_special_reg));
4673 match(RegL);
4674 match(iRegL_R0);
4675 format %{ %}
4676 interface(REG_INTER);
4677 %}
4678
4679 // Pointer Register Operands
4680 // Pointer Register
4681 operand iRegP()
4682 %{
4683 constraint(ALLOC_IN_RC(ptr_reg));
4684 match(RegP);
4685 match(iRegPNoSp);
4686 match(iRegP_R0);
4687 //match(iRegP_R2);
4688 //match(iRegP_R4);
4689 match(iRegP_R5);
4690 match(thread_RegP);
4691 op_cost(0);
4692 format %{ %}
4693 interface(REG_INTER);
4694 %}
4695
4696 // Pointer 64 bit Register not Special
4697 operand iRegPNoSp()
4698 %{
4699 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4700 match(RegP);
4701 // match(iRegP);
4702 // match(iRegP_R0);
4703 // match(iRegP_R2);
4704 // match(iRegP_R4);
4705 // match(iRegP_R5);
4706 // match(thread_RegP);
4707 op_cost(0);
4708 format %{ %}
4709 interface(REG_INTER);
4710 %}
4711
4712 // This operand is not allowed to use rfp even if
4713 // rfp is not used to hold the frame pointer.
4714 operand iRegPNoSpNoRfp()
4715 %{
4716 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4717 match(RegP);
4718 match(iRegPNoSp);
4719 op_cost(0);
4720 format %{ %}
4721 interface(REG_INTER);
4722 %}
4723
4724 // Pointer 64 bit Register R0 only
4725 operand iRegP_R0()
4726 %{
4727 constraint(ALLOC_IN_RC(r0_reg));
4728 match(RegP);
4729 // match(iRegP);
4730 match(iRegPNoSp);
4731 op_cost(0);
4732 format %{ %}
4733 interface(REG_INTER);
4734 %}
4735
4736 // Pointer 64 bit Register R1 only
4737 operand iRegP_R1()
4738 %{
4739 constraint(ALLOC_IN_RC(r1_reg));
4740 match(RegP);
4741 // match(iRegP);
4742 match(iRegPNoSp);
4743 op_cost(0);
4744 format %{ %}
4745 interface(REG_INTER);
4746 %}
4747
4748 // Pointer 64 bit Register R2 only
4749 operand iRegP_R2()
4750 %{
4751 constraint(ALLOC_IN_RC(r2_reg));
4752 match(RegP);
4753 // match(iRegP);
4754 match(iRegPNoSp);
4755 op_cost(0);
4756 format %{ %}
4757 interface(REG_INTER);
4758 %}
4759
4760 // Pointer 64 bit Register R3 only
4761 operand iRegP_R3()
4762 %{
4763 constraint(ALLOC_IN_RC(r3_reg));
4764 match(RegP);
4765 // match(iRegP);
4766 match(iRegPNoSp);
4767 op_cost(0);
4768 format %{ %}
4769 interface(REG_INTER);
4770 %}
4771
4772 // Pointer 64 bit Register R4 only
4773 operand iRegP_R4()
4774 %{
4775 constraint(ALLOC_IN_RC(r4_reg));
4776 match(RegP);
4777 // match(iRegP);
4778 match(iRegPNoSp);
4779 op_cost(0);
4780 format %{ %}
4781 interface(REG_INTER);
4782 %}
4783
4784 // Pointer 64 bit Register R5 only
4785 operand iRegP_R5()
4786 %{
4787 constraint(ALLOC_IN_RC(r5_reg));
4788 match(RegP);
4789 // match(iRegP);
4790 match(iRegPNoSp);
4791 op_cost(0);
4792 format %{ %}
4793 interface(REG_INTER);
4794 %}
4795
4796 // Pointer 64 bit Register R10 only
4797 operand iRegP_R10()
4798 %{
4799 constraint(ALLOC_IN_RC(r10_reg));
4800 match(RegP);
4801 // match(iRegP);
4802 match(iRegPNoSp);
4803 op_cost(0);
4804 format %{ %}
4805 interface(REG_INTER);
4806 %}
4807
4808 // Long 64 bit Register R0 only
4809 operand iRegL_R0()
4810 %{
4811 constraint(ALLOC_IN_RC(r0_reg));
4812 match(RegL);
4813 match(iRegLNoSp);
4814 op_cost(0);
4815 format %{ %}
4816 interface(REG_INTER);
4817 %}
4818
4819 // Long 64 bit Register R11 only
4820 operand iRegL_R11()
4821 %{
4822 constraint(ALLOC_IN_RC(r11_reg));
4823 match(RegL);
4824 match(iRegLNoSp);
4825 op_cost(0);
4826 format %{ %}
4827 interface(REG_INTER);
4828 %}
4829
4830 // Register R0 only
4831 operand iRegI_R0()
4832 %{
4833 constraint(ALLOC_IN_RC(int_r0_reg));
4834 match(RegI);
4835 match(iRegINoSp);
4836 op_cost(0);
4837 format %{ %}
4838 interface(REG_INTER);
4839 %}
4840
4841 // Register R2 only
4842 operand iRegI_R2()
4843 %{
4844 constraint(ALLOC_IN_RC(int_r2_reg));
4845 match(RegI);
4846 match(iRegINoSp);
4847 op_cost(0);
4848 format %{ %}
4849 interface(REG_INTER);
4850 %}
4851
4852 // Register R3 only
4853 operand iRegI_R3()
4854 %{
4855 constraint(ALLOC_IN_RC(int_r3_reg));
4856 match(RegI);
4857 match(iRegINoSp);
4858 op_cost(0);
4859 format %{ %}
4860 interface(REG_INTER);
4861 %}
4862
4863
4864 // Register R4 only
4865 operand iRegI_R4()
4866 %{
4867 constraint(ALLOC_IN_RC(int_r4_reg));
4868 match(RegI);
4869 match(iRegINoSp);
4870 op_cost(0);
4871 format %{ %}
4872 interface(REG_INTER);
4873 %}
4874
4875
4876 // Pointer Register Operands
4877 // Narrow Pointer Register
4878 operand iRegN()
4879 %{
4880 constraint(ALLOC_IN_RC(any_reg32));
4881 match(RegN);
4882 match(iRegNNoSp);
4883 op_cost(0);
4884 format %{ %}
4885 interface(REG_INTER);
4886 %}
4887
4888 // Integer 64 bit Register not Special
4889 operand iRegNNoSp()
4890 %{
4891 constraint(ALLOC_IN_RC(no_special_reg32));
4892 match(RegN);
4893 op_cost(0);
4894 format %{ %}
4895 interface(REG_INTER);
4896 %}
4897
4898 // Float Register
4899 // Float register operands
4900 operand vRegF()
4901 %{
4902 constraint(ALLOC_IN_RC(float_reg));
4903 match(RegF);
4904
4905 op_cost(0);
4906 format %{ %}
4907 interface(REG_INTER);
4908 %}
4909
4910 // Double Register
4911 // Double register operands
4912 operand vRegD()
4913 %{
4914 constraint(ALLOC_IN_RC(double_reg));
4915 match(RegD);
4916
4917 op_cost(0);
4918 format %{ %}
4919 interface(REG_INTER);
4920 %}
4921
4922 // Generic vector class. This will be used for
4923 // all vector operands, including NEON and SVE.
4924 operand vReg()
4925 %{
4926 constraint(ALLOC_IN_RC(dynamic));
4927 match(VecA);
4928 match(VecD);
4929 match(VecX);
4930
4931 op_cost(0);
4932 format %{ %}
4933 interface(REG_INTER);
4934 %}
4935
4936 operand vecA()
4937 %{
4938 constraint(ALLOC_IN_RC(vectora_reg));
4939 match(VecA);
4940
4941 op_cost(0);
4942 format %{ %}
4943 interface(REG_INTER);
4944 %}
4945
4946 operand vecD()
4947 %{
4948 constraint(ALLOC_IN_RC(vectord_reg));
4949 match(VecD);
4950
4951 op_cost(0);
4952 format %{ %}
4953 interface(REG_INTER);
4954 %}
4955
4956 operand vecX()
4957 %{
4958 constraint(ALLOC_IN_RC(vectorx_reg));
4959 match(VecX);
4960
4961 op_cost(0);
4962 format %{ %}
4963 interface(REG_INTER);
4964 %}
4965
4966 operand vRegD_V0()
4967 %{
4968 constraint(ALLOC_IN_RC(v0_reg));
4969 match(RegD);
4970 op_cost(0);
4971 format %{ %}
4972 interface(REG_INTER);
4973 %}
4974
4975 operand vRegD_V1()
4976 %{
4977 constraint(ALLOC_IN_RC(v1_reg));
4978 match(RegD);
4979 op_cost(0);
4980 format %{ %}
4981 interface(REG_INTER);
4982 %}
4983
4984 operand vRegD_V2()
4985 %{
4986 constraint(ALLOC_IN_RC(v2_reg));
4987 match(RegD);
4988 op_cost(0);
4989 format %{ %}
4990 interface(REG_INTER);
4991 %}
4992
4993 operand vRegD_V3()
4994 %{
4995 constraint(ALLOC_IN_RC(v3_reg));
4996 match(RegD);
4997 op_cost(0);
4998 format %{ %}
4999 interface(REG_INTER);
5000 %}
5001
5002 operand vRegD_V4()
5003 %{
5004 constraint(ALLOC_IN_RC(v4_reg));
5005 match(RegD);
5006 op_cost(0);
5007 format %{ %}
5008 interface(REG_INTER);
5009 %}
5010
5011 operand vRegD_V5()
5012 %{
5013 constraint(ALLOC_IN_RC(v5_reg));
5014 match(RegD);
5015 op_cost(0);
5016 format %{ %}
5017 interface(REG_INTER);
5018 %}
5019
5020 operand vRegD_V6()
5021 %{
5022 constraint(ALLOC_IN_RC(v6_reg));
5023 match(RegD);
5024 op_cost(0);
5025 format %{ %}
5026 interface(REG_INTER);
5027 %}
5028
5029 operand vRegD_V7()
5030 %{
5031 constraint(ALLOC_IN_RC(v7_reg));
5032 match(RegD);
5033 op_cost(0);
5034 format %{ %}
5035 interface(REG_INTER);
5036 %}
5037
5038 operand vRegD_V12()
5039 %{
5040 constraint(ALLOC_IN_RC(v12_reg));
5041 match(RegD);
5042 op_cost(0);
5043 format %{ %}
5044 interface(REG_INTER);
5045 %}
5046
5047 operand vRegD_V13()
5048 %{
5049 constraint(ALLOC_IN_RC(v13_reg));
5050 match(RegD);
5051 op_cost(0);
5052 format %{ %}
5053 interface(REG_INTER);
5054 %}
5055
5056 operand pReg()
5057 %{
5058 constraint(ALLOC_IN_RC(pr_reg));
5059 match(RegVectMask);
5060 match(pRegGov);
5061 op_cost(0);
5062 format %{ %}
5063 interface(REG_INTER);
5064 %}
5065
5066 operand pRegGov()
5067 %{
5068 constraint(ALLOC_IN_RC(gov_pr));
5069 match(RegVectMask);
5070 match(pReg);
5071 op_cost(0);
5072 format %{ %}
5073 interface(REG_INTER);
5074 %}
5075
5076 operand pRegGov_P0()
5077 %{
5078 constraint(ALLOC_IN_RC(p0_reg));
5079 match(RegVectMask);
5080 op_cost(0);
5081 format %{ %}
5082 interface(REG_INTER);
5083 %}
5084
5085 operand pRegGov_P1()
5086 %{
5087 constraint(ALLOC_IN_RC(p1_reg));
5088 match(RegVectMask);
5089 op_cost(0);
5090 format %{ %}
5091 interface(REG_INTER);
5092 %}
5093
5094 // Flags register, used as output of signed compare instructions
5095
5096 // note that on AArch64 we also use this register as the output for
5097 // for floating point compare instructions (CmpF CmpD). this ensures
5098 // that ordered inequality tests use GT, GE, LT or LE none of which
5099 // pass through cases where the result is unordered i.e. one or both
5100 // inputs to the compare is a NaN. this means that the ideal code can
5101 // replace e.g. a GT with an LE and not end up capturing the NaN case
5102 // (where the comparison should always fail). EQ and NE tests are
5103 // always generated in ideal code so that unordered folds into the NE
5104 // case, matching the behaviour of AArch64 NE.
5105 //
5106 // This differs from x86 where the outputs of FP compares use a
5107 // special FP flags registers and where compares based on this
5108 // register are distinguished into ordered inequalities (cmpOpUCF) and
5109 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5110 // to explicitly handle the unordered case in branches. x86 also has
5111 // to include extra CMoveX rules to accept a cmpOpUCF input.
5112
5113 operand rFlagsReg()
5114 %{
5115 constraint(ALLOC_IN_RC(int_flags));
5116 match(RegFlags);
5117
5118 op_cost(0);
5119 format %{ "RFLAGS" %}
5120 interface(REG_INTER);
5121 %}
5122
5123 // Flags register, used as output of unsigned compare instructions
5124 operand rFlagsRegU()
5125 %{
5126 constraint(ALLOC_IN_RC(int_flags));
5127 match(RegFlags);
5128
5129 op_cost(0);
5130 format %{ "RFLAGSU" %}
5131 interface(REG_INTER);
5132 %}
5133
5134 // Special Registers
5135
5136 // Method Register
5137 operand inline_cache_RegP(iRegP reg)
5138 %{
5139 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5140 match(reg);
5141 match(iRegPNoSp);
5142 op_cost(0);
5143 format %{ %}
5144 interface(REG_INTER);
5145 %}
5146
5147 // Thread Register
5148 operand thread_RegP(iRegP reg)
5149 %{
5150 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5151 match(reg);
5152 op_cost(0);
5153 format %{ %}
5154 interface(REG_INTER);
5155 %}
5156
5157 //----------Memory Operands----------------------------------------------------
5158
5159 operand indirect(iRegP reg)
5160 %{
5161 constraint(ALLOC_IN_RC(ptr_reg));
5162 match(reg);
5163 op_cost(0);
5164 format %{ "[$reg]" %}
5165 interface(MEMORY_INTER) %{
5166 base($reg);
5167 index(0xffffffff);
5168 scale(0x0);
5169 disp(0x0);
5170 %}
5171 %}
5172
5173 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5174 %{
5175 constraint(ALLOC_IN_RC(ptr_reg));
5176 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5177 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5178 op_cost(0);
5179 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5180 interface(MEMORY_INTER) %{
5181 base($reg);
5182 index($ireg);
5183 scale($scale);
5184 disp(0x0);
5185 %}
5186 %}
5187
5188 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5189 %{
5190 constraint(ALLOC_IN_RC(ptr_reg));
5191 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5192 match(AddP reg (LShiftL lreg scale));
5193 op_cost(0);
5194 format %{ "$reg, $lreg lsl($scale)" %}
5195 interface(MEMORY_INTER) %{
5196 base($reg);
5197 index($lreg);
5198 scale($scale);
5199 disp(0x0);
5200 %}
5201 %}
5202
5203 operand indIndexI2L(iRegP reg, iRegI ireg)
5204 %{
5205 constraint(ALLOC_IN_RC(ptr_reg));
5206 match(AddP reg (ConvI2L ireg));
5207 op_cost(0);
5208 format %{ "$reg, $ireg, 0, I2L" %}
5209 interface(MEMORY_INTER) %{
5210 base($reg);
5211 index($ireg);
5212 scale(0x0);
5213 disp(0x0);
5214 %}
5215 %}
5216
5217 operand indIndex(iRegP reg, iRegL lreg)
5218 %{
5219 constraint(ALLOC_IN_RC(ptr_reg));
5220 match(AddP reg lreg);
5221 op_cost(0);
5222 format %{ "$reg, $lreg" %}
5223 interface(MEMORY_INTER) %{
5224 base($reg);
5225 index($lreg);
5226 scale(0x0);
5227 disp(0x0);
5228 %}
5229 %}
5230
5231 operand indOffI1(iRegP reg, immIOffset1 off)
5232 %{
5233 constraint(ALLOC_IN_RC(ptr_reg));
5234 match(AddP reg off);
5235 op_cost(0);
5236 format %{ "[$reg, $off]" %}
5237 interface(MEMORY_INTER) %{
5238 base($reg);
5239 index(0xffffffff);
5240 scale(0x0);
5241 disp($off);
5242 %}
5243 %}
5244
5245 operand indOffI2(iRegP reg, immIOffset2 off)
5246 %{
5247 constraint(ALLOC_IN_RC(ptr_reg));
5248 match(AddP reg off);
5249 op_cost(0);
5250 format %{ "[$reg, $off]" %}
5251 interface(MEMORY_INTER) %{
5252 base($reg);
5253 index(0xffffffff);
5254 scale(0x0);
5255 disp($off);
5256 %}
5257 %}
5258
5259 operand indOffI4(iRegP reg, immIOffset4 off)
5260 %{
5261 constraint(ALLOC_IN_RC(ptr_reg));
5262 match(AddP reg off);
5263 op_cost(0);
5264 format %{ "[$reg, $off]" %}
5265 interface(MEMORY_INTER) %{
5266 base($reg);
5267 index(0xffffffff);
5268 scale(0x0);
5269 disp($off);
5270 %}
5271 %}
5272
5273 operand indOffI8(iRegP reg, immIOffset8 off)
5274 %{
5275 constraint(ALLOC_IN_RC(ptr_reg));
5276 match(AddP reg off);
5277 op_cost(0);
5278 format %{ "[$reg, $off]" %}
5279 interface(MEMORY_INTER) %{
5280 base($reg);
5281 index(0xffffffff);
5282 scale(0x0);
5283 disp($off);
5284 %}
5285 %}
5286
5287 operand indOffI16(iRegP reg, immIOffset16 off)
5288 %{
5289 constraint(ALLOC_IN_RC(ptr_reg));
5290 match(AddP reg off);
5291 op_cost(0);
5292 format %{ "[$reg, $off]" %}
5293 interface(MEMORY_INTER) %{
5294 base($reg);
5295 index(0xffffffff);
5296 scale(0x0);
5297 disp($off);
5298 %}
5299 %}
5300
5301 operand indOffL1(iRegP reg, immLoffset1 off)
5302 %{
5303 constraint(ALLOC_IN_RC(ptr_reg));
5304 match(AddP reg off);
5305 op_cost(0);
5306 format %{ "[$reg, $off]" %}
5307 interface(MEMORY_INTER) %{
5308 base($reg);
5309 index(0xffffffff);
5310 scale(0x0);
5311 disp($off);
5312 %}
5313 %}
5314
5315 operand indOffL2(iRegP reg, immLoffset2 off)
5316 %{
5317 constraint(ALLOC_IN_RC(ptr_reg));
5318 match(AddP reg off);
5319 op_cost(0);
5320 format %{ "[$reg, $off]" %}
5321 interface(MEMORY_INTER) %{
5322 base($reg);
5323 index(0xffffffff);
5324 scale(0x0);
5325 disp($off);
5326 %}
5327 %}
5328
5329 operand indOffL4(iRegP reg, immLoffset4 off)
5330 %{
5331 constraint(ALLOC_IN_RC(ptr_reg));
5332 match(AddP reg off);
5333 op_cost(0);
5334 format %{ "[$reg, $off]" %}
5335 interface(MEMORY_INTER) %{
5336 base($reg);
5337 index(0xffffffff);
5338 scale(0x0);
5339 disp($off);
5340 %}
5341 %}
5342
5343 operand indOffL8(iRegP reg, immLoffset8 off)
5344 %{
5345 constraint(ALLOC_IN_RC(ptr_reg));
5346 match(AddP reg off);
5347 op_cost(0);
5348 format %{ "[$reg, $off]" %}
5349 interface(MEMORY_INTER) %{
5350 base($reg);
5351 index(0xffffffff);
5352 scale(0x0);
5353 disp($off);
5354 %}
5355 %}
5356
5357 operand indOffL16(iRegP reg, immLoffset16 off)
5358 %{
5359 constraint(ALLOC_IN_RC(ptr_reg));
5360 match(AddP reg off);
5361 op_cost(0);
5362 format %{ "[$reg, $off]" %}
5363 interface(MEMORY_INTER) %{
5364 base($reg);
5365 index(0xffffffff);
5366 scale(0x0);
5367 disp($off);
5368 %}
5369 %}
5370
5371 operand indirectX2P(iRegL reg)
5372 %{
5373 constraint(ALLOC_IN_RC(ptr_reg));
5374 match(CastX2P reg);
5375 op_cost(0);
5376 format %{ "[$reg]\t# long -> ptr" %}
5377 interface(MEMORY_INTER) %{
5378 base($reg);
5379 index(0xffffffff);
5380 scale(0x0);
5381 disp(0x0);
5382 %}
5383 %}
5384
5385 operand indOffX2P(iRegL reg, immLOffset off)
5386 %{
5387 constraint(ALLOC_IN_RC(ptr_reg));
5388 match(AddP (CastX2P reg) off);
5389 op_cost(0);
5390 format %{ "[$reg, $off]\t# long -> ptr" %}
5391 interface(MEMORY_INTER) %{
5392 base($reg);
5393 index(0xffffffff);
5394 scale(0x0);
5395 disp($off);
5396 %}
5397 %}
5398
5399 operand indirectN(iRegN reg)
5400 %{
5401 predicate(CompressedOops::shift() == 0);
5402 constraint(ALLOC_IN_RC(ptr_reg));
5403 match(DecodeN reg);
5404 op_cost(0);
5405 format %{ "[$reg]\t# narrow" %}
5406 interface(MEMORY_INTER) %{
5407 base($reg);
5408 index(0xffffffff);
5409 scale(0x0);
5410 disp(0x0);
5411 %}
5412 %}
5413
5414 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5415 %{
5416 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5417 constraint(ALLOC_IN_RC(ptr_reg));
5418 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5419 op_cost(0);
5420 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5421 interface(MEMORY_INTER) %{
5422 base($reg);
5423 index($ireg);
5424 scale($scale);
5425 disp(0x0);
5426 %}
5427 %}
5428
5429 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5430 %{
5431 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5432 constraint(ALLOC_IN_RC(ptr_reg));
5433 match(AddP (DecodeN reg) (LShiftL lreg scale));
5434 op_cost(0);
5435 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5436 interface(MEMORY_INTER) %{
5437 base($reg);
5438 index($lreg);
5439 scale($scale);
5440 disp(0x0);
5441 %}
5442 %}
5443
5444 operand indIndexI2LN(iRegN reg, iRegI ireg)
5445 %{
5446 predicate(CompressedOops::shift() == 0);
5447 constraint(ALLOC_IN_RC(ptr_reg));
5448 match(AddP (DecodeN reg) (ConvI2L ireg));
5449 op_cost(0);
5450 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5451 interface(MEMORY_INTER) %{
5452 base($reg);
5453 index($ireg);
5454 scale(0x0);
5455 disp(0x0);
5456 %}
5457 %}
5458
5459 operand indIndexN(iRegN reg, iRegL lreg)
5460 %{
5461 predicate(CompressedOops::shift() == 0);
5462 constraint(ALLOC_IN_RC(ptr_reg));
5463 match(AddP (DecodeN reg) lreg);
5464 op_cost(0);
5465 format %{ "$reg, $lreg\t# narrow" %}
5466 interface(MEMORY_INTER) %{
5467 base($reg);
5468 index($lreg);
5469 scale(0x0);
5470 disp(0x0);
5471 %}
5472 %}
5473
5474 operand indOffIN(iRegN reg, immIOffset off)
5475 %{
5476 predicate(CompressedOops::shift() == 0);
5477 constraint(ALLOC_IN_RC(ptr_reg));
5478 match(AddP (DecodeN reg) off);
5479 op_cost(0);
5480 format %{ "[$reg, $off]\t# narrow" %}
5481 interface(MEMORY_INTER) %{
5482 base($reg);
5483 index(0xffffffff);
5484 scale(0x0);
5485 disp($off);
5486 %}
5487 %}
5488
5489 operand indOffLN(iRegN reg, immLOffset off)
5490 %{
5491 predicate(CompressedOops::shift() == 0);
5492 constraint(ALLOC_IN_RC(ptr_reg));
5493 match(AddP (DecodeN reg) off);
5494 op_cost(0);
5495 format %{ "[$reg, $off]\t# narrow" %}
5496 interface(MEMORY_INTER) %{
5497 base($reg);
5498 index(0xffffffff);
5499 scale(0x0);
5500 disp($off);
5501 %}
5502 %}
5503
5504
5505 //----------Special Memory Operands--------------------------------------------
5506 // Stack Slot Operand - This operand is used for loading and storing temporary
5507 // values on the stack where a match requires a value to
5508 // flow through memory.
5509 operand stackSlotP(sRegP reg)
5510 %{
5511 constraint(ALLOC_IN_RC(stack_slots));
5512 op_cost(100);
5513 // No match rule because this operand is only generated in matching
5514 // match(RegP);
5515 format %{ "[$reg]" %}
5516 interface(MEMORY_INTER) %{
5517 base(0x1e); // RSP
5518 index(0x0); // No Index
5519 scale(0x0); // No Scale
5520 disp($reg); // Stack Offset
5521 %}
5522 %}
5523
5524 operand stackSlotI(sRegI reg)
5525 %{
5526 constraint(ALLOC_IN_RC(stack_slots));
5527 // No match rule because this operand is only generated in matching
5528 // match(RegI);
5529 format %{ "[$reg]" %}
5530 interface(MEMORY_INTER) %{
5531 base(0x1e); // RSP
5532 index(0x0); // No Index
5533 scale(0x0); // No Scale
5534 disp($reg); // Stack Offset
5535 %}
5536 %}
5537
5538 operand stackSlotF(sRegF reg)
5539 %{
5540 constraint(ALLOC_IN_RC(stack_slots));
5541 // No match rule because this operand is only generated in matching
5542 // match(RegF);
5543 format %{ "[$reg]" %}
5544 interface(MEMORY_INTER) %{
5545 base(0x1e); // RSP
5546 index(0x0); // No Index
5547 scale(0x0); // No Scale
5548 disp($reg); // Stack Offset
5549 %}
5550 %}
5551
5552 operand stackSlotD(sRegD reg)
5553 %{
5554 constraint(ALLOC_IN_RC(stack_slots));
5555 // No match rule because this operand is only generated in matching
5556 // match(RegD);
5557 format %{ "[$reg]" %}
5558 interface(MEMORY_INTER) %{
5559 base(0x1e); // RSP
5560 index(0x0); // No Index
5561 scale(0x0); // No Scale
5562 disp($reg); // Stack Offset
5563 %}
5564 %}
5565
5566 operand stackSlotL(sRegL reg)
5567 %{
5568 constraint(ALLOC_IN_RC(stack_slots));
5569 // No match rule because this operand is only generated in matching
5570 // match(RegL);
5571 format %{ "[$reg]" %}
5572 interface(MEMORY_INTER) %{
5573 base(0x1e); // RSP
5574 index(0x0); // No Index
5575 scale(0x0); // No Scale
5576 disp($reg); // Stack Offset
5577 %}
5578 %}
5579
5580 // Operands for expressing Control Flow
5581 // NOTE: Label is a predefined operand which should not be redefined in
5582 // the AD file. It is generically handled within the ADLC.
5583
5584 //----------Conditional Branch Operands----------------------------------------
5585 // Comparison Op - This is the operation of the comparison, and is limited to
5586 // the following set of codes:
5587 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5588 //
5589 // Other attributes of the comparison, such as unsignedness, are specified
5590 // by the comparison instruction that sets a condition code flags register.
5591 // That result is represented by a flags operand whose subtype is appropriate
5592 // to the unsignedness (etc.) of the comparison.
5593 //
5594 // Later, the instruction which matches both the Comparison Op (a Bool) and
5595 // the flags (produced by the Cmp) specifies the coding of the comparison op
5596 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5597
5598 // used for signed integral comparisons and fp comparisons
5599
5600 operand cmpOp()
5601 %{
5602 match(Bool);
5603
5604 format %{ "" %}
5605 interface(COND_INTER) %{
5606 equal(0x0, "eq");
5607 not_equal(0x1, "ne");
5608 less(0xb, "lt");
5609 greater_equal(0xa, "ge");
5610 less_equal(0xd, "le");
5611 greater(0xc, "gt");
5612 overflow(0x6, "vs");
5613 no_overflow(0x7, "vc");
5614 %}
5615 %}
5616
5617 // used for unsigned integral comparisons
5618
5619 operand cmpOpU()
5620 %{
5621 match(Bool);
5622
5623 format %{ "" %}
5624 interface(COND_INTER) %{
5625 equal(0x0, "eq");
5626 not_equal(0x1, "ne");
5627 less(0x3, "lo");
5628 greater_equal(0x2, "hs");
5629 less_equal(0x9, "ls");
5630 greater(0x8, "hi");
5631 overflow(0x6, "vs");
5632 no_overflow(0x7, "vc");
5633 %}
5634 %}
5635
5636 // used for certain integral comparisons which can be
5637 // converted to cbxx or tbxx instructions
5638
5639 operand cmpOpEqNe()
5640 %{
5641 match(Bool);
5642 op_cost(0);
5643 predicate(n->as_Bool()->_test._test == BoolTest::ne
5644 || n->as_Bool()->_test._test == BoolTest::eq);
5645
5646 format %{ "" %}
5647 interface(COND_INTER) %{
5648 equal(0x0, "eq");
5649 not_equal(0x1, "ne");
5650 less(0xb, "lt");
5651 greater_equal(0xa, "ge");
5652 less_equal(0xd, "le");
5653 greater(0xc, "gt");
5654 overflow(0x6, "vs");
5655 no_overflow(0x7, "vc");
5656 %}
5657 %}
5658
5659 // used for certain integral comparisons which can be
5660 // converted to cbxx or tbxx instructions
5661
5662 operand cmpOpLtGe()
5663 %{
5664 match(Bool);
5665 op_cost(0);
5666
5667 predicate(n->as_Bool()->_test._test == BoolTest::lt
5668 || n->as_Bool()->_test._test == BoolTest::ge);
5669
5670 format %{ "" %}
5671 interface(COND_INTER) %{
5672 equal(0x0, "eq");
5673 not_equal(0x1, "ne");
5674 less(0xb, "lt");
5675 greater_equal(0xa, "ge");
5676 less_equal(0xd, "le");
5677 greater(0xc, "gt");
5678 overflow(0x6, "vs");
5679 no_overflow(0x7, "vc");
5680 %}
5681 %}
5682
5683 // used for certain unsigned integral comparisons which can be
5684 // converted to cbxx or tbxx instructions
5685
5686 operand cmpOpUEqNeLeGt()
5687 %{
5688 match(Bool);
5689 op_cost(0);
5690
5691 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5692 n->as_Bool()->_test._test == BoolTest::ne ||
5693 n->as_Bool()->_test._test == BoolTest::le ||
5694 n->as_Bool()->_test._test == BoolTest::gt);
5695
5696 format %{ "" %}
5697 interface(COND_INTER) %{
5698 equal(0x0, "eq");
5699 not_equal(0x1, "ne");
5700 less(0x3, "lo");
5701 greater_equal(0x2, "hs");
5702 less_equal(0x9, "ls");
5703 greater(0x8, "hi");
5704 overflow(0x6, "vs");
5705 no_overflow(0x7, "vc");
5706 %}
5707 %}
5708
5709 // Special operand allowing long args to int ops to be truncated for free
5710
5711 operand iRegL2I(iRegL reg) %{
5712
5713 op_cost(0);
5714
5715 match(ConvL2I reg);
5716
5717 format %{ "l2i($reg)" %}
5718
5719 interface(REG_INTER)
5720 %}
5721
5722 operand iRegL2P(iRegL reg) %{
5723
5724 op_cost(0);
5725
5726 match(CastX2P reg);
5727
5728 format %{ "l2p($reg)" %}
5729
5730 interface(REG_INTER)
5731 %}
5732
5733 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5734 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5735 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5736 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5737
5738 //----------OPERAND CLASSES----------------------------------------------------
5739 // Operand Classes are groups of operands that are used as to simplify
5740 // instruction definitions by not requiring the AD writer to specify
5741 // separate instructions for every form of operand when the
5742 // instruction accepts multiple operand types with the same basic
5743 // encoding and format. The classic case of this is memory operands.
5744
5745 // memory is used to define read/write location for load/store
5746 // instruction defs. we can turn a memory op into an Address
5747
5748 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5749 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5750
5751 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5752 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5753
5754 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5755 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5756
5757 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5758 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5759
5760 // All of the memory operands. For the pipeline description.
5761 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5762 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5763 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5764
5765
5766 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5767 // operations. it allows the src to be either an iRegI or a (ConvL2I
5768 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5769 // can be elided because the 32-bit instruction will just employ the
5770 // lower 32 bits anyway.
5771 //
5772 // n.b. this does not elide all L2I conversions. if the truncated
5773 // value is consumed by more than one operation then the ConvL2I
5774 // cannot be bundled into the consuming nodes so an l2i gets planted
5775 // (actually a movw $dst $src) and the downstream instructions consume
5776 // the result of the l2i as an iRegI input. That's a shame since the
5777 // movw is actually redundant but its not too costly.
5778
5779 opclass iRegIorL2I(iRegI, iRegL2I);
5780 opclass iRegPorL2P(iRegP, iRegL2P);
5781
5782 //----------PIPELINE-----------------------------------------------------------
5783 // Rules which define the behavior of the target architectures pipeline.
5784
5785 // For specific pipelines, eg A53, define the stages of that pipeline
5786 //pipe_desc(ISS, EX1, EX2, WR);
5787 #define ISS S0
5788 #define EX1 S1
5789 #define EX2 S2
5790 #define WR S3
5791
5792 // Integer ALU reg operation
5793 pipeline %{
5794
5795 attributes %{
5796 // ARM instructions are of fixed length
5797 fixed_size_instructions; // Fixed size instructions TODO does
5798 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5799 // ARM instructions come in 32-bit word units
5800 instruction_unit_size = 4; // An instruction is 4 bytes long
5801 instruction_fetch_unit_size = 64; // The processor fetches one line
5802 instruction_fetch_units = 1; // of 64 bytes
5803
5804 // List of nop instructions
5805 nops( MachNop );
5806 %}
5807
5808 // We don't use an actual pipeline model so don't care about resources
5809 // or description. we do use pipeline classes to introduce fixed
5810 // latencies
5811
5812 //----------RESOURCES----------------------------------------------------------
5813 // Resources are the functional units available to the machine
5814
5815 resources( INS0, INS1, INS01 = INS0 | INS1,
5816 ALU0, ALU1, ALU = ALU0 | ALU1,
5817 MAC,
5818 DIV,
5819 BRANCH,
5820 LDST,
5821 NEON_FP);
5822
5823 //----------PIPELINE DESCRIPTION-----------------------------------------------
5824 // Pipeline Description specifies the stages in the machine's pipeline
5825
5826 // Define the pipeline as a generic 6 stage pipeline
5827 pipe_desc(S0, S1, S2, S3, S4, S5);
5828
5829 //----------PIPELINE CLASSES---------------------------------------------------
5830 // Pipeline Classes describe the stages in which input and output are
5831 // referenced by the hardware pipeline.
5832
5833 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5834 %{
5835 single_instruction;
5836 src1 : S1(read);
5837 src2 : S2(read);
5838 dst : S5(write);
5839 INS01 : ISS;
5840 NEON_FP : S5;
5841 %}
5842
5843 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5844 %{
5845 single_instruction;
5846 src1 : S1(read);
5847 src2 : S2(read);
5848 dst : S5(write);
5849 INS01 : ISS;
5850 NEON_FP : S5;
5851 %}
5852
5853 pipe_class fp_uop_s(vRegF dst, vRegF src)
5854 %{
5855 single_instruction;
5856 src : S1(read);
5857 dst : S5(write);
5858 INS01 : ISS;
5859 NEON_FP : S5;
5860 %}
5861
5862 pipe_class fp_uop_d(vRegD dst, vRegD src)
5863 %{
5864 single_instruction;
5865 src : S1(read);
5866 dst : S5(write);
5867 INS01 : ISS;
5868 NEON_FP : S5;
5869 %}
5870
5871 pipe_class fp_d2f(vRegF dst, vRegD src)
5872 %{
5873 single_instruction;
5874 src : S1(read);
5875 dst : S5(write);
5876 INS01 : ISS;
5877 NEON_FP : S5;
5878 %}
5879
5880 pipe_class fp_f2d(vRegD dst, vRegF src)
5881 %{
5882 single_instruction;
5883 src : S1(read);
5884 dst : S5(write);
5885 INS01 : ISS;
5886 NEON_FP : S5;
5887 %}
5888
5889 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5890 %{
5891 single_instruction;
5892 src : S1(read);
5893 dst : S5(write);
5894 INS01 : ISS;
5895 NEON_FP : S5;
5896 %}
5897
5898 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5899 %{
5900 single_instruction;
5901 src : S1(read);
5902 dst : S5(write);
5903 INS01 : ISS;
5904 NEON_FP : S5;
5905 %}
5906
5907 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5908 %{
5909 single_instruction;
5910 src : S1(read);
5911 dst : S5(write);
5912 INS01 : ISS;
5913 NEON_FP : S5;
5914 %}
5915
5916 pipe_class fp_l2f(vRegF dst, iRegL src)
5917 %{
5918 single_instruction;
5919 src : S1(read);
5920 dst : S5(write);
5921 INS01 : ISS;
5922 NEON_FP : S5;
5923 %}
5924
5925 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5926 %{
5927 single_instruction;
5928 src : S1(read);
5929 dst : S5(write);
5930 INS01 : ISS;
5931 NEON_FP : S5;
5932 %}
5933
5934 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5935 %{
5936 single_instruction;
5937 src : S1(read);
5938 dst : S5(write);
5939 INS01 : ISS;
5940 NEON_FP : S5;
5941 %}
5942
5943 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5944 %{
5945 single_instruction;
5946 src : S1(read);
5947 dst : S5(write);
5948 INS01 : ISS;
5949 NEON_FP : S5;
5950 %}
5951
5952 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5953 %{
5954 single_instruction;
5955 src : S1(read);
5956 dst : S5(write);
5957 INS01 : ISS;
5958 NEON_FP : S5;
5959 %}
5960
5961 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5962 %{
5963 single_instruction;
5964 src1 : S1(read);
5965 src2 : S2(read);
5966 dst : S5(write);
5967 INS0 : ISS;
5968 NEON_FP : S5;
5969 %}
5970
5971 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
5972 %{
5973 single_instruction;
5974 src1 : S1(read);
5975 src2 : S2(read);
5976 dst : S5(write);
5977 INS0 : ISS;
5978 NEON_FP : S5;
5979 %}
5980
5981 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
5982 %{
5983 single_instruction;
5984 cr : S1(read);
5985 src1 : S1(read);
5986 src2 : S1(read);
5987 dst : S3(write);
5988 INS01 : ISS;
5989 NEON_FP : S3;
5990 %}
5991
5992 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
5993 %{
5994 single_instruction;
5995 cr : S1(read);
5996 src1 : S1(read);
5997 src2 : S1(read);
5998 dst : S3(write);
5999 INS01 : ISS;
6000 NEON_FP : S3;
6001 %}
6002
6003 pipe_class fp_imm_s(vRegF dst)
6004 %{
6005 single_instruction;
6006 dst : S3(write);
6007 INS01 : ISS;
6008 NEON_FP : S3;
6009 %}
6010
6011 pipe_class fp_imm_d(vRegD dst)
6012 %{
6013 single_instruction;
6014 dst : S3(write);
6015 INS01 : ISS;
6016 NEON_FP : S3;
6017 %}
6018
6019 pipe_class fp_load_constant_s(vRegF dst)
6020 %{
6021 single_instruction;
6022 dst : S4(write);
6023 INS01 : ISS;
6024 NEON_FP : S4;
6025 %}
6026
6027 pipe_class fp_load_constant_d(vRegD dst)
6028 %{
6029 single_instruction;
6030 dst : S4(write);
6031 INS01 : ISS;
6032 NEON_FP : S4;
6033 %}
6034
6035 //------- Integer ALU operations --------------------------
6036
6037 // Integer ALU reg-reg operation
6038 // Operands needed in EX1, result generated in EX2
6039 // Eg. ADD x0, x1, x2
6040 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6041 %{
6042 single_instruction;
6043 dst : EX2(write);
6044 src1 : EX1(read);
6045 src2 : EX1(read);
6046 INS01 : ISS; // Dual issue as instruction 0 or 1
6047 ALU : EX2;
6048 %}
6049
6050 // Integer ALU reg-reg operation with constant shift
6051 // Shifted register must be available in LATE_ISS instead of EX1
6052 // Eg. ADD x0, x1, x2, LSL #2
6053 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6054 %{
6055 single_instruction;
6056 dst : EX2(write);
6057 src1 : EX1(read);
6058 src2 : ISS(read);
6059 INS01 : ISS;
6060 ALU : EX2;
6061 %}
6062
6063 // Integer ALU reg operation with constant shift
6064 // Eg. LSL x0, x1, #shift
6065 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6066 %{
6067 single_instruction;
6068 dst : EX2(write);
6069 src1 : ISS(read);
6070 INS01 : ISS;
6071 ALU : EX2;
6072 %}
6073
6074 // Integer ALU reg-reg operation with variable shift
6075 // Both operands must be available in LATE_ISS instead of EX1
6076 // Result is available in EX1 instead of EX2
6077 // Eg. LSLV x0, x1, x2
6078 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6079 %{
6080 single_instruction;
6081 dst : EX1(write);
6082 src1 : ISS(read);
6083 src2 : ISS(read);
6084 INS01 : ISS;
6085 ALU : EX1;
6086 %}
6087
6088 // Integer ALU reg-reg operation with extract
6089 // As for _vshift above, but result generated in EX2
6090 // Eg. EXTR x0, x1, x2, #N
6091 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6092 %{
6093 single_instruction;
6094 dst : EX2(write);
6095 src1 : ISS(read);
6096 src2 : ISS(read);
6097 INS1 : ISS; // Can only dual issue as Instruction 1
6098 ALU : EX1;
6099 %}
6100
6101 // Integer ALU reg operation
6102 // Eg. NEG x0, x1
6103 pipe_class ialu_reg(iRegI dst, iRegI src)
6104 %{
6105 single_instruction;
6106 dst : EX2(write);
6107 src : EX1(read);
6108 INS01 : ISS;
6109 ALU : EX2;
6110 %}
6111
6112 // Integer ALU reg mmediate operation
6113 // Eg. ADD x0, x1, #N
6114 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6115 %{
6116 single_instruction;
6117 dst : EX2(write);
6118 src1 : EX1(read);
6119 INS01 : ISS;
6120 ALU : EX2;
6121 %}
6122
6123 // Integer ALU immediate operation (no source operands)
6124 // Eg. MOV x0, #N
6125 pipe_class ialu_imm(iRegI dst)
6126 %{
6127 single_instruction;
6128 dst : EX1(write);
6129 INS01 : ISS;
6130 ALU : EX1;
6131 %}
6132
6133 //------- Compare operation -------------------------------
6134
6135 // Compare reg-reg
6136 // Eg. CMP x0, x1
6137 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6138 %{
6139 single_instruction;
6140 // fixed_latency(16);
6141 cr : EX2(write);
6142 op1 : EX1(read);
6143 op2 : EX1(read);
6144 INS01 : ISS;
6145 ALU : EX2;
6146 %}
6147
6148 // Compare reg-reg
6149 // Eg. CMP x0, #N
6150 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6151 %{
6152 single_instruction;
6153 // fixed_latency(16);
6154 cr : EX2(write);
6155 op1 : EX1(read);
6156 INS01 : ISS;
6157 ALU : EX2;
6158 %}
6159
6160 //------- Conditional instructions ------------------------
6161
6162 // Conditional no operands
6163 // Eg. CSINC x0, zr, zr, <cond>
6164 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6165 %{
6166 single_instruction;
6167 cr : EX1(read);
6168 dst : EX2(write);
6169 INS01 : ISS;
6170 ALU : EX2;
6171 %}
6172
6173 // Conditional 2 operand
6174 // EG. CSEL X0, X1, X2, <cond>
6175 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6176 %{
6177 single_instruction;
6178 cr : EX1(read);
6179 src1 : EX1(read);
6180 src2 : EX1(read);
6181 dst : EX2(write);
6182 INS01 : ISS;
6183 ALU : EX2;
6184 %}
6185
6186 // Conditional 2 operand
6187 // EG. CSEL X0, X1, X2, <cond>
6188 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6189 %{
6190 single_instruction;
6191 cr : EX1(read);
6192 src : EX1(read);
6193 dst : EX2(write);
6194 INS01 : ISS;
6195 ALU : EX2;
6196 %}
6197
6198 //------- Multiply pipeline operations --------------------
6199
6200 // Multiply reg-reg
6201 // Eg. MUL w0, w1, w2
6202 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6203 %{
6204 single_instruction;
6205 dst : WR(write);
6206 src1 : ISS(read);
6207 src2 : ISS(read);
6208 INS01 : ISS;
6209 MAC : WR;
6210 %}
6211
6212 // Multiply accumulate
6213 // Eg. MADD w0, w1, w2, w3
6214 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6215 %{
6216 single_instruction;
6217 dst : WR(write);
6218 src1 : ISS(read);
6219 src2 : ISS(read);
6220 src3 : ISS(read);
6221 INS01 : ISS;
6222 MAC : WR;
6223 %}
6224
6225 // Eg. MUL w0, w1, w2
6226 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6227 %{
6228 single_instruction;
6229 fixed_latency(3); // Maximum latency for 64 bit mul
6230 dst : WR(write);
6231 src1 : ISS(read);
6232 src2 : ISS(read);
6233 INS01 : ISS;
6234 MAC : WR;
6235 %}
6236
6237 // Multiply accumulate
6238 // Eg. MADD w0, w1, w2, w3
6239 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6240 %{
6241 single_instruction;
6242 fixed_latency(3); // Maximum latency for 64 bit mul
6243 dst : WR(write);
6244 src1 : ISS(read);
6245 src2 : ISS(read);
6246 src3 : ISS(read);
6247 INS01 : ISS;
6248 MAC : WR;
6249 %}
6250
6251 //------- Divide pipeline operations --------------------
6252
6253 // Eg. SDIV w0, w1, w2
6254 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6255 %{
6256 single_instruction;
6257 fixed_latency(8); // Maximum latency for 32 bit divide
6258 dst : WR(write);
6259 src1 : ISS(read);
6260 src2 : ISS(read);
6261 INS0 : ISS; // Can only dual issue as instruction 0
6262 DIV : WR;
6263 %}
6264
6265 // Eg. SDIV x0, x1, x2
6266 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6267 %{
6268 single_instruction;
6269 fixed_latency(16); // Maximum latency for 64 bit divide
6270 dst : WR(write);
6271 src1 : ISS(read);
6272 src2 : ISS(read);
6273 INS0 : ISS; // Can only dual issue as instruction 0
6274 DIV : WR;
6275 %}
6276
6277 //------- Load pipeline operations ------------------------
6278
6279 // Load - prefetch
6280 // Eg. PFRM <mem>
6281 pipe_class iload_prefetch(memory mem)
6282 %{
6283 single_instruction;
6284 mem : ISS(read);
6285 INS01 : ISS;
6286 LDST : WR;
6287 %}
6288
6289 // Load - reg, mem
6290 // Eg. LDR x0, <mem>
6291 pipe_class iload_reg_mem(iRegI dst, memory mem)
6292 %{
6293 single_instruction;
6294 dst : WR(write);
6295 mem : ISS(read);
6296 INS01 : ISS;
6297 LDST : WR;
6298 %}
6299
6300 // Load - reg, reg
6301 // Eg. LDR x0, [sp, x1]
6302 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6303 %{
6304 single_instruction;
6305 dst : WR(write);
6306 src : ISS(read);
6307 INS01 : ISS;
6308 LDST : WR;
6309 %}
6310
6311 //------- Store pipeline operations -----------------------
6312
6313 // Store - zr, mem
6314 // Eg. STR zr, <mem>
6315 pipe_class istore_mem(memory mem)
6316 %{
6317 single_instruction;
6318 mem : ISS(read);
6319 INS01 : ISS;
6320 LDST : WR;
6321 %}
6322
6323 // Store - reg, mem
6324 // Eg. STR x0, <mem>
6325 pipe_class istore_reg_mem(iRegI src, memory mem)
6326 %{
6327 single_instruction;
6328 mem : ISS(read);
6329 src : EX2(read);
6330 INS01 : ISS;
6331 LDST : WR;
6332 %}
6333
6334 // Store - reg, reg
6335 // Eg. STR x0, [sp, x1]
6336 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6337 %{
6338 single_instruction;
6339 dst : ISS(read);
6340 src : EX2(read);
6341 INS01 : ISS;
6342 LDST : WR;
6343 %}
6344
6345 //------- Store pipeline operations -----------------------
6346
6347 // Branch
6348 pipe_class pipe_branch()
6349 %{
6350 single_instruction;
6351 INS01 : ISS;
6352 BRANCH : EX1;
6353 %}
6354
6355 // Conditional branch
6356 pipe_class pipe_branch_cond(rFlagsReg cr)
6357 %{
6358 single_instruction;
6359 cr : EX1(read);
6360 INS01 : ISS;
6361 BRANCH : EX1;
6362 %}
6363
6364 // Compare & Branch
6365 // EG. CBZ/CBNZ
6366 pipe_class pipe_cmp_branch(iRegI op1)
6367 %{
6368 single_instruction;
6369 op1 : EX1(read);
6370 INS01 : ISS;
6371 BRANCH : EX1;
6372 %}
6373
6374 //------- Synchronisation operations ----------------------
6375
6376 // Any operation requiring serialization.
6377 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6378 pipe_class pipe_serial()
6379 %{
6380 single_instruction;
6381 force_serialization;
6382 fixed_latency(16);
6383 INS01 : ISS(2); // Cannot dual issue with any other instruction
6384 LDST : WR;
6385 %}
6386
6387 // Generic big/slow expanded idiom - also serialized
6388 pipe_class pipe_slow()
6389 %{
6390 instruction_count(10);
6391 multiple_bundles;
6392 force_serialization;
6393 fixed_latency(16);
6394 INS01 : ISS(2); // Cannot dual issue with any other instruction
6395 LDST : WR;
6396 %}
6397
6398 // Empty pipeline class
6399 pipe_class pipe_class_empty()
6400 %{
6401 single_instruction;
6402 fixed_latency(0);
6403 %}
6404
6405 // Default pipeline class.
6406 pipe_class pipe_class_default()
6407 %{
6408 single_instruction;
6409 fixed_latency(2);
6410 %}
6411
6412 // Pipeline class for compares.
6413 pipe_class pipe_class_compare()
6414 %{
6415 single_instruction;
6416 fixed_latency(16);
6417 %}
6418
6419 // Pipeline class for memory operations.
6420 pipe_class pipe_class_memory()
6421 %{
6422 single_instruction;
6423 fixed_latency(16);
6424 %}
6425
6426 // Pipeline class for call.
6427 pipe_class pipe_class_call()
6428 %{
6429 single_instruction;
6430 fixed_latency(100);
6431 %}
6432
6433 // Define the class for the Nop node.
6434 define %{
6435 MachNop = pipe_class_empty;
6436 %}
6437
6438 %}
6439 //----------INSTRUCTIONS-------------------------------------------------------
6440 //
6441 // match -- States which machine-independent subtree may be replaced
6442 // by this instruction.
6443 // ins_cost -- The estimated cost of this instruction is used by instruction
6444 // selection to identify a minimum cost tree of machine
6445 // instructions that matches a tree of machine-independent
6446 // instructions.
6447 // format -- A string providing the disassembly for this instruction.
6448 // The value of an instruction's operand may be inserted
6449 // by referring to it with a '$' prefix.
6450 // opcode -- Three instruction opcodes may be provided. These are referred
6451 // to within an encode class as $primary, $secondary, and $tertiary
6452 // rrspectively. The primary opcode is commonly used to
6453 // indicate the type of machine instruction, while secondary
6454 // and tertiary are often used for prefix options or addressing
6455 // modes.
6456 // ins_encode -- A list of encode classes with parameters. The encode class
6457 // name must have been defined in an 'enc_class' specification
6458 // in the encode section of the architecture description.
6459
6460 // ============================================================================
6461 // Memory (Load/Store) Instructions
6462
6463 // Load Instructions
6464
6465 // Load Byte (8 bit signed)
6466 instruct loadB(iRegINoSp dst, memory1 mem)
6467 %{
6468 match(Set dst (LoadB mem));
6469 predicate(!needs_acquiring_load(n));
6470
6471 ins_cost(4 * INSN_COST);
6472 format %{ "ldrsbw $dst, $mem\t# byte" %}
6473
6474 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6475
6476 ins_pipe(iload_reg_mem);
6477 %}
6478
6479 // Load Byte (8 bit signed) into long
6480 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6481 %{
6482 match(Set dst (ConvI2L (LoadB mem)));
6483 predicate(!needs_acquiring_load(n->in(1)));
6484
6485 ins_cost(4 * INSN_COST);
6486 format %{ "ldrsb $dst, $mem\t# byte" %}
6487
6488 ins_encode(aarch64_enc_ldrsb(dst, mem));
6489
6490 ins_pipe(iload_reg_mem);
6491 %}
6492
6493 // Load Byte (8 bit unsigned)
6494 instruct loadUB(iRegINoSp dst, memory1 mem)
6495 %{
6496 match(Set dst (LoadUB mem));
6497 predicate(!needs_acquiring_load(n));
6498
6499 ins_cost(4 * INSN_COST);
6500 format %{ "ldrbw $dst, $mem\t# byte" %}
6501
6502 ins_encode(aarch64_enc_ldrb(dst, mem));
6503
6504 ins_pipe(iload_reg_mem);
6505 %}
6506
6507 // Load Byte (8 bit unsigned) into long
6508 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6509 %{
6510 match(Set dst (ConvI2L (LoadUB mem)));
6511 predicate(!needs_acquiring_load(n->in(1)));
6512
6513 ins_cost(4 * INSN_COST);
6514 format %{ "ldrb $dst, $mem\t# byte" %}
6515
6516 ins_encode(aarch64_enc_ldrb(dst, mem));
6517
6518 ins_pipe(iload_reg_mem);
6519 %}
6520
6521 // Load Short (16 bit signed)
6522 instruct loadS(iRegINoSp dst, memory2 mem)
6523 %{
6524 match(Set dst (LoadS mem));
6525 predicate(!needs_acquiring_load(n));
6526
6527 ins_cost(4 * INSN_COST);
6528 format %{ "ldrshw $dst, $mem\t# short" %}
6529
6530 ins_encode(aarch64_enc_ldrshw(dst, mem));
6531
6532 ins_pipe(iload_reg_mem);
6533 %}
6534
6535 // Load Short (16 bit signed) into long
6536 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6537 %{
6538 match(Set dst (ConvI2L (LoadS mem)));
6539 predicate(!needs_acquiring_load(n->in(1)));
6540
6541 ins_cost(4 * INSN_COST);
6542 format %{ "ldrsh $dst, $mem\t# short" %}
6543
6544 ins_encode(aarch64_enc_ldrsh(dst, mem));
6545
6546 ins_pipe(iload_reg_mem);
6547 %}
6548
6549 // Load Char (16 bit unsigned)
6550 instruct loadUS(iRegINoSp dst, memory2 mem)
6551 %{
6552 match(Set dst (LoadUS mem));
6553 predicate(!needs_acquiring_load(n));
6554
6555 ins_cost(4 * INSN_COST);
6556 format %{ "ldrh $dst, $mem\t# short" %}
6557
6558 ins_encode(aarch64_enc_ldrh(dst, mem));
6559
6560 ins_pipe(iload_reg_mem);
6561 %}
6562
6563 // Load Short/Char (16 bit unsigned) into long
6564 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6565 %{
6566 match(Set dst (ConvI2L (LoadUS mem)));
6567 predicate(!needs_acquiring_load(n->in(1)));
6568
6569 ins_cost(4 * INSN_COST);
6570 format %{ "ldrh $dst, $mem\t# short" %}
6571
6572 ins_encode(aarch64_enc_ldrh(dst, mem));
6573
6574 ins_pipe(iload_reg_mem);
6575 %}
6576
6577 // Load Integer (32 bit signed)
6578 instruct loadI(iRegINoSp dst, memory4 mem)
6579 %{
6580 match(Set dst (LoadI mem));
6581 predicate(!needs_acquiring_load(n));
6582
6583 ins_cost(4 * INSN_COST);
6584 format %{ "ldrw $dst, $mem\t# int" %}
6585
6586 ins_encode(aarch64_enc_ldrw(dst, mem));
6587
6588 ins_pipe(iload_reg_mem);
6589 %}
6590
6591 // Load Integer (32 bit signed) into long
6592 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6593 %{
6594 match(Set dst (ConvI2L (LoadI mem)));
6595 predicate(!needs_acquiring_load(n->in(1)));
6596
6597 ins_cost(4 * INSN_COST);
6598 format %{ "ldrsw $dst, $mem\t# int" %}
6599
6600 ins_encode(aarch64_enc_ldrsw(dst, mem));
6601
6602 ins_pipe(iload_reg_mem);
6603 %}
6604
6605 // Load Integer (32 bit unsigned) into long
6606 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6607 %{
6608 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6609 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6610
6611 ins_cost(4 * INSN_COST);
6612 format %{ "ldrw $dst, $mem\t# int" %}
6613
6614 ins_encode(aarch64_enc_ldrw(dst, mem));
6615
6616 ins_pipe(iload_reg_mem);
6617 %}
6618
6619 // Load Long (64 bit signed)
6620 instruct loadL(iRegLNoSp dst, memory8 mem)
6621 %{
6622 match(Set dst (LoadL mem));
6623 predicate(!needs_acquiring_load(n));
6624
6625 ins_cost(4 * INSN_COST);
6626 format %{ "ldr $dst, $mem\t# int" %}
6627
6628 ins_encode(aarch64_enc_ldr(dst, mem));
6629
6630 ins_pipe(iload_reg_mem);
6631 %}
6632
6633 // Load Range
6634 instruct loadRange(iRegINoSp dst, memory4 mem)
6635 %{
6636 match(Set dst (LoadRange mem));
6637
6638 ins_cost(4 * INSN_COST);
6639 format %{ "ldrw $dst, $mem\t# range" %}
6640
6641 ins_encode(aarch64_enc_ldrw(dst, mem));
6642
6643 ins_pipe(iload_reg_mem);
6644 %}
6645
6646 // Load Pointer
6647 instruct loadP(iRegPNoSp dst, memory8 mem)
6648 %{
6649 match(Set dst (LoadP mem));
6650 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6651
6652 ins_cost(4 * INSN_COST);
6653 format %{ "ldr $dst, $mem\t# ptr" %}
6654
6655 ins_encode(aarch64_enc_ldr(dst, mem));
6656
6657 ins_pipe(iload_reg_mem);
6658 %}
6659
6660 // Load Compressed Pointer
6661 instruct loadN(iRegNNoSp dst, memory4 mem)
6662 %{
6663 match(Set dst (LoadN mem));
6664 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6665
6666 ins_cost(4 * INSN_COST);
6667 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6668
6669 ins_encode(aarch64_enc_ldrw(dst, mem));
6670
6671 ins_pipe(iload_reg_mem);
6672 %}
6673
6674 // Load Klass Pointer
6675 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6676 %{
6677 match(Set dst (LoadKlass mem));
6678 predicate(!needs_acquiring_load(n));
6679
6680 ins_cost(4 * INSN_COST);
6681 format %{ "ldr $dst, $mem\t# class" %}
6682
6683 ins_encode(aarch64_enc_ldr(dst, mem));
6684
6685 ins_pipe(iload_reg_mem);
6686 %}
6687
6688 // Load Narrow Klass Pointer
6689 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6690 %{
6691 match(Set dst (LoadNKlass mem));
6692 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6693
6694 ins_cost(4 * INSN_COST);
6695 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6696
6697 ins_encode(aarch64_enc_ldrw(dst, mem));
6698
6699 ins_pipe(iload_reg_mem);
6700 %}
6701
6702 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6703 %{
6704 match(Set dst (LoadNKlass mem));
6705 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6706
6707 ins_cost(4 * INSN_COST);
6708 format %{
6709 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6710 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6711 %}
6712 ins_encode %{
6713 // inlined aarch64_enc_ldrw
6714 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6715 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6716 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6717 %}
6718 ins_pipe(iload_reg_mem);
6719 %}
6720
6721 // Load Float
6722 instruct loadF(vRegF dst, memory4 mem)
6723 %{
6724 match(Set dst (LoadF mem));
6725 predicate(!needs_acquiring_load(n));
6726
6727 ins_cost(4 * INSN_COST);
6728 format %{ "ldrs $dst, $mem\t# float" %}
6729
6730 ins_encode( aarch64_enc_ldrs(dst, mem) );
6731
6732 ins_pipe(pipe_class_memory);
6733 %}
6734
6735 // Load Double
6736 instruct loadD(vRegD dst, memory8 mem)
6737 %{
6738 match(Set dst (LoadD mem));
6739 predicate(!needs_acquiring_load(n));
6740
6741 ins_cost(4 * INSN_COST);
6742 format %{ "ldrd $dst, $mem\t# double" %}
6743
6744 ins_encode( aarch64_enc_ldrd(dst, mem) );
6745
6746 ins_pipe(pipe_class_memory);
6747 %}
6748
6749
6750 // Load Int Constant
6751 instruct loadConI(iRegINoSp dst, immI src)
6752 %{
6753 match(Set dst src);
6754
6755 ins_cost(INSN_COST);
6756 format %{ "mov $dst, $src\t# int" %}
6757
6758 ins_encode( aarch64_enc_movw_imm(dst, src) );
6759
6760 ins_pipe(ialu_imm);
6761 %}
6762
6763 // Load Long Constant
6764 instruct loadConL(iRegLNoSp dst, immL src)
6765 %{
6766 match(Set dst src);
6767
6768 ins_cost(INSN_COST);
6769 format %{ "mov $dst, $src\t# long" %}
6770
6771 ins_encode( aarch64_enc_mov_imm(dst, src) );
6772
6773 ins_pipe(ialu_imm);
6774 %}
6775
6776 // Load Pointer Constant
6777
6778 instruct loadConP(iRegPNoSp dst, immP con)
6779 %{
6780 match(Set dst con);
6781
6782 ins_cost(INSN_COST * 4);
6783 format %{
6784 "mov $dst, $con\t# ptr\n\t"
6785 %}
6786
6787 ins_encode(aarch64_enc_mov_p(dst, con));
6788
6789 ins_pipe(ialu_imm);
6790 %}
6791
6792 // Load Null Pointer Constant
6793
6794 instruct loadConP0(iRegPNoSp dst, immP0 con)
6795 %{
6796 match(Set dst con);
6797
6798 ins_cost(INSN_COST);
6799 format %{ "mov $dst, $con\t# nullptr ptr" %}
6800
6801 ins_encode(aarch64_enc_mov_p0(dst, con));
6802
6803 ins_pipe(ialu_imm);
6804 %}
6805
6806 // Load Pointer Constant One
6807
6808 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6809 %{
6810 match(Set dst con);
6811
6812 ins_cost(INSN_COST);
6813 format %{ "mov $dst, $con\t# nullptr ptr" %}
6814
6815 ins_encode(aarch64_enc_mov_p1(dst, con));
6816
6817 ins_pipe(ialu_imm);
6818 %}
6819
6820 // Load Byte Map Base Constant
6821
6822 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6823 %{
6824 match(Set dst con);
6825
6826 ins_cost(INSN_COST);
6827 format %{ "adr $dst, $con\t# Byte Map Base" %}
6828
6829 ins_encode %{
6830 __ load_byte_map_base($dst$$Register);
6831 %}
6832
6833 ins_pipe(ialu_imm);
6834 %}
6835
6836 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6837 %{
6838 match(Set dst con);
6839
6840 ins_cost(INSN_COST);
6841 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6842
6843 ins_encode %{
6844 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6845 %}
6846
6847 ins_pipe(ialu_imm);
6848 %}
6849
6850 // Load Narrow Pointer Constant
6851
6852 instruct loadConN(iRegNNoSp dst, immN con)
6853 %{
6854 match(Set dst con);
6855
6856 ins_cost(INSN_COST * 4);
6857 format %{ "mov $dst, $con\t# compressed ptr" %}
6858
6859 ins_encode(aarch64_enc_mov_n(dst, con));
6860
6861 ins_pipe(ialu_imm);
6862 %}
6863
6864 // Load Narrow Null Pointer Constant
6865
6866 instruct loadConN0(iRegNNoSp dst, immN0 con)
6867 %{
6868 match(Set dst con);
6869
6870 ins_cost(INSN_COST);
6871 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6872
6873 ins_encode(aarch64_enc_mov_n0(dst, con));
6874
6875 ins_pipe(ialu_imm);
6876 %}
6877
6878 // Load Narrow Klass Constant
6879
6880 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6881 %{
6882 match(Set dst con);
6883
6884 ins_cost(INSN_COST);
6885 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6886
6887 ins_encode(aarch64_enc_mov_nk(dst, con));
6888
6889 ins_pipe(ialu_imm);
6890 %}
6891
6892 // Load Packed Float Constant
6893
6894 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6895 match(Set dst con);
6896 ins_cost(INSN_COST * 4);
6897 format %{ "fmovs $dst, $con"%}
6898 ins_encode %{
6899 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6900 %}
6901
6902 ins_pipe(fp_imm_s);
6903 %}
6904
6905 // Load Float Constant
6906
6907 instruct loadConF(vRegF dst, immF con) %{
6908 match(Set dst con);
6909
6910 ins_cost(INSN_COST * 4);
6911
6912 format %{
6913 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6914 %}
6915
6916 ins_encode %{
6917 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6918 %}
6919
6920 ins_pipe(fp_load_constant_s);
6921 %}
6922
6923 // Load Packed Double Constant
6924
6925 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6926 match(Set dst con);
6927 ins_cost(INSN_COST);
6928 format %{ "fmovd $dst, $con"%}
6929 ins_encode %{
6930 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6931 %}
6932
6933 ins_pipe(fp_imm_d);
6934 %}
6935
6936 // Load Double Constant
6937
6938 instruct loadConD(vRegD dst, immD con) %{
6939 match(Set dst con);
6940
6941 ins_cost(INSN_COST * 5);
6942 format %{
6943 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6944 %}
6945
6946 ins_encode %{
6947 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6948 %}
6949
6950 ins_pipe(fp_load_constant_d);
6951 %}
6952
6953 // Store Instructions
6954
6955 // Store Byte
6956 instruct storeB(iRegIorL2I src, memory1 mem)
6957 %{
6958 match(Set mem (StoreB mem src));
6959 predicate(!needs_releasing_store(n));
6960
6961 ins_cost(INSN_COST);
6962 format %{ "strb $src, $mem\t# byte" %}
6963
6964 ins_encode(aarch64_enc_strb(src, mem));
6965
6966 ins_pipe(istore_reg_mem);
6967 %}
6968
6969
6970 instruct storeimmB0(immI0 zero, memory1 mem)
6971 %{
6972 match(Set mem (StoreB mem zero));
6973 predicate(!needs_releasing_store(n));
6974
6975 ins_cost(INSN_COST);
6976 format %{ "strb rscractch2, $mem\t# byte" %}
6977
6978 ins_encode(aarch64_enc_strb0(mem));
6979
6980 ins_pipe(istore_mem);
6981 %}
6982
6983 // Store Char/Short
6984 instruct storeC(iRegIorL2I src, memory2 mem)
6985 %{
6986 match(Set mem (StoreC mem src));
6987 predicate(!needs_releasing_store(n));
6988
6989 ins_cost(INSN_COST);
6990 format %{ "strh $src, $mem\t# short" %}
6991
6992 ins_encode(aarch64_enc_strh(src, mem));
6993
6994 ins_pipe(istore_reg_mem);
6995 %}
6996
6997 instruct storeimmC0(immI0 zero, memory2 mem)
6998 %{
6999 match(Set mem (StoreC mem zero));
7000 predicate(!needs_releasing_store(n));
7001
7002 ins_cost(INSN_COST);
7003 format %{ "strh zr, $mem\t# short" %}
7004
7005 ins_encode(aarch64_enc_strh0(mem));
7006
7007 ins_pipe(istore_mem);
7008 %}
7009
7010 // Store Integer
7011
7012 instruct storeI(iRegIorL2I src, memory4 mem)
7013 %{
7014 match(Set mem(StoreI mem src));
7015 predicate(!needs_releasing_store(n));
7016
7017 ins_cost(INSN_COST);
7018 format %{ "strw $src, $mem\t# int" %}
7019
7020 ins_encode(aarch64_enc_strw(src, mem));
7021
7022 ins_pipe(istore_reg_mem);
7023 %}
7024
7025 instruct storeimmI0(immI0 zero, memory4 mem)
7026 %{
7027 match(Set mem(StoreI mem zero));
7028 predicate(!needs_releasing_store(n));
7029
7030 ins_cost(INSN_COST);
7031 format %{ "strw zr, $mem\t# int" %}
7032
7033 ins_encode(aarch64_enc_strw0(mem));
7034
7035 ins_pipe(istore_mem);
7036 %}
7037
7038 // Store Long (64 bit signed)
7039 instruct storeL(iRegL src, memory8 mem)
7040 %{
7041 match(Set mem (StoreL mem src));
7042 predicate(!needs_releasing_store(n));
7043
7044 ins_cost(INSN_COST);
7045 format %{ "str $src, $mem\t# int" %}
7046
7047 ins_encode(aarch64_enc_str(src, mem));
7048
7049 ins_pipe(istore_reg_mem);
7050 %}
7051
7052 // Store Long (64 bit signed)
7053 instruct storeimmL0(immL0 zero, memory8 mem)
7054 %{
7055 match(Set mem (StoreL mem zero));
7056 predicate(!needs_releasing_store(n));
7057
7058 ins_cost(INSN_COST);
7059 format %{ "str zr, $mem\t# int" %}
7060
7061 ins_encode(aarch64_enc_str0(mem));
7062
7063 ins_pipe(istore_mem);
7064 %}
7065
7066 // Store Pointer
7067 instruct storeP(iRegP src, memory8 mem)
7068 %{
7069 match(Set mem (StoreP mem src));
7070 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7071
7072 ins_cost(INSN_COST);
7073 format %{ "str $src, $mem\t# ptr" %}
7074
7075 ins_encode(aarch64_enc_str(src, mem));
7076
7077 ins_pipe(istore_reg_mem);
7078 %}
7079
7080 // Store Pointer
7081 instruct storeimmP0(immP0 zero, memory8 mem)
7082 %{
7083 match(Set mem (StoreP mem zero));
7084 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7085
7086 ins_cost(INSN_COST);
7087 format %{ "str zr, $mem\t# ptr" %}
7088
7089 ins_encode(aarch64_enc_str0(mem));
7090
7091 ins_pipe(istore_mem);
7092 %}
7093
7094 // Store Compressed Pointer
7095 instruct storeN(iRegN src, memory4 mem)
7096 %{
7097 match(Set mem (StoreN mem src));
7098 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7099
7100 ins_cost(INSN_COST);
7101 format %{ "strw $src, $mem\t# compressed ptr" %}
7102
7103 ins_encode(aarch64_enc_strw(src, mem));
7104
7105 ins_pipe(istore_reg_mem);
7106 %}
7107
7108 instruct storeImmN0(immN0 zero, memory4 mem)
7109 %{
7110 match(Set mem (StoreN mem zero));
7111 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7112
7113 ins_cost(INSN_COST);
7114 format %{ "strw zr, $mem\t# compressed ptr" %}
7115
7116 ins_encode(aarch64_enc_strw0(mem));
7117
7118 ins_pipe(istore_mem);
7119 %}
7120
7121 // Store Float
7122 instruct storeF(vRegF src, memory4 mem)
7123 %{
7124 match(Set mem (StoreF mem src));
7125 predicate(!needs_releasing_store(n));
7126
7127 ins_cost(INSN_COST);
7128 format %{ "strs $src, $mem\t# float" %}
7129
7130 ins_encode( aarch64_enc_strs(src, mem) );
7131
7132 ins_pipe(pipe_class_memory);
7133 %}
7134
7135 // TODO
7136 // implement storeImmF0 and storeFImmPacked
7137
7138 // Store Double
7139 instruct storeD(vRegD src, memory8 mem)
7140 %{
7141 match(Set mem (StoreD mem src));
7142 predicate(!needs_releasing_store(n));
7143
7144 ins_cost(INSN_COST);
7145 format %{ "strd $src, $mem\t# double" %}
7146
7147 ins_encode( aarch64_enc_strd(src, mem) );
7148
7149 ins_pipe(pipe_class_memory);
7150 %}
7151
7152 // Store Compressed Klass Pointer
7153 instruct storeNKlass(iRegN src, memory4 mem)
7154 %{
7155 predicate(!needs_releasing_store(n));
7156 match(Set mem (StoreNKlass mem src));
7157
7158 ins_cost(INSN_COST);
7159 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7160
7161 ins_encode(aarch64_enc_strw(src, mem));
7162
7163 ins_pipe(istore_reg_mem);
7164 %}
7165
7166 // TODO
7167 // implement storeImmD0 and storeDImmPacked
7168
7169 // prefetch instructions
7170 // Must be safe to execute with invalid address (cannot fault).
7171
7172 instruct prefetchalloc( memory8 mem ) %{
7173 match(PrefetchAllocation mem);
7174
7175 ins_cost(INSN_COST);
7176 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7177
7178 ins_encode( aarch64_enc_prefetchw(mem) );
7179
7180 ins_pipe(iload_prefetch);
7181 %}
7182
7183 // ---------------- volatile loads and stores ----------------
7184
7185 // Load Byte (8 bit signed)
7186 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7187 %{
7188 match(Set dst (LoadB mem));
7189
7190 ins_cost(VOLATILE_REF_COST);
7191 format %{ "ldarsb $dst, $mem\t# byte" %}
7192
7193 ins_encode(aarch64_enc_ldarsb(dst, mem));
7194
7195 ins_pipe(pipe_serial);
7196 %}
7197
7198 // Load Byte (8 bit signed) into long
7199 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7200 %{
7201 match(Set dst (ConvI2L (LoadB mem)));
7202
7203 ins_cost(VOLATILE_REF_COST);
7204 format %{ "ldarsb $dst, $mem\t# byte" %}
7205
7206 ins_encode(aarch64_enc_ldarsb(dst, mem));
7207
7208 ins_pipe(pipe_serial);
7209 %}
7210
7211 // Load Byte (8 bit unsigned)
7212 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7213 %{
7214 match(Set dst (LoadUB mem));
7215
7216 ins_cost(VOLATILE_REF_COST);
7217 format %{ "ldarb $dst, $mem\t# byte" %}
7218
7219 ins_encode(aarch64_enc_ldarb(dst, mem));
7220
7221 ins_pipe(pipe_serial);
7222 %}
7223
7224 // Load Byte (8 bit unsigned) into long
7225 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7226 %{
7227 match(Set dst (ConvI2L (LoadUB mem)));
7228
7229 ins_cost(VOLATILE_REF_COST);
7230 format %{ "ldarb $dst, $mem\t# byte" %}
7231
7232 ins_encode(aarch64_enc_ldarb(dst, mem));
7233
7234 ins_pipe(pipe_serial);
7235 %}
7236
7237 // Load Short (16 bit signed)
7238 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7239 %{
7240 match(Set dst (LoadS mem));
7241
7242 ins_cost(VOLATILE_REF_COST);
7243 format %{ "ldarshw $dst, $mem\t# short" %}
7244
7245 ins_encode(aarch64_enc_ldarshw(dst, mem));
7246
7247 ins_pipe(pipe_serial);
7248 %}
7249
7250 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7251 %{
7252 match(Set dst (LoadUS mem));
7253
7254 ins_cost(VOLATILE_REF_COST);
7255 format %{ "ldarhw $dst, $mem\t# short" %}
7256
7257 ins_encode(aarch64_enc_ldarhw(dst, mem));
7258
7259 ins_pipe(pipe_serial);
7260 %}
7261
7262 // Load Short/Char (16 bit unsigned) into long
7263 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7264 %{
7265 match(Set dst (ConvI2L (LoadUS mem)));
7266
7267 ins_cost(VOLATILE_REF_COST);
7268 format %{ "ldarh $dst, $mem\t# short" %}
7269
7270 ins_encode(aarch64_enc_ldarh(dst, mem));
7271
7272 ins_pipe(pipe_serial);
7273 %}
7274
7275 // Load Short/Char (16 bit signed) into long
7276 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7277 %{
7278 match(Set dst (ConvI2L (LoadS mem)));
7279
7280 ins_cost(VOLATILE_REF_COST);
7281 format %{ "ldarh $dst, $mem\t# short" %}
7282
7283 ins_encode(aarch64_enc_ldarsh(dst, mem));
7284
7285 ins_pipe(pipe_serial);
7286 %}
7287
7288 // Load Integer (32 bit signed)
7289 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7290 %{
7291 match(Set dst (LoadI mem));
7292
7293 ins_cost(VOLATILE_REF_COST);
7294 format %{ "ldarw $dst, $mem\t# int" %}
7295
7296 ins_encode(aarch64_enc_ldarw(dst, mem));
7297
7298 ins_pipe(pipe_serial);
7299 %}
7300
7301 // Load Integer (32 bit unsigned) into long
7302 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7303 %{
7304 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7305
7306 ins_cost(VOLATILE_REF_COST);
7307 format %{ "ldarw $dst, $mem\t# int" %}
7308
7309 ins_encode(aarch64_enc_ldarw(dst, mem));
7310
7311 ins_pipe(pipe_serial);
7312 %}
7313
7314 // Load Long (64 bit signed)
7315 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7316 %{
7317 match(Set dst (LoadL mem));
7318
7319 ins_cost(VOLATILE_REF_COST);
7320 format %{ "ldar $dst, $mem\t# int" %}
7321
7322 ins_encode(aarch64_enc_ldar(dst, mem));
7323
7324 ins_pipe(pipe_serial);
7325 %}
7326
7327 // Load Pointer
7328 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7329 %{
7330 match(Set dst (LoadP mem));
7331 predicate(n->as_Load()->barrier_data() == 0);
7332
7333 ins_cost(VOLATILE_REF_COST);
7334 format %{ "ldar $dst, $mem\t# ptr" %}
7335
7336 ins_encode(aarch64_enc_ldar(dst, mem));
7337
7338 ins_pipe(pipe_serial);
7339 %}
7340
7341 // Load Compressed Pointer
7342 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7343 %{
7344 match(Set dst (LoadN mem));
7345 predicate(n->as_Load()->barrier_data() == 0);
7346
7347 ins_cost(VOLATILE_REF_COST);
7348 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7349
7350 ins_encode(aarch64_enc_ldarw(dst, mem));
7351
7352 ins_pipe(pipe_serial);
7353 %}
7354
7355 // Load Float
7356 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7357 %{
7358 match(Set dst (LoadF mem));
7359
7360 ins_cost(VOLATILE_REF_COST);
7361 format %{ "ldars $dst, $mem\t# float" %}
7362
7363 ins_encode( aarch64_enc_fldars(dst, mem) );
7364
7365 ins_pipe(pipe_serial);
7366 %}
7367
7368 // Load Double
7369 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7370 %{
7371 match(Set dst (LoadD mem));
7372
7373 ins_cost(VOLATILE_REF_COST);
7374 format %{ "ldard $dst, $mem\t# double" %}
7375
7376 ins_encode( aarch64_enc_fldard(dst, mem) );
7377
7378 ins_pipe(pipe_serial);
7379 %}
7380
7381 // Store Byte
7382 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7383 %{
7384 match(Set mem (StoreB mem src));
7385
7386 ins_cost(VOLATILE_REF_COST);
7387 format %{ "stlrb $src, $mem\t# byte" %}
7388
7389 ins_encode(aarch64_enc_stlrb(src, mem));
7390
7391 ins_pipe(pipe_class_memory);
7392 %}
7393
7394 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7395 %{
7396 match(Set mem (StoreB mem zero));
7397
7398 ins_cost(VOLATILE_REF_COST);
7399 format %{ "stlrb zr, $mem\t# byte" %}
7400
7401 ins_encode(aarch64_enc_stlrb0(mem));
7402
7403 ins_pipe(pipe_class_memory);
7404 %}
7405
7406 // Store Char/Short
7407 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7408 %{
7409 match(Set mem (StoreC mem src));
7410
7411 ins_cost(VOLATILE_REF_COST);
7412 format %{ "stlrh $src, $mem\t# short" %}
7413
7414 ins_encode(aarch64_enc_stlrh(src, mem));
7415
7416 ins_pipe(pipe_class_memory);
7417 %}
7418
7419 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7420 %{
7421 match(Set mem (StoreC mem zero));
7422
7423 ins_cost(VOLATILE_REF_COST);
7424 format %{ "stlrh zr, $mem\t# short" %}
7425
7426 ins_encode(aarch64_enc_stlrh0(mem));
7427
7428 ins_pipe(pipe_class_memory);
7429 %}
7430
7431 // Store Integer
7432
7433 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7434 %{
7435 match(Set mem(StoreI mem src));
7436
7437 ins_cost(VOLATILE_REF_COST);
7438 format %{ "stlrw $src, $mem\t# int" %}
7439
7440 ins_encode(aarch64_enc_stlrw(src, mem));
7441
7442 ins_pipe(pipe_class_memory);
7443 %}
7444
7445 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7446 %{
7447 match(Set mem(StoreI mem zero));
7448
7449 ins_cost(VOLATILE_REF_COST);
7450 format %{ "stlrw zr, $mem\t# int" %}
7451
7452 ins_encode(aarch64_enc_stlrw0(mem));
7453
7454 ins_pipe(pipe_class_memory);
7455 %}
7456
7457 // Store Long (64 bit signed)
7458 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7459 %{
7460 match(Set mem (StoreL mem src));
7461
7462 ins_cost(VOLATILE_REF_COST);
7463 format %{ "stlr $src, $mem\t# int" %}
7464
7465 ins_encode(aarch64_enc_stlr(src, mem));
7466
7467 ins_pipe(pipe_class_memory);
7468 %}
7469
7470 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7471 %{
7472 match(Set mem (StoreL mem zero));
7473
7474 ins_cost(VOLATILE_REF_COST);
7475 format %{ "stlr zr, $mem\t# int" %}
7476
7477 ins_encode(aarch64_enc_stlr0(mem));
7478
7479 ins_pipe(pipe_class_memory);
7480 %}
7481
7482 // Store Pointer
7483 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7484 %{
7485 match(Set mem (StoreP mem src));
7486 predicate(n->as_Store()->barrier_data() == 0);
7487
7488 ins_cost(VOLATILE_REF_COST);
7489 format %{ "stlr $src, $mem\t# ptr" %}
7490
7491 ins_encode(aarch64_enc_stlr(src, mem));
7492
7493 ins_pipe(pipe_class_memory);
7494 %}
7495
7496 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7497 %{
7498 match(Set mem (StoreP mem zero));
7499 predicate(n->as_Store()->barrier_data() == 0);
7500
7501 ins_cost(VOLATILE_REF_COST);
7502 format %{ "stlr zr, $mem\t# ptr" %}
7503
7504 ins_encode(aarch64_enc_stlr0(mem));
7505
7506 ins_pipe(pipe_class_memory);
7507 %}
7508
7509 // Store Compressed Pointer
7510 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7511 %{
7512 match(Set mem (StoreN mem src));
7513 predicate(n->as_Store()->barrier_data() == 0);
7514
7515 ins_cost(VOLATILE_REF_COST);
7516 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7517
7518 ins_encode(aarch64_enc_stlrw(src, mem));
7519
7520 ins_pipe(pipe_class_memory);
7521 %}
7522
7523 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7524 %{
7525 match(Set mem (StoreN mem zero));
7526 predicate(n->as_Store()->barrier_data() == 0);
7527
7528 ins_cost(VOLATILE_REF_COST);
7529 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7530
7531 ins_encode(aarch64_enc_stlrw0(mem));
7532
7533 ins_pipe(pipe_class_memory);
7534 %}
7535
7536 // Store Float
7537 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7538 %{
7539 match(Set mem (StoreF mem src));
7540
7541 ins_cost(VOLATILE_REF_COST);
7542 format %{ "stlrs $src, $mem\t# float" %}
7543
7544 ins_encode( aarch64_enc_fstlrs(src, mem) );
7545
7546 ins_pipe(pipe_class_memory);
7547 %}
7548
7549 // TODO
7550 // implement storeImmF0 and storeFImmPacked
7551
7552 // Store Double
7553 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7554 %{
7555 match(Set mem (StoreD mem src));
7556
7557 ins_cost(VOLATILE_REF_COST);
7558 format %{ "stlrd $src, $mem\t# double" %}
7559
7560 ins_encode( aarch64_enc_fstlrd(src, mem) );
7561
7562 ins_pipe(pipe_class_memory);
7563 %}
7564
7565 // ---------------- end of volatile loads and stores ----------------
7566
7567 instruct cacheWB(indirect addr)
7568 %{
7569 predicate(VM_Version::supports_data_cache_line_flush());
7570 match(CacheWB addr);
7571
7572 ins_cost(100);
7573 format %{"cache wb $addr" %}
7574 ins_encode %{
7575 assert($addr->index_position() < 0, "should be");
7576 assert($addr$$disp == 0, "should be");
7577 __ cache_wb(Address($addr$$base$$Register, 0));
7578 %}
7579 ins_pipe(pipe_slow); // XXX
7580 %}
7581
7582 instruct cacheWBPreSync()
7583 %{
7584 predicate(VM_Version::supports_data_cache_line_flush());
7585 match(CacheWBPreSync);
7586
7587 ins_cost(100);
7588 format %{"cache wb presync" %}
7589 ins_encode %{
7590 __ cache_wbsync(true);
7591 %}
7592 ins_pipe(pipe_slow); // XXX
7593 %}
7594
7595 instruct cacheWBPostSync()
7596 %{
7597 predicate(VM_Version::supports_data_cache_line_flush());
7598 match(CacheWBPostSync);
7599
7600 ins_cost(100);
7601 format %{"cache wb postsync" %}
7602 ins_encode %{
7603 __ cache_wbsync(false);
7604 %}
7605 ins_pipe(pipe_slow); // XXX
7606 %}
7607
7608 // ============================================================================
7609 // BSWAP Instructions
7610
7611 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7612 match(Set dst (ReverseBytesI src));
7613
7614 ins_cost(INSN_COST);
7615 format %{ "revw $dst, $src" %}
7616
7617 ins_encode %{
7618 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7619 %}
7620
7621 ins_pipe(ialu_reg);
7622 %}
7623
7624 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7625 match(Set dst (ReverseBytesL src));
7626
7627 ins_cost(INSN_COST);
7628 format %{ "rev $dst, $src" %}
7629
7630 ins_encode %{
7631 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7632 %}
7633
7634 ins_pipe(ialu_reg);
7635 %}
7636
7637 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7638 match(Set dst (ReverseBytesUS src));
7639
7640 ins_cost(INSN_COST);
7641 format %{ "rev16w $dst, $src" %}
7642
7643 ins_encode %{
7644 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7645 %}
7646
7647 ins_pipe(ialu_reg);
7648 %}
7649
7650 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7651 match(Set dst (ReverseBytesS src));
7652
7653 ins_cost(INSN_COST);
7654 format %{ "rev16w $dst, $src\n\t"
7655 "sbfmw $dst, $dst, #0, #15" %}
7656
7657 ins_encode %{
7658 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7659 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7660 %}
7661
7662 ins_pipe(ialu_reg);
7663 %}
7664
7665 // ============================================================================
7666 // Zero Count Instructions
7667
7668 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7669 match(Set dst (CountLeadingZerosI src));
7670
7671 ins_cost(INSN_COST);
7672 format %{ "clzw $dst, $src" %}
7673 ins_encode %{
7674 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7675 %}
7676
7677 ins_pipe(ialu_reg);
7678 %}
7679
7680 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7681 match(Set dst (CountLeadingZerosL src));
7682
7683 ins_cost(INSN_COST);
7684 format %{ "clz $dst, $src" %}
7685 ins_encode %{
7686 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7687 %}
7688
7689 ins_pipe(ialu_reg);
7690 %}
7691
7692 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7693 match(Set dst (CountTrailingZerosI src));
7694
7695 ins_cost(INSN_COST * 2);
7696 format %{ "rbitw $dst, $src\n\t"
7697 "clzw $dst, $dst" %}
7698 ins_encode %{
7699 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7700 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7701 %}
7702
7703 ins_pipe(ialu_reg);
7704 %}
7705
7706 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7707 match(Set dst (CountTrailingZerosL src));
7708
7709 ins_cost(INSN_COST * 2);
7710 format %{ "rbit $dst, $src\n\t"
7711 "clz $dst, $dst" %}
7712 ins_encode %{
7713 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7714 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7715 %}
7716
7717 ins_pipe(ialu_reg);
7718 %}
7719
7720 //---------- Population Count Instructions -------------------------------------
7721 //
7722
7723 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7724 match(Set dst (PopCountI src));
7725 effect(TEMP tmp);
7726 ins_cost(INSN_COST * 13);
7727
7728 format %{ "movw $src, $src\n\t"
7729 "mov $tmp, $src\t# vector (1D)\n\t"
7730 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7731 "addv $tmp, $tmp\t# vector (8B)\n\t"
7732 "mov $dst, $tmp\t# vector (1D)" %}
7733 ins_encode %{
7734 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7735 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7736 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7737 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7738 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7739 %}
7740
7741 ins_pipe(pipe_class_default);
7742 %}
7743
7744 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7745 match(Set dst (PopCountI (LoadI mem)));
7746 effect(TEMP tmp);
7747 ins_cost(INSN_COST * 13);
7748
7749 format %{ "ldrs $tmp, $mem\n\t"
7750 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7751 "addv $tmp, $tmp\t# vector (8B)\n\t"
7752 "mov $dst, $tmp\t# vector (1D)" %}
7753 ins_encode %{
7754 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7755 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7756 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7757 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7758 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7759 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7760 %}
7761
7762 ins_pipe(pipe_class_default);
7763 %}
7764
7765 // Note: Long.bitCount(long) returns an int.
7766 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7767 match(Set dst (PopCountL src));
7768 effect(TEMP tmp);
7769 ins_cost(INSN_COST * 13);
7770
7771 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7772 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7773 "addv $tmp, $tmp\t# vector (8B)\n\t"
7774 "mov $dst, $tmp\t# vector (1D)" %}
7775 ins_encode %{
7776 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7777 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7778 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7779 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7780 %}
7781
7782 ins_pipe(pipe_class_default);
7783 %}
7784
7785 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7786 match(Set dst (PopCountL (LoadL mem)));
7787 effect(TEMP tmp);
7788 ins_cost(INSN_COST * 13);
7789
7790 format %{ "ldrd $tmp, $mem\n\t"
7791 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7792 "addv $tmp, $tmp\t# vector (8B)\n\t"
7793 "mov $dst, $tmp\t# vector (1D)" %}
7794 ins_encode %{
7795 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7796 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7797 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7798 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7799 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7800 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7801 %}
7802
7803 ins_pipe(pipe_class_default);
7804 %}
7805
7806 // ============================================================================
7807 // VerifyVectorAlignment Instruction
7808
7809 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7810 match(Set addr (VerifyVectorAlignment addr mask));
7811 effect(KILL cr);
7812 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7813 ins_encode %{
7814 Label Lskip;
7815 // check if masked bits of addr are zero
7816 __ tst($addr$$Register, $mask$$constant);
7817 __ br(Assembler::EQ, Lskip);
7818 __ stop("verify_vector_alignment found a misaligned vector memory access");
7819 __ bind(Lskip);
7820 %}
7821 ins_pipe(pipe_slow);
7822 %}
7823
7824 // ============================================================================
7825 // MemBar Instruction
7826
7827 instruct load_fence() %{
7828 match(LoadFence);
7829 ins_cost(VOLATILE_REF_COST);
7830
7831 format %{ "load_fence" %}
7832
7833 ins_encode %{
7834 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7835 %}
7836 ins_pipe(pipe_serial);
7837 %}
7838
7839 instruct unnecessary_membar_acquire() %{
7840 predicate(unnecessary_acquire(n));
7841 match(MemBarAcquire);
7842 ins_cost(0);
7843
7844 format %{ "membar_acquire (elided)" %}
7845
7846 ins_encode %{
7847 __ block_comment("membar_acquire (elided)");
7848 %}
7849
7850 ins_pipe(pipe_class_empty);
7851 %}
7852
7853 instruct membar_acquire() %{
7854 match(MemBarAcquire);
7855 ins_cost(VOLATILE_REF_COST);
7856
7857 format %{ "membar_acquire\n\t"
7858 "dmb ishld" %}
7859
7860 ins_encode %{
7861 __ block_comment("membar_acquire");
7862 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7863 %}
7864
7865 ins_pipe(pipe_serial);
7866 %}
7867
7868
7869 instruct membar_acquire_lock() %{
7870 match(MemBarAcquireLock);
7871 ins_cost(VOLATILE_REF_COST);
7872
7873 format %{ "membar_acquire_lock (elided)" %}
7874
7875 ins_encode %{
7876 __ block_comment("membar_acquire_lock (elided)");
7877 %}
7878
7879 ins_pipe(pipe_serial);
7880 %}
7881
7882 instruct store_fence() %{
7883 match(StoreFence);
7884 ins_cost(VOLATILE_REF_COST);
7885
7886 format %{ "store_fence" %}
7887
7888 ins_encode %{
7889 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7890 %}
7891 ins_pipe(pipe_serial);
7892 %}
7893
7894 instruct unnecessary_membar_release() %{
7895 predicate(unnecessary_release(n));
7896 match(MemBarRelease);
7897 ins_cost(0);
7898
7899 format %{ "membar_release (elided)" %}
7900
7901 ins_encode %{
7902 __ block_comment("membar_release (elided)");
7903 %}
7904 ins_pipe(pipe_serial);
7905 %}
7906
7907 instruct membar_release() %{
7908 match(MemBarRelease);
7909 ins_cost(VOLATILE_REF_COST);
7910
7911 format %{ "membar_release\n\t"
7912 "dmb ishst\n\tdmb ishld" %}
7913
7914 ins_encode %{
7915 __ block_comment("membar_release");
7916 // These will be merged if AlwaysMergeDMB is enabled.
7917 __ membar(Assembler::StoreStore);
7918 __ membar(Assembler::LoadStore);
7919 %}
7920 ins_pipe(pipe_serial);
7921 %}
7922
7923 instruct membar_storestore() %{
7924 match(MemBarStoreStore);
7925 match(StoreStoreFence);
7926 ins_cost(VOLATILE_REF_COST);
7927
7928 format %{ "MEMBAR-store-store" %}
7929
7930 ins_encode %{
7931 __ membar(Assembler::StoreStore);
7932 %}
7933 ins_pipe(pipe_serial);
7934 %}
7935
7936 instruct membar_release_lock() %{
7937 match(MemBarReleaseLock);
7938 ins_cost(VOLATILE_REF_COST);
7939
7940 format %{ "membar_release_lock (elided)" %}
7941
7942 ins_encode %{
7943 __ block_comment("membar_release_lock (elided)");
7944 %}
7945
7946 ins_pipe(pipe_serial);
7947 %}
7948
7949 instruct unnecessary_membar_volatile() %{
7950 predicate(unnecessary_volatile(n));
7951 match(MemBarVolatile);
7952 ins_cost(0);
7953
7954 format %{ "membar_volatile (elided)" %}
7955
7956 ins_encode %{
7957 __ block_comment("membar_volatile (elided)");
7958 %}
7959
7960 ins_pipe(pipe_serial);
7961 %}
7962
7963 instruct membar_volatile() %{
7964 match(MemBarVolatile);
7965 ins_cost(VOLATILE_REF_COST*100);
7966
7967 format %{ "membar_volatile\n\t"
7968 "dmb ish"%}
7969
7970 ins_encode %{
7971 __ block_comment("membar_volatile");
7972 __ membar(Assembler::StoreLoad);
7973 %}
7974
7975 ins_pipe(pipe_serial);
7976 %}
7977
7978 // ============================================================================
7979 // Cast/Convert Instructions
7980
7981 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7982 match(Set dst (CastX2P src));
7983
7984 ins_cost(INSN_COST);
7985 format %{ "mov $dst, $src\t# long -> ptr" %}
7986
7987 ins_encode %{
7988 if ($dst$$reg != $src$$reg) {
7989 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7990 }
7991 %}
7992
7993 ins_pipe(ialu_reg);
7994 %}
7995
7996 instruct castP2X(iRegLNoSp dst, iRegP src) %{
7997 match(Set dst (CastP2X src));
7998
7999 ins_cost(INSN_COST);
8000 format %{ "mov $dst, $src\t# ptr -> long" %}
8001
8002 ins_encode %{
8003 if ($dst$$reg != $src$$reg) {
8004 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8005 }
8006 %}
8007
8008 ins_pipe(ialu_reg);
8009 %}
8010
8011 // Convert oop into int for vectors alignment masking
8012 instruct convP2I(iRegINoSp dst, iRegP src) %{
8013 match(Set dst (ConvL2I (CastP2X src)));
8014
8015 ins_cost(INSN_COST);
8016 format %{ "movw $dst, $src\t# ptr -> int" %}
8017 ins_encode %{
8018 __ movw($dst$$Register, $src$$Register);
8019 %}
8020
8021 ins_pipe(ialu_reg);
8022 %}
8023
8024 // Convert compressed oop into int for vectors alignment masking
8025 // in case of 32bit oops (heap < 4Gb).
8026 instruct convN2I(iRegINoSp dst, iRegN src)
8027 %{
8028 predicate(CompressedOops::shift() == 0);
8029 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8030
8031 ins_cost(INSN_COST);
8032 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8033 ins_encode %{
8034 __ movw($dst$$Register, $src$$Register);
8035 %}
8036
8037 ins_pipe(ialu_reg);
8038 %}
8039
8040
8041 // Convert oop pointer into compressed form
8042 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8043 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8044 match(Set dst (EncodeP src));
8045 effect(KILL cr);
8046 ins_cost(INSN_COST * 3);
8047 format %{ "encode_heap_oop $dst, $src" %}
8048 ins_encode %{
8049 Register s = $src$$Register;
8050 Register d = $dst$$Register;
8051 __ encode_heap_oop(d, s);
8052 %}
8053 ins_pipe(ialu_reg);
8054 %}
8055
8056 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8057 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8058 match(Set dst (EncodeP src));
8059 ins_cost(INSN_COST * 3);
8060 format %{ "encode_heap_oop_not_null $dst, $src" %}
8061 ins_encode %{
8062 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8063 %}
8064 ins_pipe(ialu_reg);
8065 %}
8066
8067 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8068 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8069 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8070 match(Set dst (DecodeN src));
8071 ins_cost(INSN_COST * 3);
8072 format %{ "decode_heap_oop $dst, $src" %}
8073 ins_encode %{
8074 Register s = $src$$Register;
8075 Register d = $dst$$Register;
8076 __ decode_heap_oop(d, s);
8077 %}
8078 ins_pipe(ialu_reg);
8079 %}
8080
8081 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8082 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8083 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8084 match(Set dst (DecodeN src));
8085 ins_cost(INSN_COST * 3);
8086 format %{ "decode_heap_oop_not_null $dst, $src" %}
8087 ins_encode %{
8088 Register s = $src$$Register;
8089 Register d = $dst$$Register;
8090 __ decode_heap_oop_not_null(d, s);
8091 %}
8092 ins_pipe(ialu_reg);
8093 %}
8094
8095 // n.b. AArch64 implementations of encode_klass_not_null and
8096 // decode_klass_not_null do not modify the flags register so, unlike
8097 // Intel, we don't kill CR as a side effect here
8098
8099 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8100 match(Set dst (EncodePKlass src));
8101
8102 ins_cost(INSN_COST * 3);
8103 format %{ "encode_klass_not_null $dst,$src" %}
8104
8105 ins_encode %{
8106 Register src_reg = as_Register($src$$reg);
8107 Register dst_reg = as_Register($dst$$reg);
8108 __ encode_klass_not_null(dst_reg, src_reg);
8109 %}
8110
8111 ins_pipe(ialu_reg);
8112 %}
8113
8114 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8115 match(Set dst (DecodeNKlass src));
8116
8117 ins_cost(INSN_COST * 3);
8118 format %{ "decode_klass_not_null $dst,$src" %}
8119
8120 ins_encode %{
8121 Register src_reg = as_Register($src$$reg);
8122 Register dst_reg = as_Register($dst$$reg);
8123 if (dst_reg != src_reg) {
8124 __ decode_klass_not_null(dst_reg, src_reg);
8125 } else {
8126 __ decode_klass_not_null(dst_reg);
8127 }
8128 %}
8129
8130 ins_pipe(ialu_reg);
8131 %}
8132
8133 instruct checkCastPP(iRegPNoSp dst)
8134 %{
8135 match(Set dst (CheckCastPP dst));
8136
8137 size(0);
8138 format %{ "# checkcastPP of $dst" %}
8139 ins_encode(/* empty encoding */);
8140 ins_pipe(pipe_class_empty);
8141 %}
8142
8143 instruct castPP(iRegPNoSp dst)
8144 %{
8145 match(Set dst (CastPP dst));
8146
8147 size(0);
8148 format %{ "# castPP of $dst" %}
8149 ins_encode(/* empty encoding */);
8150 ins_pipe(pipe_class_empty);
8151 %}
8152
8153 instruct castII(iRegI dst)
8154 %{
8155 match(Set dst (CastII dst));
8156
8157 size(0);
8158 format %{ "# castII of $dst" %}
8159 ins_encode(/* empty encoding */);
8160 ins_cost(0);
8161 ins_pipe(pipe_class_empty);
8162 %}
8163
8164 instruct castLL(iRegL dst)
8165 %{
8166 match(Set dst (CastLL dst));
8167
8168 size(0);
8169 format %{ "# castLL of $dst" %}
8170 ins_encode(/* empty encoding */);
8171 ins_cost(0);
8172 ins_pipe(pipe_class_empty);
8173 %}
8174
8175 instruct castFF(vRegF dst)
8176 %{
8177 match(Set dst (CastFF dst));
8178
8179 size(0);
8180 format %{ "# castFF of $dst" %}
8181 ins_encode(/* empty encoding */);
8182 ins_cost(0);
8183 ins_pipe(pipe_class_empty);
8184 %}
8185
8186 instruct castDD(vRegD dst)
8187 %{
8188 match(Set dst (CastDD dst));
8189
8190 size(0);
8191 format %{ "# castDD of $dst" %}
8192 ins_encode(/* empty encoding */);
8193 ins_cost(0);
8194 ins_pipe(pipe_class_empty);
8195 %}
8196
8197 instruct castVV(vReg dst)
8198 %{
8199 match(Set dst (CastVV dst));
8200
8201 size(0);
8202 format %{ "# castVV of $dst" %}
8203 ins_encode(/* empty encoding */);
8204 ins_cost(0);
8205 ins_pipe(pipe_class_empty);
8206 %}
8207
8208 instruct castVVMask(pRegGov dst)
8209 %{
8210 match(Set dst (CastVV dst));
8211
8212 size(0);
8213 format %{ "# castVV of $dst" %}
8214 ins_encode(/* empty encoding */);
8215 ins_cost(0);
8216 ins_pipe(pipe_class_empty);
8217 %}
8218
8219 // ============================================================================
8220 // Atomic operation instructions
8221 //
8222
8223 // standard CompareAndSwapX when we are using barriers
8224 // these have higher priority than the rules selected by a predicate
8225
8226 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8227 // can't match them
8228
8229 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8230
8231 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8232 ins_cost(2 * VOLATILE_REF_COST);
8233
8234 effect(KILL cr);
8235
8236 format %{
8237 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8238 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8239 %}
8240
8241 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8242 aarch64_enc_cset_eq(res));
8243
8244 ins_pipe(pipe_slow);
8245 %}
8246
8247 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8248
8249 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8250 ins_cost(2 * VOLATILE_REF_COST);
8251
8252 effect(KILL cr);
8253
8254 format %{
8255 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8256 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8257 %}
8258
8259 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8260 aarch64_enc_cset_eq(res));
8261
8262 ins_pipe(pipe_slow);
8263 %}
8264
8265 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8266
8267 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8268 ins_cost(2 * VOLATILE_REF_COST);
8269
8270 effect(KILL cr);
8271
8272 format %{
8273 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8274 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8275 %}
8276
8277 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8278 aarch64_enc_cset_eq(res));
8279
8280 ins_pipe(pipe_slow);
8281 %}
8282
8283 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8284
8285 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8286 ins_cost(2 * VOLATILE_REF_COST);
8287
8288 effect(KILL cr);
8289
8290 format %{
8291 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8292 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8293 %}
8294
8295 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8296 aarch64_enc_cset_eq(res));
8297
8298 ins_pipe(pipe_slow);
8299 %}
8300
8301 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8302
8303 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8304 predicate(n->as_LoadStore()->barrier_data() == 0);
8305 ins_cost(2 * VOLATILE_REF_COST);
8306
8307 effect(KILL cr);
8308
8309 format %{
8310 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8311 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8312 %}
8313
8314 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8315 aarch64_enc_cset_eq(res));
8316
8317 ins_pipe(pipe_slow);
8318 %}
8319
8320 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8321
8322 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8323 predicate(n->as_LoadStore()->barrier_data() == 0);
8324 ins_cost(2 * VOLATILE_REF_COST);
8325
8326 effect(KILL cr);
8327
8328 format %{
8329 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8330 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8331 %}
8332
8333 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8334 aarch64_enc_cset_eq(res));
8335
8336 ins_pipe(pipe_slow);
8337 %}
8338
8339 // alternative CompareAndSwapX when we are eliding barriers
8340
8341 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8342
8343 predicate(needs_acquiring_load_exclusive(n));
8344 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8345 ins_cost(VOLATILE_REF_COST);
8346
8347 effect(KILL cr);
8348
8349 format %{
8350 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8351 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8352 %}
8353
8354 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8355 aarch64_enc_cset_eq(res));
8356
8357 ins_pipe(pipe_slow);
8358 %}
8359
8360 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8361
8362 predicate(needs_acquiring_load_exclusive(n));
8363 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8364 ins_cost(VOLATILE_REF_COST);
8365
8366 effect(KILL cr);
8367
8368 format %{
8369 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8370 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8371 %}
8372
8373 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8374 aarch64_enc_cset_eq(res));
8375
8376 ins_pipe(pipe_slow);
8377 %}
8378
8379 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8380
8381 predicate(needs_acquiring_load_exclusive(n));
8382 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8383 ins_cost(VOLATILE_REF_COST);
8384
8385 effect(KILL cr);
8386
8387 format %{
8388 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8389 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8390 %}
8391
8392 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8393 aarch64_enc_cset_eq(res));
8394
8395 ins_pipe(pipe_slow);
8396 %}
8397
8398 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8399
8400 predicate(needs_acquiring_load_exclusive(n));
8401 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8402 ins_cost(VOLATILE_REF_COST);
8403
8404 effect(KILL cr);
8405
8406 format %{
8407 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8408 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8409 %}
8410
8411 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8412 aarch64_enc_cset_eq(res));
8413
8414 ins_pipe(pipe_slow);
8415 %}
8416
8417 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8418
8419 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8420 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8421 ins_cost(VOLATILE_REF_COST);
8422
8423 effect(KILL cr);
8424
8425 format %{
8426 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8427 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8428 %}
8429
8430 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8431 aarch64_enc_cset_eq(res));
8432
8433 ins_pipe(pipe_slow);
8434 %}
8435
8436 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8437
8438 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8439 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8440 ins_cost(VOLATILE_REF_COST);
8441
8442 effect(KILL cr);
8443
8444 format %{
8445 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8446 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8447 %}
8448
8449 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8450 aarch64_enc_cset_eq(res));
8451
8452 ins_pipe(pipe_slow);
8453 %}
8454
8455
8456 // ---------------------------------------------------------------------
8457
8458 // BEGIN This section of the file is automatically generated. Do not edit --------------
8459
8460 // Sundry CAS operations. Note that release is always true,
8461 // regardless of the memory ordering of the CAS. This is because we
8462 // need the volatile case to be sequentially consistent but there is
8463 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8464 // can't check the type of memory ordering here, so we always emit a
8465 // STLXR.
8466
8467 // This section is generated from cas.m4
8468
8469
8470 // This pattern is generated automatically from cas.m4.
8471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8472 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8473 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8474 ins_cost(2 * VOLATILE_REF_COST);
8475 effect(TEMP_DEF res, KILL cr);
8476 format %{
8477 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8478 %}
8479 ins_encode %{
8480 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8481 Assembler::byte, /*acquire*/ false, /*release*/ true,
8482 /*weak*/ false, $res$$Register);
8483 __ sxtbw($res$$Register, $res$$Register);
8484 %}
8485 ins_pipe(pipe_slow);
8486 %}
8487
8488 // This pattern is generated automatically from cas.m4.
8489 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8490 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8491 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8492 ins_cost(2 * VOLATILE_REF_COST);
8493 effect(TEMP_DEF res, KILL cr);
8494 format %{
8495 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8496 %}
8497 ins_encode %{
8498 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8499 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8500 /*weak*/ false, $res$$Register);
8501 __ sxthw($res$$Register, $res$$Register);
8502 %}
8503 ins_pipe(pipe_slow);
8504 %}
8505
8506 // This pattern is generated automatically from cas.m4.
8507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8508 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8509 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8510 ins_cost(2 * VOLATILE_REF_COST);
8511 effect(TEMP_DEF res, KILL cr);
8512 format %{
8513 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8514 %}
8515 ins_encode %{
8516 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8517 Assembler::word, /*acquire*/ false, /*release*/ true,
8518 /*weak*/ false, $res$$Register);
8519 %}
8520 ins_pipe(pipe_slow);
8521 %}
8522
8523 // This pattern is generated automatically from cas.m4.
8524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8525 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8526 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8527 ins_cost(2 * VOLATILE_REF_COST);
8528 effect(TEMP_DEF res, KILL cr);
8529 format %{
8530 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8531 %}
8532 ins_encode %{
8533 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8534 Assembler::xword, /*acquire*/ false, /*release*/ true,
8535 /*weak*/ false, $res$$Register);
8536 %}
8537 ins_pipe(pipe_slow);
8538 %}
8539
8540 // This pattern is generated automatically from cas.m4.
8541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8542 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8543 predicate(n->as_LoadStore()->barrier_data() == 0);
8544 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8545 ins_cost(2 * VOLATILE_REF_COST);
8546 effect(TEMP_DEF res, KILL cr);
8547 format %{
8548 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8549 %}
8550 ins_encode %{
8551 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8552 Assembler::word, /*acquire*/ false, /*release*/ true,
8553 /*weak*/ false, $res$$Register);
8554 %}
8555 ins_pipe(pipe_slow);
8556 %}
8557
8558 // This pattern is generated automatically from cas.m4.
8559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8560 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8561 predicate(n->as_LoadStore()->barrier_data() == 0);
8562 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8563 ins_cost(2 * VOLATILE_REF_COST);
8564 effect(TEMP_DEF res, KILL cr);
8565 format %{
8566 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8567 %}
8568 ins_encode %{
8569 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8570 Assembler::xword, /*acquire*/ false, /*release*/ true,
8571 /*weak*/ false, $res$$Register);
8572 %}
8573 ins_pipe(pipe_slow);
8574 %}
8575
8576 // This pattern is generated automatically from cas.m4.
8577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8578 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8579 predicate(needs_acquiring_load_exclusive(n));
8580 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8581 ins_cost(VOLATILE_REF_COST);
8582 effect(TEMP_DEF res, KILL cr);
8583 format %{
8584 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8585 %}
8586 ins_encode %{
8587 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8588 Assembler::byte, /*acquire*/ true, /*release*/ true,
8589 /*weak*/ false, $res$$Register);
8590 __ sxtbw($res$$Register, $res$$Register);
8591 %}
8592 ins_pipe(pipe_slow);
8593 %}
8594
8595 // This pattern is generated automatically from cas.m4.
8596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8597 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8598 predicate(needs_acquiring_load_exclusive(n));
8599 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8600 ins_cost(VOLATILE_REF_COST);
8601 effect(TEMP_DEF res, KILL cr);
8602 format %{
8603 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8604 %}
8605 ins_encode %{
8606 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8607 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8608 /*weak*/ false, $res$$Register);
8609 __ sxthw($res$$Register, $res$$Register);
8610 %}
8611 ins_pipe(pipe_slow);
8612 %}
8613
8614 // This pattern is generated automatically from cas.m4.
8615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8616 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8617 predicate(needs_acquiring_load_exclusive(n));
8618 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8619 ins_cost(VOLATILE_REF_COST);
8620 effect(TEMP_DEF res, KILL cr);
8621 format %{
8622 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8623 %}
8624 ins_encode %{
8625 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8626 Assembler::word, /*acquire*/ true, /*release*/ true,
8627 /*weak*/ false, $res$$Register);
8628 %}
8629 ins_pipe(pipe_slow);
8630 %}
8631
8632 // This pattern is generated automatically from cas.m4.
8633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8634 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8635 predicate(needs_acquiring_load_exclusive(n));
8636 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8637 ins_cost(VOLATILE_REF_COST);
8638 effect(TEMP_DEF res, KILL cr);
8639 format %{
8640 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8641 %}
8642 ins_encode %{
8643 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8644 Assembler::xword, /*acquire*/ true, /*release*/ true,
8645 /*weak*/ false, $res$$Register);
8646 %}
8647 ins_pipe(pipe_slow);
8648 %}
8649
8650 // This pattern is generated automatically from cas.m4.
8651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8652 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8653 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8654 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8655 ins_cost(VOLATILE_REF_COST);
8656 effect(TEMP_DEF res, KILL cr);
8657 format %{
8658 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8659 %}
8660 ins_encode %{
8661 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8662 Assembler::word, /*acquire*/ true, /*release*/ true,
8663 /*weak*/ false, $res$$Register);
8664 %}
8665 ins_pipe(pipe_slow);
8666 %}
8667
8668 // This pattern is generated automatically from cas.m4.
8669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8670 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8671 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8672 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8673 ins_cost(VOLATILE_REF_COST);
8674 effect(TEMP_DEF res, KILL cr);
8675 format %{
8676 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8677 %}
8678 ins_encode %{
8679 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8680 Assembler::xword, /*acquire*/ true, /*release*/ true,
8681 /*weak*/ false, $res$$Register);
8682 %}
8683 ins_pipe(pipe_slow);
8684 %}
8685
8686 // This pattern is generated automatically from cas.m4.
8687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8688 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8689 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8690 ins_cost(2 * VOLATILE_REF_COST);
8691 effect(KILL cr);
8692 format %{
8693 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8694 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8695 %}
8696 ins_encode %{
8697 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8698 Assembler::byte, /*acquire*/ false, /*release*/ true,
8699 /*weak*/ true, noreg);
8700 __ csetw($res$$Register, Assembler::EQ);
8701 %}
8702 ins_pipe(pipe_slow);
8703 %}
8704
8705 // This pattern is generated automatically from cas.m4.
8706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8707 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8708 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8709 ins_cost(2 * VOLATILE_REF_COST);
8710 effect(KILL cr);
8711 format %{
8712 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8713 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8714 %}
8715 ins_encode %{
8716 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8717 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8718 /*weak*/ true, noreg);
8719 __ csetw($res$$Register, Assembler::EQ);
8720 %}
8721 ins_pipe(pipe_slow);
8722 %}
8723
8724 // This pattern is generated automatically from cas.m4.
8725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8726 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8727 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8728 ins_cost(2 * VOLATILE_REF_COST);
8729 effect(KILL cr);
8730 format %{
8731 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8732 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8733 %}
8734 ins_encode %{
8735 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8736 Assembler::word, /*acquire*/ false, /*release*/ true,
8737 /*weak*/ true, noreg);
8738 __ csetw($res$$Register, Assembler::EQ);
8739 %}
8740 ins_pipe(pipe_slow);
8741 %}
8742
8743 // This pattern is generated automatically from cas.m4.
8744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8745 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8746 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8747 ins_cost(2 * VOLATILE_REF_COST);
8748 effect(KILL cr);
8749 format %{
8750 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8751 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8752 %}
8753 ins_encode %{
8754 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8755 Assembler::xword, /*acquire*/ false, /*release*/ true,
8756 /*weak*/ true, noreg);
8757 __ csetw($res$$Register, Assembler::EQ);
8758 %}
8759 ins_pipe(pipe_slow);
8760 %}
8761
8762 // This pattern is generated automatically from cas.m4.
8763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8764 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8765 predicate(n->as_LoadStore()->barrier_data() == 0);
8766 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8767 ins_cost(2 * VOLATILE_REF_COST);
8768 effect(KILL cr);
8769 format %{
8770 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8771 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8772 %}
8773 ins_encode %{
8774 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8775 Assembler::word, /*acquire*/ false, /*release*/ true,
8776 /*weak*/ true, noreg);
8777 __ csetw($res$$Register, Assembler::EQ);
8778 %}
8779 ins_pipe(pipe_slow);
8780 %}
8781
8782 // This pattern is generated automatically from cas.m4.
8783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8784 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8785 predicate(n->as_LoadStore()->barrier_data() == 0);
8786 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8787 ins_cost(2 * VOLATILE_REF_COST);
8788 effect(KILL cr);
8789 format %{
8790 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8791 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8792 %}
8793 ins_encode %{
8794 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8795 Assembler::xword, /*acquire*/ false, /*release*/ true,
8796 /*weak*/ true, noreg);
8797 __ csetw($res$$Register, Assembler::EQ);
8798 %}
8799 ins_pipe(pipe_slow);
8800 %}
8801
8802 // This pattern is generated automatically from cas.m4.
8803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8804 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8805 predicate(needs_acquiring_load_exclusive(n));
8806 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8807 ins_cost(VOLATILE_REF_COST);
8808 effect(KILL cr);
8809 format %{
8810 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8811 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8812 %}
8813 ins_encode %{
8814 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8815 Assembler::byte, /*acquire*/ true, /*release*/ true,
8816 /*weak*/ true, noreg);
8817 __ csetw($res$$Register, Assembler::EQ);
8818 %}
8819 ins_pipe(pipe_slow);
8820 %}
8821
8822 // This pattern is generated automatically from cas.m4.
8823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8824 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8825 predicate(needs_acquiring_load_exclusive(n));
8826 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8827 ins_cost(VOLATILE_REF_COST);
8828 effect(KILL cr);
8829 format %{
8830 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8831 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8832 %}
8833 ins_encode %{
8834 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8835 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8836 /*weak*/ true, noreg);
8837 __ csetw($res$$Register, Assembler::EQ);
8838 %}
8839 ins_pipe(pipe_slow);
8840 %}
8841
8842 // This pattern is generated automatically from cas.m4.
8843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8844 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8845 predicate(needs_acquiring_load_exclusive(n));
8846 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8847 ins_cost(VOLATILE_REF_COST);
8848 effect(KILL cr);
8849 format %{
8850 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8851 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8852 %}
8853 ins_encode %{
8854 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8855 Assembler::word, /*acquire*/ true, /*release*/ true,
8856 /*weak*/ true, noreg);
8857 __ csetw($res$$Register, Assembler::EQ);
8858 %}
8859 ins_pipe(pipe_slow);
8860 %}
8861
8862 // This pattern is generated automatically from cas.m4.
8863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8864 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8865 predicate(needs_acquiring_load_exclusive(n));
8866 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8867 ins_cost(VOLATILE_REF_COST);
8868 effect(KILL cr);
8869 format %{
8870 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8871 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8872 %}
8873 ins_encode %{
8874 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8875 Assembler::xword, /*acquire*/ true, /*release*/ true,
8876 /*weak*/ true, noreg);
8877 __ csetw($res$$Register, Assembler::EQ);
8878 %}
8879 ins_pipe(pipe_slow);
8880 %}
8881
8882 // This pattern is generated automatically from cas.m4.
8883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8884 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8885 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8886 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8887 ins_cost(VOLATILE_REF_COST);
8888 effect(KILL cr);
8889 format %{
8890 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8891 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8892 %}
8893 ins_encode %{
8894 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8895 Assembler::word, /*acquire*/ true, /*release*/ true,
8896 /*weak*/ true, noreg);
8897 __ csetw($res$$Register, Assembler::EQ);
8898 %}
8899 ins_pipe(pipe_slow);
8900 %}
8901
8902 // This pattern is generated automatically from cas.m4.
8903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8904 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8905 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8906 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8907 ins_cost(VOLATILE_REF_COST);
8908 effect(KILL cr);
8909 format %{
8910 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8911 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8912 %}
8913 ins_encode %{
8914 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8915 Assembler::xword, /*acquire*/ true, /*release*/ true,
8916 /*weak*/ true, noreg);
8917 __ csetw($res$$Register, Assembler::EQ);
8918 %}
8919 ins_pipe(pipe_slow);
8920 %}
8921
8922 // END This section of the file is automatically generated. Do not edit --------------
8923 // ---------------------------------------------------------------------
8924
8925 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8926 match(Set prev (GetAndSetI mem newv));
8927 ins_cost(2 * VOLATILE_REF_COST);
8928 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8929 ins_encode %{
8930 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8931 %}
8932 ins_pipe(pipe_serial);
8933 %}
8934
8935 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8936 match(Set prev (GetAndSetL mem newv));
8937 ins_cost(2 * VOLATILE_REF_COST);
8938 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8939 ins_encode %{
8940 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8941 %}
8942 ins_pipe(pipe_serial);
8943 %}
8944
8945 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8946 predicate(n->as_LoadStore()->barrier_data() == 0);
8947 match(Set prev (GetAndSetN mem newv));
8948 ins_cost(2 * VOLATILE_REF_COST);
8949 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8950 ins_encode %{
8951 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8952 %}
8953 ins_pipe(pipe_serial);
8954 %}
8955
8956 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
8957 predicate(n->as_LoadStore()->barrier_data() == 0);
8958 match(Set prev (GetAndSetP mem newv));
8959 ins_cost(2 * VOLATILE_REF_COST);
8960 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8961 ins_encode %{
8962 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8963 %}
8964 ins_pipe(pipe_serial);
8965 %}
8966
8967 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
8968 predicate(needs_acquiring_load_exclusive(n));
8969 match(Set prev (GetAndSetI mem newv));
8970 ins_cost(VOLATILE_REF_COST);
8971 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8972 ins_encode %{
8973 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8974 %}
8975 ins_pipe(pipe_serial);
8976 %}
8977
8978 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
8979 predicate(needs_acquiring_load_exclusive(n));
8980 match(Set prev (GetAndSetL mem newv));
8981 ins_cost(VOLATILE_REF_COST);
8982 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8983 ins_encode %{
8984 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8985 %}
8986 ins_pipe(pipe_serial);
8987 %}
8988
8989 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
8990 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8991 match(Set prev (GetAndSetN mem newv));
8992 ins_cost(VOLATILE_REF_COST);
8993 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8994 ins_encode %{
8995 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8996 %}
8997 ins_pipe(pipe_serial);
8998 %}
8999
9000 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9001 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9002 match(Set prev (GetAndSetP mem newv));
9003 ins_cost(VOLATILE_REF_COST);
9004 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9005 ins_encode %{
9006 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9007 %}
9008 ins_pipe(pipe_serial);
9009 %}
9010
9011
9012 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9013 match(Set newval (GetAndAddL mem incr));
9014 ins_cost(2 * VOLATILE_REF_COST + 1);
9015 format %{ "get_and_addL $newval, [$mem], $incr" %}
9016 ins_encode %{
9017 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9018 %}
9019 ins_pipe(pipe_serial);
9020 %}
9021
9022 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9023 predicate(n->as_LoadStore()->result_not_used());
9024 match(Set dummy (GetAndAddL mem incr));
9025 ins_cost(2 * VOLATILE_REF_COST);
9026 format %{ "get_and_addL [$mem], $incr" %}
9027 ins_encode %{
9028 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9029 %}
9030 ins_pipe(pipe_serial);
9031 %}
9032
9033 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9034 match(Set newval (GetAndAddL mem incr));
9035 ins_cost(2 * VOLATILE_REF_COST + 1);
9036 format %{ "get_and_addL $newval, [$mem], $incr" %}
9037 ins_encode %{
9038 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9039 %}
9040 ins_pipe(pipe_serial);
9041 %}
9042
9043 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9044 predicate(n->as_LoadStore()->result_not_used());
9045 match(Set dummy (GetAndAddL mem incr));
9046 ins_cost(2 * VOLATILE_REF_COST);
9047 format %{ "get_and_addL [$mem], $incr" %}
9048 ins_encode %{
9049 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9050 %}
9051 ins_pipe(pipe_serial);
9052 %}
9053
9054 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9055 match(Set newval (GetAndAddI mem incr));
9056 ins_cost(2 * VOLATILE_REF_COST + 1);
9057 format %{ "get_and_addI $newval, [$mem], $incr" %}
9058 ins_encode %{
9059 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9060 %}
9061 ins_pipe(pipe_serial);
9062 %}
9063
9064 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9065 predicate(n->as_LoadStore()->result_not_used());
9066 match(Set dummy (GetAndAddI mem incr));
9067 ins_cost(2 * VOLATILE_REF_COST);
9068 format %{ "get_and_addI [$mem], $incr" %}
9069 ins_encode %{
9070 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9071 %}
9072 ins_pipe(pipe_serial);
9073 %}
9074
9075 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9076 match(Set newval (GetAndAddI mem incr));
9077 ins_cost(2 * VOLATILE_REF_COST + 1);
9078 format %{ "get_and_addI $newval, [$mem], $incr" %}
9079 ins_encode %{
9080 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9081 %}
9082 ins_pipe(pipe_serial);
9083 %}
9084
9085 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9086 predicate(n->as_LoadStore()->result_not_used());
9087 match(Set dummy (GetAndAddI mem incr));
9088 ins_cost(2 * VOLATILE_REF_COST);
9089 format %{ "get_and_addI [$mem], $incr" %}
9090 ins_encode %{
9091 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9092 %}
9093 ins_pipe(pipe_serial);
9094 %}
9095
9096 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9097 predicate(needs_acquiring_load_exclusive(n));
9098 match(Set newval (GetAndAddL mem incr));
9099 ins_cost(VOLATILE_REF_COST + 1);
9100 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9101 ins_encode %{
9102 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9103 %}
9104 ins_pipe(pipe_serial);
9105 %}
9106
9107 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9108 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9109 match(Set dummy (GetAndAddL mem incr));
9110 ins_cost(VOLATILE_REF_COST);
9111 format %{ "get_and_addL_acq [$mem], $incr" %}
9112 ins_encode %{
9113 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9114 %}
9115 ins_pipe(pipe_serial);
9116 %}
9117
9118 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9119 predicate(needs_acquiring_load_exclusive(n));
9120 match(Set newval (GetAndAddL mem incr));
9121 ins_cost(VOLATILE_REF_COST + 1);
9122 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9123 ins_encode %{
9124 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9125 %}
9126 ins_pipe(pipe_serial);
9127 %}
9128
9129 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9130 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9131 match(Set dummy (GetAndAddL mem incr));
9132 ins_cost(VOLATILE_REF_COST);
9133 format %{ "get_and_addL_acq [$mem], $incr" %}
9134 ins_encode %{
9135 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9136 %}
9137 ins_pipe(pipe_serial);
9138 %}
9139
9140 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9141 predicate(needs_acquiring_load_exclusive(n));
9142 match(Set newval (GetAndAddI mem incr));
9143 ins_cost(VOLATILE_REF_COST + 1);
9144 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9145 ins_encode %{
9146 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9147 %}
9148 ins_pipe(pipe_serial);
9149 %}
9150
9151 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9152 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9153 match(Set dummy (GetAndAddI mem incr));
9154 ins_cost(VOLATILE_REF_COST);
9155 format %{ "get_and_addI_acq [$mem], $incr" %}
9156 ins_encode %{
9157 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9158 %}
9159 ins_pipe(pipe_serial);
9160 %}
9161
9162 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9163 predicate(needs_acquiring_load_exclusive(n));
9164 match(Set newval (GetAndAddI mem incr));
9165 ins_cost(VOLATILE_REF_COST + 1);
9166 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9167 ins_encode %{
9168 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9169 %}
9170 ins_pipe(pipe_serial);
9171 %}
9172
9173 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9174 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9175 match(Set dummy (GetAndAddI mem incr));
9176 ins_cost(VOLATILE_REF_COST);
9177 format %{ "get_and_addI_acq [$mem], $incr" %}
9178 ins_encode %{
9179 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9180 %}
9181 ins_pipe(pipe_serial);
9182 %}
9183
9184 // Manifest a CmpU result in an integer register.
9185 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9186 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9187 %{
9188 match(Set dst (CmpU3 src1 src2));
9189 effect(KILL flags);
9190
9191 ins_cost(INSN_COST * 3);
9192 format %{
9193 "cmpw $src1, $src2\n\t"
9194 "csetw $dst, ne\n\t"
9195 "cnegw $dst, lo\t# CmpU3(reg)"
9196 %}
9197 ins_encode %{
9198 __ cmpw($src1$$Register, $src2$$Register);
9199 __ csetw($dst$$Register, Assembler::NE);
9200 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9201 %}
9202
9203 ins_pipe(pipe_class_default);
9204 %}
9205
9206 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9207 %{
9208 match(Set dst (CmpU3 src1 src2));
9209 effect(KILL flags);
9210
9211 ins_cost(INSN_COST * 3);
9212 format %{
9213 "subsw zr, $src1, $src2\n\t"
9214 "csetw $dst, ne\n\t"
9215 "cnegw $dst, lo\t# CmpU3(imm)"
9216 %}
9217 ins_encode %{
9218 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9219 __ csetw($dst$$Register, Assembler::NE);
9220 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9221 %}
9222
9223 ins_pipe(pipe_class_default);
9224 %}
9225
9226 // Manifest a CmpUL result in an integer register.
9227 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9228 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9229 %{
9230 match(Set dst (CmpUL3 src1 src2));
9231 effect(KILL flags);
9232
9233 ins_cost(INSN_COST * 3);
9234 format %{
9235 "cmp $src1, $src2\n\t"
9236 "csetw $dst, ne\n\t"
9237 "cnegw $dst, lo\t# CmpUL3(reg)"
9238 %}
9239 ins_encode %{
9240 __ cmp($src1$$Register, $src2$$Register);
9241 __ csetw($dst$$Register, Assembler::NE);
9242 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9243 %}
9244
9245 ins_pipe(pipe_class_default);
9246 %}
9247
9248 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9249 %{
9250 match(Set dst (CmpUL3 src1 src2));
9251 effect(KILL flags);
9252
9253 ins_cost(INSN_COST * 3);
9254 format %{
9255 "subs zr, $src1, $src2\n\t"
9256 "csetw $dst, ne\n\t"
9257 "cnegw $dst, lo\t# CmpUL3(imm)"
9258 %}
9259 ins_encode %{
9260 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9261 __ csetw($dst$$Register, Assembler::NE);
9262 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9263 %}
9264
9265 ins_pipe(pipe_class_default);
9266 %}
9267
9268 // Manifest a CmpL result in an integer register.
9269 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9270 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9271 %{
9272 match(Set dst (CmpL3 src1 src2));
9273 effect(KILL flags);
9274
9275 ins_cost(INSN_COST * 3);
9276 format %{
9277 "cmp $src1, $src2\n\t"
9278 "csetw $dst, ne\n\t"
9279 "cnegw $dst, lt\t# CmpL3(reg)"
9280 %}
9281 ins_encode %{
9282 __ cmp($src1$$Register, $src2$$Register);
9283 __ csetw($dst$$Register, Assembler::NE);
9284 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9285 %}
9286
9287 ins_pipe(pipe_class_default);
9288 %}
9289
9290 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9291 %{
9292 match(Set dst (CmpL3 src1 src2));
9293 effect(KILL flags);
9294
9295 ins_cost(INSN_COST * 3);
9296 format %{
9297 "subs zr, $src1, $src2\n\t"
9298 "csetw $dst, ne\n\t"
9299 "cnegw $dst, lt\t# CmpL3(imm)"
9300 %}
9301 ins_encode %{
9302 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9303 __ csetw($dst$$Register, Assembler::NE);
9304 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9305 %}
9306
9307 ins_pipe(pipe_class_default);
9308 %}
9309
9310 // ============================================================================
9311 // Conditional Move Instructions
9312
9313 // n.b. we have identical rules for both a signed compare op (cmpOp)
9314 // and an unsigned compare op (cmpOpU). it would be nice if we could
9315 // define an op class which merged both inputs and use it to type the
9316 // argument to a single rule. unfortunatelyt his fails because the
9317 // opclass does not live up to the COND_INTER interface of its
9318 // component operands. When the generic code tries to negate the
9319 // operand it ends up running the generci Machoper::negate method
9320 // which throws a ShouldNotHappen. So, we have to provide two flavours
9321 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9322
9323 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9324 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9325
9326 ins_cost(INSN_COST * 2);
9327 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9328
9329 ins_encode %{
9330 __ cselw(as_Register($dst$$reg),
9331 as_Register($src2$$reg),
9332 as_Register($src1$$reg),
9333 (Assembler::Condition)$cmp$$cmpcode);
9334 %}
9335
9336 ins_pipe(icond_reg_reg);
9337 %}
9338
9339 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9340 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9341
9342 ins_cost(INSN_COST * 2);
9343 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9344
9345 ins_encode %{
9346 __ cselw(as_Register($dst$$reg),
9347 as_Register($src2$$reg),
9348 as_Register($src1$$reg),
9349 (Assembler::Condition)$cmp$$cmpcode);
9350 %}
9351
9352 ins_pipe(icond_reg_reg);
9353 %}
9354
9355 // special cases where one arg is zero
9356
9357 // n.b. this is selected in preference to the rule above because it
9358 // avoids loading constant 0 into a source register
9359
9360 // TODO
9361 // we ought only to be able to cull one of these variants as the ideal
9362 // transforms ought always to order the zero consistently (to left/right?)
9363
9364 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9365 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9366
9367 ins_cost(INSN_COST * 2);
9368 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9369
9370 ins_encode %{
9371 __ cselw(as_Register($dst$$reg),
9372 as_Register($src$$reg),
9373 zr,
9374 (Assembler::Condition)$cmp$$cmpcode);
9375 %}
9376
9377 ins_pipe(icond_reg);
9378 %}
9379
9380 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9381 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9382
9383 ins_cost(INSN_COST * 2);
9384 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9385
9386 ins_encode %{
9387 __ cselw(as_Register($dst$$reg),
9388 as_Register($src$$reg),
9389 zr,
9390 (Assembler::Condition)$cmp$$cmpcode);
9391 %}
9392
9393 ins_pipe(icond_reg);
9394 %}
9395
9396 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9397 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9398
9399 ins_cost(INSN_COST * 2);
9400 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9401
9402 ins_encode %{
9403 __ cselw(as_Register($dst$$reg),
9404 zr,
9405 as_Register($src$$reg),
9406 (Assembler::Condition)$cmp$$cmpcode);
9407 %}
9408
9409 ins_pipe(icond_reg);
9410 %}
9411
9412 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9413 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9414
9415 ins_cost(INSN_COST * 2);
9416 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9417
9418 ins_encode %{
9419 __ cselw(as_Register($dst$$reg),
9420 zr,
9421 as_Register($src$$reg),
9422 (Assembler::Condition)$cmp$$cmpcode);
9423 %}
9424
9425 ins_pipe(icond_reg);
9426 %}
9427
9428 // special case for creating a boolean 0 or 1
9429
9430 // n.b. this is selected in preference to the rule above because it
9431 // avoids loading constants 0 and 1 into a source register
9432
9433 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9434 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9435
9436 ins_cost(INSN_COST * 2);
9437 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9438
9439 ins_encode %{
9440 // equivalently
9441 // cset(as_Register($dst$$reg),
9442 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9443 __ csincw(as_Register($dst$$reg),
9444 zr,
9445 zr,
9446 (Assembler::Condition)$cmp$$cmpcode);
9447 %}
9448
9449 ins_pipe(icond_none);
9450 %}
9451
9452 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9453 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9454
9455 ins_cost(INSN_COST * 2);
9456 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9457
9458 ins_encode %{
9459 // equivalently
9460 // cset(as_Register($dst$$reg),
9461 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9462 __ csincw(as_Register($dst$$reg),
9463 zr,
9464 zr,
9465 (Assembler::Condition)$cmp$$cmpcode);
9466 %}
9467
9468 ins_pipe(icond_none);
9469 %}
9470
9471 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9472 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9473
9474 ins_cost(INSN_COST * 2);
9475 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9476
9477 ins_encode %{
9478 __ csel(as_Register($dst$$reg),
9479 as_Register($src2$$reg),
9480 as_Register($src1$$reg),
9481 (Assembler::Condition)$cmp$$cmpcode);
9482 %}
9483
9484 ins_pipe(icond_reg_reg);
9485 %}
9486
9487 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9488 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9489
9490 ins_cost(INSN_COST * 2);
9491 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9492
9493 ins_encode %{
9494 __ csel(as_Register($dst$$reg),
9495 as_Register($src2$$reg),
9496 as_Register($src1$$reg),
9497 (Assembler::Condition)$cmp$$cmpcode);
9498 %}
9499
9500 ins_pipe(icond_reg_reg);
9501 %}
9502
9503 // special cases where one arg is zero
9504
9505 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9506 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9507
9508 ins_cost(INSN_COST * 2);
9509 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9510
9511 ins_encode %{
9512 __ csel(as_Register($dst$$reg),
9513 zr,
9514 as_Register($src$$reg),
9515 (Assembler::Condition)$cmp$$cmpcode);
9516 %}
9517
9518 ins_pipe(icond_reg);
9519 %}
9520
9521 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9522 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9523
9524 ins_cost(INSN_COST * 2);
9525 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9526
9527 ins_encode %{
9528 __ csel(as_Register($dst$$reg),
9529 zr,
9530 as_Register($src$$reg),
9531 (Assembler::Condition)$cmp$$cmpcode);
9532 %}
9533
9534 ins_pipe(icond_reg);
9535 %}
9536
9537 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9538 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9539
9540 ins_cost(INSN_COST * 2);
9541 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9542
9543 ins_encode %{
9544 __ csel(as_Register($dst$$reg),
9545 as_Register($src$$reg),
9546 zr,
9547 (Assembler::Condition)$cmp$$cmpcode);
9548 %}
9549
9550 ins_pipe(icond_reg);
9551 %}
9552
9553 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9554 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9555
9556 ins_cost(INSN_COST * 2);
9557 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9558
9559 ins_encode %{
9560 __ csel(as_Register($dst$$reg),
9561 as_Register($src$$reg),
9562 zr,
9563 (Assembler::Condition)$cmp$$cmpcode);
9564 %}
9565
9566 ins_pipe(icond_reg);
9567 %}
9568
9569 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9570 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9571
9572 ins_cost(INSN_COST * 2);
9573 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9574
9575 ins_encode %{
9576 __ csel(as_Register($dst$$reg),
9577 as_Register($src2$$reg),
9578 as_Register($src1$$reg),
9579 (Assembler::Condition)$cmp$$cmpcode);
9580 %}
9581
9582 ins_pipe(icond_reg_reg);
9583 %}
9584
9585 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9586 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9587
9588 ins_cost(INSN_COST * 2);
9589 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9590
9591 ins_encode %{
9592 __ csel(as_Register($dst$$reg),
9593 as_Register($src2$$reg),
9594 as_Register($src1$$reg),
9595 (Assembler::Condition)$cmp$$cmpcode);
9596 %}
9597
9598 ins_pipe(icond_reg_reg);
9599 %}
9600
9601 // special cases where one arg is zero
9602
9603 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9604 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9605
9606 ins_cost(INSN_COST * 2);
9607 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9608
9609 ins_encode %{
9610 __ csel(as_Register($dst$$reg),
9611 zr,
9612 as_Register($src$$reg),
9613 (Assembler::Condition)$cmp$$cmpcode);
9614 %}
9615
9616 ins_pipe(icond_reg);
9617 %}
9618
9619 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9620 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9621
9622 ins_cost(INSN_COST * 2);
9623 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9624
9625 ins_encode %{
9626 __ csel(as_Register($dst$$reg),
9627 zr,
9628 as_Register($src$$reg),
9629 (Assembler::Condition)$cmp$$cmpcode);
9630 %}
9631
9632 ins_pipe(icond_reg);
9633 %}
9634
9635 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9636 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9637
9638 ins_cost(INSN_COST * 2);
9639 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9640
9641 ins_encode %{
9642 __ csel(as_Register($dst$$reg),
9643 as_Register($src$$reg),
9644 zr,
9645 (Assembler::Condition)$cmp$$cmpcode);
9646 %}
9647
9648 ins_pipe(icond_reg);
9649 %}
9650
9651 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9652 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9653
9654 ins_cost(INSN_COST * 2);
9655 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9656
9657 ins_encode %{
9658 __ csel(as_Register($dst$$reg),
9659 as_Register($src$$reg),
9660 zr,
9661 (Assembler::Condition)$cmp$$cmpcode);
9662 %}
9663
9664 ins_pipe(icond_reg);
9665 %}
9666
9667 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9668 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9669
9670 ins_cost(INSN_COST * 2);
9671 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9672
9673 ins_encode %{
9674 __ cselw(as_Register($dst$$reg),
9675 as_Register($src2$$reg),
9676 as_Register($src1$$reg),
9677 (Assembler::Condition)$cmp$$cmpcode);
9678 %}
9679
9680 ins_pipe(icond_reg_reg);
9681 %}
9682
9683 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9684 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9685
9686 ins_cost(INSN_COST * 2);
9687 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9688
9689 ins_encode %{
9690 __ cselw(as_Register($dst$$reg),
9691 as_Register($src2$$reg),
9692 as_Register($src1$$reg),
9693 (Assembler::Condition)$cmp$$cmpcode);
9694 %}
9695
9696 ins_pipe(icond_reg_reg);
9697 %}
9698
9699 // special cases where one arg is zero
9700
9701 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9702 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9703
9704 ins_cost(INSN_COST * 2);
9705 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9706
9707 ins_encode %{
9708 __ cselw(as_Register($dst$$reg),
9709 zr,
9710 as_Register($src$$reg),
9711 (Assembler::Condition)$cmp$$cmpcode);
9712 %}
9713
9714 ins_pipe(icond_reg);
9715 %}
9716
9717 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9718 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9719
9720 ins_cost(INSN_COST * 2);
9721 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9722
9723 ins_encode %{
9724 __ cselw(as_Register($dst$$reg),
9725 zr,
9726 as_Register($src$$reg),
9727 (Assembler::Condition)$cmp$$cmpcode);
9728 %}
9729
9730 ins_pipe(icond_reg);
9731 %}
9732
9733 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9734 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9735
9736 ins_cost(INSN_COST * 2);
9737 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9738
9739 ins_encode %{
9740 __ cselw(as_Register($dst$$reg),
9741 as_Register($src$$reg),
9742 zr,
9743 (Assembler::Condition)$cmp$$cmpcode);
9744 %}
9745
9746 ins_pipe(icond_reg);
9747 %}
9748
9749 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9750 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9751
9752 ins_cost(INSN_COST * 2);
9753 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9754
9755 ins_encode %{
9756 __ cselw(as_Register($dst$$reg),
9757 as_Register($src$$reg),
9758 zr,
9759 (Assembler::Condition)$cmp$$cmpcode);
9760 %}
9761
9762 ins_pipe(icond_reg);
9763 %}
9764
9765 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9766 %{
9767 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9768
9769 ins_cost(INSN_COST * 3);
9770
9771 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9772 ins_encode %{
9773 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9774 __ fcsels(as_FloatRegister($dst$$reg),
9775 as_FloatRegister($src2$$reg),
9776 as_FloatRegister($src1$$reg),
9777 cond);
9778 %}
9779
9780 ins_pipe(fp_cond_reg_reg_s);
9781 %}
9782
9783 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9784 %{
9785 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9786
9787 ins_cost(INSN_COST * 3);
9788
9789 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9790 ins_encode %{
9791 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9792 __ fcsels(as_FloatRegister($dst$$reg),
9793 as_FloatRegister($src2$$reg),
9794 as_FloatRegister($src1$$reg),
9795 cond);
9796 %}
9797
9798 ins_pipe(fp_cond_reg_reg_s);
9799 %}
9800
9801 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9802 %{
9803 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9804
9805 ins_cost(INSN_COST * 3);
9806
9807 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9808 ins_encode %{
9809 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9810 __ fcseld(as_FloatRegister($dst$$reg),
9811 as_FloatRegister($src2$$reg),
9812 as_FloatRegister($src1$$reg),
9813 cond);
9814 %}
9815
9816 ins_pipe(fp_cond_reg_reg_d);
9817 %}
9818
9819 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9820 %{
9821 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9822
9823 ins_cost(INSN_COST * 3);
9824
9825 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9826 ins_encode %{
9827 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9828 __ fcseld(as_FloatRegister($dst$$reg),
9829 as_FloatRegister($src2$$reg),
9830 as_FloatRegister($src1$$reg),
9831 cond);
9832 %}
9833
9834 ins_pipe(fp_cond_reg_reg_d);
9835 %}
9836
9837 // ============================================================================
9838 // Arithmetic Instructions
9839 //
9840
9841 // Integer Addition
9842
9843 // TODO
9844 // these currently employ operations which do not set CR and hence are
9845 // not flagged as killing CR but we would like to isolate the cases
9846 // where we want to set flags from those where we don't. need to work
9847 // out how to do that.
9848
9849 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9850 match(Set dst (AddI src1 src2));
9851
9852 ins_cost(INSN_COST);
9853 format %{ "addw $dst, $src1, $src2" %}
9854
9855 ins_encode %{
9856 __ addw(as_Register($dst$$reg),
9857 as_Register($src1$$reg),
9858 as_Register($src2$$reg));
9859 %}
9860
9861 ins_pipe(ialu_reg_reg);
9862 %}
9863
9864 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9865 match(Set dst (AddI src1 src2));
9866
9867 ins_cost(INSN_COST);
9868 format %{ "addw $dst, $src1, $src2" %}
9869
9870 // use opcode to indicate that this is an add not a sub
9871 opcode(0x0);
9872
9873 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9874
9875 ins_pipe(ialu_reg_imm);
9876 %}
9877
9878 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9879 match(Set dst (AddI (ConvL2I src1) src2));
9880
9881 ins_cost(INSN_COST);
9882 format %{ "addw $dst, $src1, $src2" %}
9883
9884 // use opcode to indicate that this is an add not a sub
9885 opcode(0x0);
9886
9887 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9888
9889 ins_pipe(ialu_reg_imm);
9890 %}
9891
9892 // Pointer Addition
9893 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9894 match(Set dst (AddP src1 src2));
9895
9896 ins_cost(INSN_COST);
9897 format %{ "add $dst, $src1, $src2\t# ptr" %}
9898
9899 ins_encode %{
9900 __ add(as_Register($dst$$reg),
9901 as_Register($src1$$reg),
9902 as_Register($src2$$reg));
9903 %}
9904
9905 ins_pipe(ialu_reg_reg);
9906 %}
9907
9908 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9909 match(Set dst (AddP src1 (ConvI2L src2)));
9910
9911 ins_cost(1.9 * INSN_COST);
9912 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9913
9914 ins_encode %{
9915 __ add(as_Register($dst$$reg),
9916 as_Register($src1$$reg),
9917 as_Register($src2$$reg), ext::sxtw);
9918 %}
9919
9920 ins_pipe(ialu_reg_reg);
9921 %}
9922
9923 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9924 match(Set dst (AddP src1 (LShiftL src2 scale)));
9925
9926 ins_cost(1.9 * INSN_COST);
9927 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9928
9929 ins_encode %{
9930 __ lea(as_Register($dst$$reg),
9931 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9932 Address::lsl($scale$$constant)));
9933 %}
9934
9935 ins_pipe(ialu_reg_reg_shift);
9936 %}
9937
9938 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9939 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9940
9941 ins_cost(1.9 * INSN_COST);
9942 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9943
9944 ins_encode %{
9945 __ lea(as_Register($dst$$reg),
9946 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9947 Address::sxtw($scale$$constant)));
9948 %}
9949
9950 ins_pipe(ialu_reg_reg_shift);
9951 %}
9952
9953 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9954 match(Set dst (LShiftL (ConvI2L src) scale));
9955
9956 ins_cost(INSN_COST);
9957 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9958
9959 ins_encode %{
9960 __ sbfiz(as_Register($dst$$reg),
9961 as_Register($src$$reg),
9962 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9963 %}
9964
9965 ins_pipe(ialu_reg_shift);
9966 %}
9967
9968 // Pointer Immediate Addition
9969 // n.b. this needs to be more expensive than using an indirect memory
9970 // operand
9971 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9972 match(Set dst (AddP src1 src2));
9973
9974 ins_cost(INSN_COST);
9975 format %{ "add $dst, $src1, $src2\t# ptr" %}
9976
9977 // use opcode to indicate that this is an add not a sub
9978 opcode(0x0);
9979
9980 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9981
9982 ins_pipe(ialu_reg_imm);
9983 %}
9984
9985 // Long Addition
9986 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9987
9988 match(Set dst (AddL src1 src2));
9989
9990 ins_cost(INSN_COST);
9991 format %{ "add $dst, $src1, $src2" %}
9992
9993 ins_encode %{
9994 __ add(as_Register($dst$$reg),
9995 as_Register($src1$$reg),
9996 as_Register($src2$$reg));
9997 %}
9998
9999 ins_pipe(ialu_reg_reg);
10000 %}
10001
10002 // No constant pool entries requiredLong Immediate Addition.
10003 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10004 match(Set dst (AddL src1 src2));
10005
10006 ins_cost(INSN_COST);
10007 format %{ "add $dst, $src1, $src2" %}
10008
10009 // use opcode to indicate that this is an add not a sub
10010 opcode(0x0);
10011
10012 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10013
10014 ins_pipe(ialu_reg_imm);
10015 %}
10016
10017 // Integer Subtraction
10018 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10019 match(Set dst (SubI src1 src2));
10020
10021 ins_cost(INSN_COST);
10022 format %{ "subw $dst, $src1, $src2" %}
10023
10024 ins_encode %{
10025 __ subw(as_Register($dst$$reg),
10026 as_Register($src1$$reg),
10027 as_Register($src2$$reg));
10028 %}
10029
10030 ins_pipe(ialu_reg_reg);
10031 %}
10032
10033 // Immediate Subtraction
10034 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10035 match(Set dst (SubI src1 src2));
10036
10037 ins_cost(INSN_COST);
10038 format %{ "subw $dst, $src1, $src2" %}
10039
10040 // use opcode to indicate that this is a sub not an add
10041 opcode(0x1);
10042
10043 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10044
10045 ins_pipe(ialu_reg_imm);
10046 %}
10047
10048 // Long Subtraction
10049 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10050
10051 match(Set dst (SubL src1 src2));
10052
10053 ins_cost(INSN_COST);
10054 format %{ "sub $dst, $src1, $src2" %}
10055
10056 ins_encode %{
10057 __ sub(as_Register($dst$$reg),
10058 as_Register($src1$$reg),
10059 as_Register($src2$$reg));
10060 %}
10061
10062 ins_pipe(ialu_reg_reg);
10063 %}
10064
10065 // No constant pool entries requiredLong Immediate Subtraction.
10066 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10067 match(Set dst (SubL src1 src2));
10068
10069 ins_cost(INSN_COST);
10070 format %{ "sub$dst, $src1, $src2" %}
10071
10072 // use opcode to indicate that this is a sub not an add
10073 opcode(0x1);
10074
10075 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10076
10077 ins_pipe(ialu_reg_imm);
10078 %}
10079
10080 // Integer Negation (special case for sub)
10081
10082 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10083 match(Set dst (SubI zero src));
10084
10085 ins_cost(INSN_COST);
10086 format %{ "negw $dst, $src\t# int" %}
10087
10088 ins_encode %{
10089 __ negw(as_Register($dst$$reg),
10090 as_Register($src$$reg));
10091 %}
10092
10093 ins_pipe(ialu_reg);
10094 %}
10095
10096 // Long Negation
10097
10098 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10099 match(Set dst (SubL zero src));
10100
10101 ins_cost(INSN_COST);
10102 format %{ "neg $dst, $src\t# long" %}
10103
10104 ins_encode %{
10105 __ neg(as_Register($dst$$reg),
10106 as_Register($src$$reg));
10107 %}
10108
10109 ins_pipe(ialu_reg);
10110 %}
10111
10112 // Integer Multiply
10113
10114 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10115 match(Set dst (MulI src1 src2));
10116
10117 ins_cost(INSN_COST * 3);
10118 format %{ "mulw $dst, $src1, $src2" %}
10119
10120 ins_encode %{
10121 __ mulw(as_Register($dst$$reg),
10122 as_Register($src1$$reg),
10123 as_Register($src2$$reg));
10124 %}
10125
10126 ins_pipe(imul_reg_reg);
10127 %}
10128
10129 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10130 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10131
10132 ins_cost(INSN_COST * 3);
10133 format %{ "smull $dst, $src1, $src2" %}
10134
10135 ins_encode %{
10136 __ smull(as_Register($dst$$reg),
10137 as_Register($src1$$reg),
10138 as_Register($src2$$reg));
10139 %}
10140
10141 ins_pipe(imul_reg_reg);
10142 %}
10143
10144 // Long Multiply
10145
10146 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10147 match(Set dst (MulL src1 src2));
10148
10149 ins_cost(INSN_COST * 5);
10150 format %{ "mul $dst, $src1, $src2" %}
10151
10152 ins_encode %{
10153 __ mul(as_Register($dst$$reg),
10154 as_Register($src1$$reg),
10155 as_Register($src2$$reg));
10156 %}
10157
10158 ins_pipe(lmul_reg_reg);
10159 %}
10160
10161 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10162 %{
10163 match(Set dst (MulHiL src1 src2));
10164
10165 ins_cost(INSN_COST * 7);
10166 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10167
10168 ins_encode %{
10169 __ smulh(as_Register($dst$$reg),
10170 as_Register($src1$$reg),
10171 as_Register($src2$$reg));
10172 %}
10173
10174 ins_pipe(lmul_reg_reg);
10175 %}
10176
10177 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10178 %{
10179 match(Set dst (UMulHiL src1 src2));
10180
10181 ins_cost(INSN_COST * 7);
10182 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10183
10184 ins_encode %{
10185 __ umulh(as_Register($dst$$reg),
10186 as_Register($src1$$reg),
10187 as_Register($src2$$reg));
10188 %}
10189
10190 ins_pipe(lmul_reg_reg);
10191 %}
10192
10193 // Combined Integer Multiply & Add/Sub
10194
10195 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10196 match(Set dst (AddI src3 (MulI src1 src2)));
10197
10198 ins_cost(INSN_COST * 3);
10199 format %{ "madd $dst, $src1, $src2, $src3" %}
10200
10201 ins_encode %{
10202 __ maddw(as_Register($dst$$reg),
10203 as_Register($src1$$reg),
10204 as_Register($src2$$reg),
10205 as_Register($src3$$reg));
10206 %}
10207
10208 ins_pipe(imac_reg_reg);
10209 %}
10210
10211 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10212 match(Set dst (SubI src3 (MulI src1 src2)));
10213
10214 ins_cost(INSN_COST * 3);
10215 format %{ "msub $dst, $src1, $src2, $src3" %}
10216
10217 ins_encode %{
10218 __ msubw(as_Register($dst$$reg),
10219 as_Register($src1$$reg),
10220 as_Register($src2$$reg),
10221 as_Register($src3$$reg));
10222 %}
10223
10224 ins_pipe(imac_reg_reg);
10225 %}
10226
10227 // Combined Integer Multiply & Neg
10228
10229 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10230 match(Set dst (MulI (SubI zero src1) src2));
10231
10232 ins_cost(INSN_COST * 3);
10233 format %{ "mneg $dst, $src1, $src2" %}
10234
10235 ins_encode %{
10236 __ mnegw(as_Register($dst$$reg),
10237 as_Register($src1$$reg),
10238 as_Register($src2$$reg));
10239 %}
10240
10241 ins_pipe(imac_reg_reg);
10242 %}
10243
10244 // Combined Long Multiply & Add/Sub
10245
10246 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10247 match(Set dst (AddL src3 (MulL src1 src2)));
10248
10249 ins_cost(INSN_COST * 5);
10250 format %{ "madd $dst, $src1, $src2, $src3" %}
10251
10252 ins_encode %{
10253 __ madd(as_Register($dst$$reg),
10254 as_Register($src1$$reg),
10255 as_Register($src2$$reg),
10256 as_Register($src3$$reg));
10257 %}
10258
10259 ins_pipe(lmac_reg_reg);
10260 %}
10261
10262 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10263 match(Set dst (SubL src3 (MulL src1 src2)));
10264
10265 ins_cost(INSN_COST * 5);
10266 format %{ "msub $dst, $src1, $src2, $src3" %}
10267
10268 ins_encode %{
10269 __ msub(as_Register($dst$$reg),
10270 as_Register($src1$$reg),
10271 as_Register($src2$$reg),
10272 as_Register($src3$$reg));
10273 %}
10274
10275 ins_pipe(lmac_reg_reg);
10276 %}
10277
10278 // Combined Long Multiply & Neg
10279
10280 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10281 match(Set dst (MulL (SubL zero src1) src2));
10282
10283 ins_cost(INSN_COST * 5);
10284 format %{ "mneg $dst, $src1, $src2" %}
10285
10286 ins_encode %{
10287 __ mneg(as_Register($dst$$reg),
10288 as_Register($src1$$reg),
10289 as_Register($src2$$reg));
10290 %}
10291
10292 ins_pipe(lmac_reg_reg);
10293 %}
10294
10295 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10296
10297 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10298 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10299
10300 ins_cost(INSN_COST * 3);
10301 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10302
10303 ins_encode %{
10304 __ smaddl(as_Register($dst$$reg),
10305 as_Register($src1$$reg),
10306 as_Register($src2$$reg),
10307 as_Register($src3$$reg));
10308 %}
10309
10310 ins_pipe(imac_reg_reg);
10311 %}
10312
10313 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10314 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10315
10316 ins_cost(INSN_COST * 3);
10317 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10318
10319 ins_encode %{
10320 __ smsubl(as_Register($dst$$reg),
10321 as_Register($src1$$reg),
10322 as_Register($src2$$reg),
10323 as_Register($src3$$reg));
10324 %}
10325
10326 ins_pipe(imac_reg_reg);
10327 %}
10328
10329 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10330 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10331
10332 ins_cost(INSN_COST * 3);
10333 format %{ "smnegl $dst, $src1, $src2" %}
10334
10335 ins_encode %{
10336 __ smnegl(as_Register($dst$$reg),
10337 as_Register($src1$$reg),
10338 as_Register($src2$$reg));
10339 %}
10340
10341 ins_pipe(imac_reg_reg);
10342 %}
10343
10344 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10345
10346 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10347 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10348
10349 ins_cost(INSN_COST * 5);
10350 format %{ "mulw rscratch1, $src1, $src2\n\t"
10351 "maddw $dst, $src3, $src4, rscratch1" %}
10352
10353 ins_encode %{
10354 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10355 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10356
10357 ins_pipe(imac_reg_reg);
10358 %}
10359
10360 // Integer Divide
10361
10362 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10363 match(Set dst (DivI src1 src2));
10364
10365 ins_cost(INSN_COST * 19);
10366 format %{ "sdivw $dst, $src1, $src2" %}
10367
10368 ins_encode(aarch64_enc_divw(dst, src1, src2));
10369 ins_pipe(idiv_reg_reg);
10370 %}
10371
10372 // Long Divide
10373
10374 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10375 match(Set dst (DivL src1 src2));
10376
10377 ins_cost(INSN_COST * 35);
10378 format %{ "sdiv $dst, $src1, $src2" %}
10379
10380 ins_encode(aarch64_enc_div(dst, src1, src2));
10381 ins_pipe(ldiv_reg_reg);
10382 %}
10383
10384 // Integer Remainder
10385
10386 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10387 match(Set dst (ModI src1 src2));
10388
10389 ins_cost(INSN_COST * 22);
10390 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10391 "msubw $dst, rscratch1, $src2, $src1" %}
10392
10393 ins_encode(aarch64_enc_modw(dst, src1, src2));
10394 ins_pipe(idiv_reg_reg);
10395 %}
10396
10397 // Long Remainder
10398
10399 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10400 match(Set dst (ModL src1 src2));
10401
10402 ins_cost(INSN_COST * 38);
10403 format %{ "sdiv rscratch1, $src1, $src2\n"
10404 "msub $dst, rscratch1, $src2, $src1" %}
10405
10406 ins_encode(aarch64_enc_mod(dst, src1, src2));
10407 ins_pipe(ldiv_reg_reg);
10408 %}
10409
10410 // Unsigned Integer Divide
10411
10412 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10413 match(Set dst (UDivI src1 src2));
10414
10415 ins_cost(INSN_COST * 19);
10416 format %{ "udivw $dst, $src1, $src2" %}
10417
10418 ins_encode %{
10419 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10420 %}
10421
10422 ins_pipe(idiv_reg_reg);
10423 %}
10424
10425 // Unsigned Long Divide
10426
10427 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10428 match(Set dst (UDivL src1 src2));
10429
10430 ins_cost(INSN_COST * 35);
10431 format %{ "udiv $dst, $src1, $src2" %}
10432
10433 ins_encode %{
10434 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10435 %}
10436
10437 ins_pipe(ldiv_reg_reg);
10438 %}
10439
10440 // Unsigned Integer Remainder
10441
10442 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10443 match(Set dst (UModI src1 src2));
10444
10445 ins_cost(INSN_COST * 22);
10446 format %{ "udivw rscratch1, $src1, $src2\n\t"
10447 "msubw $dst, rscratch1, $src2, $src1" %}
10448
10449 ins_encode %{
10450 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10451 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10452 %}
10453
10454 ins_pipe(idiv_reg_reg);
10455 %}
10456
10457 // Unsigned Long Remainder
10458
10459 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10460 match(Set dst (UModL src1 src2));
10461
10462 ins_cost(INSN_COST * 38);
10463 format %{ "udiv rscratch1, $src1, $src2\n"
10464 "msub $dst, rscratch1, $src2, $src1" %}
10465
10466 ins_encode %{
10467 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10468 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10469 %}
10470
10471 ins_pipe(ldiv_reg_reg);
10472 %}
10473
10474 // Integer Shifts
10475
10476 // Shift Left Register
10477 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10478 match(Set dst (LShiftI src1 src2));
10479
10480 ins_cost(INSN_COST * 2);
10481 format %{ "lslvw $dst, $src1, $src2" %}
10482
10483 ins_encode %{
10484 __ lslvw(as_Register($dst$$reg),
10485 as_Register($src1$$reg),
10486 as_Register($src2$$reg));
10487 %}
10488
10489 ins_pipe(ialu_reg_reg_vshift);
10490 %}
10491
10492 // Shift Left Immediate
10493 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10494 match(Set dst (LShiftI src1 src2));
10495
10496 ins_cost(INSN_COST);
10497 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10498
10499 ins_encode %{
10500 __ lslw(as_Register($dst$$reg),
10501 as_Register($src1$$reg),
10502 $src2$$constant & 0x1f);
10503 %}
10504
10505 ins_pipe(ialu_reg_shift);
10506 %}
10507
10508 // Shift Right Logical Register
10509 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10510 match(Set dst (URShiftI src1 src2));
10511
10512 ins_cost(INSN_COST * 2);
10513 format %{ "lsrvw $dst, $src1, $src2" %}
10514
10515 ins_encode %{
10516 __ lsrvw(as_Register($dst$$reg),
10517 as_Register($src1$$reg),
10518 as_Register($src2$$reg));
10519 %}
10520
10521 ins_pipe(ialu_reg_reg_vshift);
10522 %}
10523
10524 // Shift Right Logical Immediate
10525 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10526 match(Set dst (URShiftI src1 src2));
10527
10528 ins_cost(INSN_COST);
10529 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10530
10531 ins_encode %{
10532 __ lsrw(as_Register($dst$$reg),
10533 as_Register($src1$$reg),
10534 $src2$$constant & 0x1f);
10535 %}
10536
10537 ins_pipe(ialu_reg_shift);
10538 %}
10539
10540 // Shift Right Arithmetic Register
10541 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10542 match(Set dst (RShiftI src1 src2));
10543
10544 ins_cost(INSN_COST * 2);
10545 format %{ "asrvw $dst, $src1, $src2" %}
10546
10547 ins_encode %{
10548 __ asrvw(as_Register($dst$$reg),
10549 as_Register($src1$$reg),
10550 as_Register($src2$$reg));
10551 %}
10552
10553 ins_pipe(ialu_reg_reg_vshift);
10554 %}
10555
10556 // Shift Right Arithmetic Immediate
10557 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10558 match(Set dst (RShiftI src1 src2));
10559
10560 ins_cost(INSN_COST);
10561 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10562
10563 ins_encode %{
10564 __ asrw(as_Register($dst$$reg),
10565 as_Register($src1$$reg),
10566 $src2$$constant & 0x1f);
10567 %}
10568
10569 ins_pipe(ialu_reg_shift);
10570 %}
10571
10572 // Combined Int Mask and Right Shift (using UBFM)
10573 // TODO
10574
10575 // Long Shifts
10576
10577 // Shift Left Register
10578 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10579 match(Set dst (LShiftL src1 src2));
10580
10581 ins_cost(INSN_COST * 2);
10582 format %{ "lslv $dst, $src1, $src2" %}
10583
10584 ins_encode %{
10585 __ lslv(as_Register($dst$$reg),
10586 as_Register($src1$$reg),
10587 as_Register($src2$$reg));
10588 %}
10589
10590 ins_pipe(ialu_reg_reg_vshift);
10591 %}
10592
10593 // Shift Left Immediate
10594 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10595 match(Set dst (LShiftL src1 src2));
10596
10597 ins_cost(INSN_COST);
10598 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10599
10600 ins_encode %{
10601 __ lsl(as_Register($dst$$reg),
10602 as_Register($src1$$reg),
10603 $src2$$constant & 0x3f);
10604 %}
10605
10606 ins_pipe(ialu_reg_shift);
10607 %}
10608
10609 // Shift Right Logical Register
10610 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10611 match(Set dst (URShiftL src1 src2));
10612
10613 ins_cost(INSN_COST * 2);
10614 format %{ "lsrv $dst, $src1, $src2" %}
10615
10616 ins_encode %{
10617 __ lsrv(as_Register($dst$$reg),
10618 as_Register($src1$$reg),
10619 as_Register($src2$$reg));
10620 %}
10621
10622 ins_pipe(ialu_reg_reg_vshift);
10623 %}
10624
10625 // Shift Right Logical Immediate
10626 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10627 match(Set dst (URShiftL src1 src2));
10628
10629 ins_cost(INSN_COST);
10630 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10631
10632 ins_encode %{
10633 __ lsr(as_Register($dst$$reg),
10634 as_Register($src1$$reg),
10635 $src2$$constant & 0x3f);
10636 %}
10637
10638 ins_pipe(ialu_reg_shift);
10639 %}
10640
10641 // A special-case pattern for card table stores.
10642 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10643 match(Set dst (URShiftL (CastP2X src1) src2));
10644
10645 ins_cost(INSN_COST);
10646 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10647
10648 ins_encode %{
10649 __ lsr(as_Register($dst$$reg),
10650 as_Register($src1$$reg),
10651 $src2$$constant & 0x3f);
10652 %}
10653
10654 ins_pipe(ialu_reg_shift);
10655 %}
10656
10657 // Shift Right Arithmetic Register
10658 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10659 match(Set dst (RShiftL src1 src2));
10660
10661 ins_cost(INSN_COST * 2);
10662 format %{ "asrv $dst, $src1, $src2" %}
10663
10664 ins_encode %{
10665 __ asrv(as_Register($dst$$reg),
10666 as_Register($src1$$reg),
10667 as_Register($src2$$reg));
10668 %}
10669
10670 ins_pipe(ialu_reg_reg_vshift);
10671 %}
10672
10673 // Shift Right Arithmetic Immediate
10674 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10675 match(Set dst (RShiftL src1 src2));
10676
10677 ins_cost(INSN_COST);
10678 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10679
10680 ins_encode %{
10681 __ asr(as_Register($dst$$reg),
10682 as_Register($src1$$reg),
10683 $src2$$constant & 0x3f);
10684 %}
10685
10686 ins_pipe(ialu_reg_shift);
10687 %}
10688
10689 // BEGIN This section of the file is automatically generated. Do not edit --------------
10690 // This section is generated from aarch64_ad.m4
10691
10692 // This pattern is automatically generated from aarch64_ad.m4
10693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10694 instruct regL_not_reg(iRegLNoSp dst,
10695 iRegL src1, immL_M1 m1,
10696 rFlagsReg cr) %{
10697 match(Set dst (XorL src1 m1));
10698 ins_cost(INSN_COST);
10699 format %{ "eon $dst, $src1, zr" %}
10700
10701 ins_encode %{
10702 __ eon(as_Register($dst$$reg),
10703 as_Register($src1$$reg),
10704 zr,
10705 Assembler::LSL, 0);
10706 %}
10707
10708 ins_pipe(ialu_reg);
10709 %}
10710
10711 // This pattern is automatically generated from aarch64_ad.m4
10712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10713 instruct regI_not_reg(iRegINoSp dst,
10714 iRegIorL2I src1, immI_M1 m1,
10715 rFlagsReg cr) %{
10716 match(Set dst (XorI src1 m1));
10717 ins_cost(INSN_COST);
10718 format %{ "eonw $dst, $src1, zr" %}
10719
10720 ins_encode %{
10721 __ eonw(as_Register($dst$$reg),
10722 as_Register($src1$$reg),
10723 zr,
10724 Assembler::LSL, 0);
10725 %}
10726
10727 ins_pipe(ialu_reg);
10728 %}
10729
10730 // This pattern is automatically generated from aarch64_ad.m4
10731 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10732 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10733 immI0 zero, iRegIorL2I src1, immI src2) %{
10734 match(Set dst (SubI zero (URShiftI src1 src2)));
10735
10736 ins_cost(1.9 * INSN_COST);
10737 format %{ "negw $dst, $src1, LSR $src2" %}
10738
10739 ins_encode %{
10740 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10741 Assembler::LSR, $src2$$constant & 0x1f);
10742 %}
10743
10744 ins_pipe(ialu_reg_shift);
10745 %}
10746
10747 // This pattern is automatically generated from aarch64_ad.m4
10748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10749 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10750 immI0 zero, iRegIorL2I src1, immI src2) %{
10751 match(Set dst (SubI zero (RShiftI src1 src2)));
10752
10753 ins_cost(1.9 * INSN_COST);
10754 format %{ "negw $dst, $src1, ASR $src2" %}
10755
10756 ins_encode %{
10757 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10758 Assembler::ASR, $src2$$constant & 0x1f);
10759 %}
10760
10761 ins_pipe(ialu_reg_shift);
10762 %}
10763
10764 // This pattern is automatically generated from aarch64_ad.m4
10765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10766 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10767 immI0 zero, iRegIorL2I src1, immI src2) %{
10768 match(Set dst (SubI zero (LShiftI src1 src2)));
10769
10770 ins_cost(1.9 * INSN_COST);
10771 format %{ "negw $dst, $src1, LSL $src2" %}
10772
10773 ins_encode %{
10774 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10775 Assembler::LSL, $src2$$constant & 0x1f);
10776 %}
10777
10778 ins_pipe(ialu_reg_shift);
10779 %}
10780
10781 // This pattern is automatically generated from aarch64_ad.m4
10782 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10783 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10784 immL0 zero, iRegL src1, immI src2) %{
10785 match(Set dst (SubL zero (URShiftL src1 src2)));
10786
10787 ins_cost(1.9 * INSN_COST);
10788 format %{ "neg $dst, $src1, LSR $src2" %}
10789
10790 ins_encode %{
10791 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10792 Assembler::LSR, $src2$$constant & 0x3f);
10793 %}
10794
10795 ins_pipe(ialu_reg_shift);
10796 %}
10797
10798 // This pattern is automatically generated from aarch64_ad.m4
10799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10800 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10801 immL0 zero, iRegL src1, immI src2) %{
10802 match(Set dst (SubL zero (RShiftL src1 src2)));
10803
10804 ins_cost(1.9 * INSN_COST);
10805 format %{ "neg $dst, $src1, ASR $src2" %}
10806
10807 ins_encode %{
10808 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10809 Assembler::ASR, $src2$$constant & 0x3f);
10810 %}
10811
10812 ins_pipe(ialu_reg_shift);
10813 %}
10814
10815 // This pattern is automatically generated from aarch64_ad.m4
10816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10817 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10818 immL0 zero, iRegL src1, immI src2) %{
10819 match(Set dst (SubL zero (LShiftL src1 src2)));
10820
10821 ins_cost(1.9 * INSN_COST);
10822 format %{ "neg $dst, $src1, LSL $src2" %}
10823
10824 ins_encode %{
10825 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10826 Assembler::LSL, $src2$$constant & 0x3f);
10827 %}
10828
10829 ins_pipe(ialu_reg_shift);
10830 %}
10831
10832 // This pattern is automatically generated from aarch64_ad.m4
10833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10834 instruct AndI_reg_not_reg(iRegINoSp dst,
10835 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10836 match(Set dst (AndI src1 (XorI src2 m1)));
10837 ins_cost(INSN_COST);
10838 format %{ "bicw $dst, $src1, $src2" %}
10839
10840 ins_encode %{
10841 __ bicw(as_Register($dst$$reg),
10842 as_Register($src1$$reg),
10843 as_Register($src2$$reg),
10844 Assembler::LSL, 0);
10845 %}
10846
10847 ins_pipe(ialu_reg_reg);
10848 %}
10849
10850 // This pattern is automatically generated from aarch64_ad.m4
10851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10852 instruct AndL_reg_not_reg(iRegLNoSp dst,
10853 iRegL src1, iRegL src2, immL_M1 m1) %{
10854 match(Set dst (AndL src1 (XorL src2 m1)));
10855 ins_cost(INSN_COST);
10856 format %{ "bic $dst, $src1, $src2" %}
10857
10858 ins_encode %{
10859 __ bic(as_Register($dst$$reg),
10860 as_Register($src1$$reg),
10861 as_Register($src2$$reg),
10862 Assembler::LSL, 0);
10863 %}
10864
10865 ins_pipe(ialu_reg_reg);
10866 %}
10867
10868 // This pattern is automatically generated from aarch64_ad.m4
10869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10870 instruct OrI_reg_not_reg(iRegINoSp dst,
10871 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10872 match(Set dst (OrI src1 (XorI src2 m1)));
10873 ins_cost(INSN_COST);
10874 format %{ "ornw $dst, $src1, $src2" %}
10875
10876 ins_encode %{
10877 __ ornw(as_Register($dst$$reg),
10878 as_Register($src1$$reg),
10879 as_Register($src2$$reg),
10880 Assembler::LSL, 0);
10881 %}
10882
10883 ins_pipe(ialu_reg_reg);
10884 %}
10885
10886 // This pattern is automatically generated from aarch64_ad.m4
10887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10888 instruct OrL_reg_not_reg(iRegLNoSp dst,
10889 iRegL src1, iRegL src2, immL_M1 m1) %{
10890 match(Set dst (OrL src1 (XorL src2 m1)));
10891 ins_cost(INSN_COST);
10892 format %{ "orn $dst, $src1, $src2" %}
10893
10894 ins_encode %{
10895 __ orn(as_Register($dst$$reg),
10896 as_Register($src1$$reg),
10897 as_Register($src2$$reg),
10898 Assembler::LSL, 0);
10899 %}
10900
10901 ins_pipe(ialu_reg_reg);
10902 %}
10903
10904 // This pattern is automatically generated from aarch64_ad.m4
10905 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10906 instruct XorI_reg_not_reg(iRegINoSp dst,
10907 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10908 match(Set dst (XorI m1 (XorI src2 src1)));
10909 ins_cost(INSN_COST);
10910 format %{ "eonw $dst, $src1, $src2" %}
10911
10912 ins_encode %{
10913 __ eonw(as_Register($dst$$reg),
10914 as_Register($src1$$reg),
10915 as_Register($src2$$reg),
10916 Assembler::LSL, 0);
10917 %}
10918
10919 ins_pipe(ialu_reg_reg);
10920 %}
10921
10922 // This pattern is automatically generated from aarch64_ad.m4
10923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10924 instruct XorL_reg_not_reg(iRegLNoSp dst,
10925 iRegL src1, iRegL src2, immL_M1 m1) %{
10926 match(Set dst (XorL m1 (XorL src2 src1)));
10927 ins_cost(INSN_COST);
10928 format %{ "eon $dst, $src1, $src2" %}
10929
10930 ins_encode %{
10931 __ eon(as_Register($dst$$reg),
10932 as_Register($src1$$reg),
10933 as_Register($src2$$reg),
10934 Assembler::LSL, 0);
10935 %}
10936
10937 ins_pipe(ialu_reg_reg);
10938 %}
10939
10940 // This pattern is automatically generated from aarch64_ad.m4
10941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10942 // val & (-1 ^ (val >>> shift)) ==> bicw
10943 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10944 iRegIorL2I src1, iRegIorL2I src2,
10945 immI src3, immI_M1 src4) %{
10946 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10947 ins_cost(1.9 * INSN_COST);
10948 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10949
10950 ins_encode %{
10951 __ bicw(as_Register($dst$$reg),
10952 as_Register($src1$$reg),
10953 as_Register($src2$$reg),
10954 Assembler::LSR,
10955 $src3$$constant & 0x1f);
10956 %}
10957
10958 ins_pipe(ialu_reg_reg_shift);
10959 %}
10960
10961 // This pattern is automatically generated from aarch64_ad.m4
10962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10963 // val & (-1 ^ (val >>> shift)) ==> bic
10964 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10965 iRegL src1, iRegL src2,
10966 immI src3, immL_M1 src4) %{
10967 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10968 ins_cost(1.9 * INSN_COST);
10969 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10970
10971 ins_encode %{
10972 __ bic(as_Register($dst$$reg),
10973 as_Register($src1$$reg),
10974 as_Register($src2$$reg),
10975 Assembler::LSR,
10976 $src3$$constant & 0x3f);
10977 %}
10978
10979 ins_pipe(ialu_reg_reg_shift);
10980 %}
10981
10982 // This pattern is automatically generated from aarch64_ad.m4
10983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10984 // val & (-1 ^ (val >> shift)) ==> bicw
10985 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10986 iRegIorL2I src1, iRegIorL2I src2,
10987 immI src3, immI_M1 src4) %{
10988 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10989 ins_cost(1.9 * INSN_COST);
10990 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10991
10992 ins_encode %{
10993 __ bicw(as_Register($dst$$reg),
10994 as_Register($src1$$reg),
10995 as_Register($src2$$reg),
10996 Assembler::ASR,
10997 $src3$$constant & 0x1f);
10998 %}
10999
11000 ins_pipe(ialu_reg_reg_shift);
11001 %}
11002
11003 // This pattern is automatically generated from aarch64_ad.m4
11004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11005 // val & (-1 ^ (val >> shift)) ==> bic
11006 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11007 iRegL src1, iRegL src2,
11008 immI src3, immL_M1 src4) %{
11009 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11010 ins_cost(1.9 * INSN_COST);
11011 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11012
11013 ins_encode %{
11014 __ bic(as_Register($dst$$reg),
11015 as_Register($src1$$reg),
11016 as_Register($src2$$reg),
11017 Assembler::ASR,
11018 $src3$$constant & 0x3f);
11019 %}
11020
11021 ins_pipe(ialu_reg_reg_shift);
11022 %}
11023
11024 // This pattern is automatically generated from aarch64_ad.m4
11025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11026 // val & (-1 ^ (val ror shift)) ==> bicw
11027 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11028 iRegIorL2I src1, iRegIorL2I src2,
11029 immI src3, immI_M1 src4) %{
11030 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11031 ins_cost(1.9 * INSN_COST);
11032 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11033
11034 ins_encode %{
11035 __ bicw(as_Register($dst$$reg),
11036 as_Register($src1$$reg),
11037 as_Register($src2$$reg),
11038 Assembler::ROR,
11039 $src3$$constant & 0x1f);
11040 %}
11041
11042 ins_pipe(ialu_reg_reg_shift);
11043 %}
11044
11045 // This pattern is automatically generated from aarch64_ad.m4
11046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11047 // val & (-1 ^ (val ror shift)) ==> bic
11048 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11049 iRegL src1, iRegL src2,
11050 immI src3, immL_M1 src4) %{
11051 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11052 ins_cost(1.9 * INSN_COST);
11053 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11054
11055 ins_encode %{
11056 __ bic(as_Register($dst$$reg),
11057 as_Register($src1$$reg),
11058 as_Register($src2$$reg),
11059 Assembler::ROR,
11060 $src3$$constant & 0x3f);
11061 %}
11062
11063 ins_pipe(ialu_reg_reg_shift);
11064 %}
11065
11066 // This pattern is automatically generated from aarch64_ad.m4
11067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11068 // val & (-1 ^ (val << shift)) ==> bicw
11069 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11070 iRegIorL2I src1, iRegIorL2I src2,
11071 immI src3, immI_M1 src4) %{
11072 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11073 ins_cost(1.9 * INSN_COST);
11074 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11075
11076 ins_encode %{
11077 __ bicw(as_Register($dst$$reg),
11078 as_Register($src1$$reg),
11079 as_Register($src2$$reg),
11080 Assembler::LSL,
11081 $src3$$constant & 0x1f);
11082 %}
11083
11084 ins_pipe(ialu_reg_reg_shift);
11085 %}
11086
11087 // This pattern is automatically generated from aarch64_ad.m4
11088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11089 // val & (-1 ^ (val << shift)) ==> bic
11090 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11091 iRegL src1, iRegL src2,
11092 immI src3, immL_M1 src4) %{
11093 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11094 ins_cost(1.9 * INSN_COST);
11095 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11096
11097 ins_encode %{
11098 __ bic(as_Register($dst$$reg),
11099 as_Register($src1$$reg),
11100 as_Register($src2$$reg),
11101 Assembler::LSL,
11102 $src3$$constant & 0x3f);
11103 %}
11104
11105 ins_pipe(ialu_reg_reg_shift);
11106 %}
11107
11108 // This pattern is automatically generated from aarch64_ad.m4
11109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11110 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11111 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11112 iRegIorL2I src1, iRegIorL2I src2,
11113 immI src3, immI_M1 src4) %{
11114 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11115 ins_cost(1.9 * INSN_COST);
11116 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11117
11118 ins_encode %{
11119 __ eonw(as_Register($dst$$reg),
11120 as_Register($src1$$reg),
11121 as_Register($src2$$reg),
11122 Assembler::LSR,
11123 $src3$$constant & 0x1f);
11124 %}
11125
11126 ins_pipe(ialu_reg_reg_shift);
11127 %}
11128
11129 // This pattern is automatically generated from aarch64_ad.m4
11130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11131 // val ^ (-1 ^ (val >>> shift)) ==> eon
11132 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11133 iRegL src1, iRegL src2,
11134 immI src3, immL_M1 src4) %{
11135 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11136 ins_cost(1.9 * INSN_COST);
11137 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11138
11139 ins_encode %{
11140 __ eon(as_Register($dst$$reg),
11141 as_Register($src1$$reg),
11142 as_Register($src2$$reg),
11143 Assembler::LSR,
11144 $src3$$constant & 0x3f);
11145 %}
11146
11147 ins_pipe(ialu_reg_reg_shift);
11148 %}
11149
11150 // This pattern is automatically generated from aarch64_ad.m4
11151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11152 // val ^ (-1 ^ (val >> shift)) ==> eonw
11153 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11154 iRegIorL2I src1, iRegIorL2I src2,
11155 immI src3, immI_M1 src4) %{
11156 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11157 ins_cost(1.9 * INSN_COST);
11158 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11159
11160 ins_encode %{
11161 __ eonw(as_Register($dst$$reg),
11162 as_Register($src1$$reg),
11163 as_Register($src2$$reg),
11164 Assembler::ASR,
11165 $src3$$constant & 0x1f);
11166 %}
11167
11168 ins_pipe(ialu_reg_reg_shift);
11169 %}
11170
11171 // This pattern is automatically generated from aarch64_ad.m4
11172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11173 // val ^ (-1 ^ (val >> shift)) ==> eon
11174 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11175 iRegL src1, iRegL src2,
11176 immI src3, immL_M1 src4) %{
11177 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11178 ins_cost(1.9 * INSN_COST);
11179 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11180
11181 ins_encode %{
11182 __ eon(as_Register($dst$$reg),
11183 as_Register($src1$$reg),
11184 as_Register($src2$$reg),
11185 Assembler::ASR,
11186 $src3$$constant & 0x3f);
11187 %}
11188
11189 ins_pipe(ialu_reg_reg_shift);
11190 %}
11191
11192 // This pattern is automatically generated from aarch64_ad.m4
11193 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11194 // val ^ (-1 ^ (val ror shift)) ==> eonw
11195 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11196 iRegIorL2I src1, iRegIorL2I src2,
11197 immI src3, immI_M1 src4) %{
11198 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11199 ins_cost(1.9 * INSN_COST);
11200 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11201
11202 ins_encode %{
11203 __ eonw(as_Register($dst$$reg),
11204 as_Register($src1$$reg),
11205 as_Register($src2$$reg),
11206 Assembler::ROR,
11207 $src3$$constant & 0x1f);
11208 %}
11209
11210 ins_pipe(ialu_reg_reg_shift);
11211 %}
11212
11213 // This pattern is automatically generated from aarch64_ad.m4
11214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11215 // val ^ (-1 ^ (val ror shift)) ==> eon
11216 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11217 iRegL src1, iRegL src2,
11218 immI src3, immL_M1 src4) %{
11219 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11220 ins_cost(1.9 * INSN_COST);
11221 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11222
11223 ins_encode %{
11224 __ eon(as_Register($dst$$reg),
11225 as_Register($src1$$reg),
11226 as_Register($src2$$reg),
11227 Assembler::ROR,
11228 $src3$$constant & 0x3f);
11229 %}
11230
11231 ins_pipe(ialu_reg_reg_shift);
11232 %}
11233
11234 // This pattern is automatically generated from aarch64_ad.m4
11235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11236 // val ^ (-1 ^ (val << shift)) ==> eonw
11237 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11238 iRegIorL2I src1, iRegIorL2I src2,
11239 immI src3, immI_M1 src4) %{
11240 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11241 ins_cost(1.9 * INSN_COST);
11242 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11243
11244 ins_encode %{
11245 __ eonw(as_Register($dst$$reg),
11246 as_Register($src1$$reg),
11247 as_Register($src2$$reg),
11248 Assembler::LSL,
11249 $src3$$constant & 0x1f);
11250 %}
11251
11252 ins_pipe(ialu_reg_reg_shift);
11253 %}
11254
11255 // This pattern is automatically generated from aarch64_ad.m4
11256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11257 // val ^ (-1 ^ (val << shift)) ==> eon
11258 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11259 iRegL src1, iRegL src2,
11260 immI src3, immL_M1 src4) %{
11261 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11262 ins_cost(1.9 * INSN_COST);
11263 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11264
11265 ins_encode %{
11266 __ eon(as_Register($dst$$reg),
11267 as_Register($src1$$reg),
11268 as_Register($src2$$reg),
11269 Assembler::LSL,
11270 $src3$$constant & 0x3f);
11271 %}
11272
11273 ins_pipe(ialu_reg_reg_shift);
11274 %}
11275
11276 // This pattern is automatically generated from aarch64_ad.m4
11277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11278 // val | (-1 ^ (val >>> shift)) ==> ornw
11279 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11280 iRegIorL2I src1, iRegIorL2I src2,
11281 immI src3, immI_M1 src4) %{
11282 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11283 ins_cost(1.9 * INSN_COST);
11284 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11285
11286 ins_encode %{
11287 __ ornw(as_Register($dst$$reg),
11288 as_Register($src1$$reg),
11289 as_Register($src2$$reg),
11290 Assembler::LSR,
11291 $src3$$constant & 0x1f);
11292 %}
11293
11294 ins_pipe(ialu_reg_reg_shift);
11295 %}
11296
11297 // This pattern is automatically generated from aarch64_ad.m4
11298 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11299 // val | (-1 ^ (val >>> shift)) ==> orn
11300 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11301 iRegL src1, iRegL src2,
11302 immI src3, immL_M1 src4) %{
11303 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11304 ins_cost(1.9 * INSN_COST);
11305 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11306
11307 ins_encode %{
11308 __ orn(as_Register($dst$$reg),
11309 as_Register($src1$$reg),
11310 as_Register($src2$$reg),
11311 Assembler::LSR,
11312 $src3$$constant & 0x3f);
11313 %}
11314
11315 ins_pipe(ialu_reg_reg_shift);
11316 %}
11317
11318 // This pattern is automatically generated from aarch64_ad.m4
11319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11320 // val | (-1 ^ (val >> shift)) ==> ornw
11321 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11322 iRegIorL2I src1, iRegIorL2I src2,
11323 immI src3, immI_M1 src4) %{
11324 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11325 ins_cost(1.9 * INSN_COST);
11326 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11327
11328 ins_encode %{
11329 __ ornw(as_Register($dst$$reg),
11330 as_Register($src1$$reg),
11331 as_Register($src2$$reg),
11332 Assembler::ASR,
11333 $src3$$constant & 0x1f);
11334 %}
11335
11336 ins_pipe(ialu_reg_reg_shift);
11337 %}
11338
11339 // This pattern is automatically generated from aarch64_ad.m4
11340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11341 // val | (-1 ^ (val >> shift)) ==> orn
11342 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11343 iRegL src1, iRegL src2,
11344 immI src3, immL_M1 src4) %{
11345 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11346 ins_cost(1.9 * INSN_COST);
11347 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11348
11349 ins_encode %{
11350 __ orn(as_Register($dst$$reg),
11351 as_Register($src1$$reg),
11352 as_Register($src2$$reg),
11353 Assembler::ASR,
11354 $src3$$constant & 0x3f);
11355 %}
11356
11357 ins_pipe(ialu_reg_reg_shift);
11358 %}
11359
11360 // This pattern is automatically generated from aarch64_ad.m4
11361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11362 // val | (-1 ^ (val ror shift)) ==> ornw
11363 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11364 iRegIorL2I src1, iRegIorL2I src2,
11365 immI src3, immI_M1 src4) %{
11366 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11367 ins_cost(1.9 * INSN_COST);
11368 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11369
11370 ins_encode %{
11371 __ ornw(as_Register($dst$$reg),
11372 as_Register($src1$$reg),
11373 as_Register($src2$$reg),
11374 Assembler::ROR,
11375 $src3$$constant & 0x1f);
11376 %}
11377
11378 ins_pipe(ialu_reg_reg_shift);
11379 %}
11380
11381 // This pattern is automatically generated from aarch64_ad.m4
11382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11383 // val | (-1 ^ (val ror shift)) ==> orn
11384 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11385 iRegL src1, iRegL src2,
11386 immI src3, immL_M1 src4) %{
11387 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11388 ins_cost(1.9 * INSN_COST);
11389 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11390
11391 ins_encode %{
11392 __ orn(as_Register($dst$$reg),
11393 as_Register($src1$$reg),
11394 as_Register($src2$$reg),
11395 Assembler::ROR,
11396 $src3$$constant & 0x3f);
11397 %}
11398
11399 ins_pipe(ialu_reg_reg_shift);
11400 %}
11401
11402 // This pattern is automatically generated from aarch64_ad.m4
11403 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11404 // val | (-1 ^ (val << shift)) ==> ornw
11405 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11406 iRegIorL2I src1, iRegIorL2I src2,
11407 immI src3, immI_M1 src4) %{
11408 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11409 ins_cost(1.9 * INSN_COST);
11410 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11411
11412 ins_encode %{
11413 __ ornw(as_Register($dst$$reg),
11414 as_Register($src1$$reg),
11415 as_Register($src2$$reg),
11416 Assembler::LSL,
11417 $src3$$constant & 0x1f);
11418 %}
11419
11420 ins_pipe(ialu_reg_reg_shift);
11421 %}
11422
11423 // This pattern is automatically generated from aarch64_ad.m4
11424 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11425 // val | (-1 ^ (val << shift)) ==> orn
11426 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11427 iRegL src1, iRegL src2,
11428 immI src3, immL_M1 src4) %{
11429 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11430 ins_cost(1.9 * INSN_COST);
11431 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11432
11433 ins_encode %{
11434 __ orn(as_Register($dst$$reg),
11435 as_Register($src1$$reg),
11436 as_Register($src2$$reg),
11437 Assembler::LSL,
11438 $src3$$constant & 0x3f);
11439 %}
11440
11441 ins_pipe(ialu_reg_reg_shift);
11442 %}
11443
11444 // This pattern is automatically generated from aarch64_ad.m4
11445 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11446 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11447 iRegIorL2I src1, iRegIorL2I src2,
11448 immI src3) %{
11449 match(Set dst (AndI src1 (URShiftI src2 src3)));
11450
11451 ins_cost(1.9 * INSN_COST);
11452 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11453
11454 ins_encode %{
11455 __ andw(as_Register($dst$$reg),
11456 as_Register($src1$$reg),
11457 as_Register($src2$$reg),
11458 Assembler::LSR,
11459 $src3$$constant & 0x1f);
11460 %}
11461
11462 ins_pipe(ialu_reg_reg_shift);
11463 %}
11464
11465 // This pattern is automatically generated from aarch64_ad.m4
11466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11467 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11468 iRegL src1, iRegL src2,
11469 immI src3) %{
11470 match(Set dst (AndL src1 (URShiftL src2 src3)));
11471
11472 ins_cost(1.9 * INSN_COST);
11473 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11474
11475 ins_encode %{
11476 __ andr(as_Register($dst$$reg),
11477 as_Register($src1$$reg),
11478 as_Register($src2$$reg),
11479 Assembler::LSR,
11480 $src3$$constant & 0x3f);
11481 %}
11482
11483 ins_pipe(ialu_reg_reg_shift);
11484 %}
11485
11486 // This pattern is automatically generated from aarch64_ad.m4
11487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11488 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11489 iRegIorL2I src1, iRegIorL2I src2,
11490 immI src3) %{
11491 match(Set dst (AndI src1 (RShiftI src2 src3)));
11492
11493 ins_cost(1.9 * INSN_COST);
11494 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11495
11496 ins_encode %{
11497 __ andw(as_Register($dst$$reg),
11498 as_Register($src1$$reg),
11499 as_Register($src2$$reg),
11500 Assembler::ASR,
11501 $src3$$constant & 0x1f);
11502 %}
11503
11504 ins_pipe(ialu_reg_reg_shift);
11505 %}
11506
11507 // This pattern is automatically generated from aarch64_ad.m4
11508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11509 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11510 iRegL src1, iRegL src2,
11511 immI src3) %{
11512 match(Set dst (AndL src1 (RShiftL src2 src3)));
11513
11514 ins_cost(1.9 * INSN_COST);
11515 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11516
11517 ins_encode %{
11518 __ andr(as_Register($dst$$reg),
11519 as_Register($src1$$reg),
11520 as_Register($src2$$reg),
11521 Assembler::ASR,
11522 $src3$$constant & 0x3f);
11523 %}
11524
11525 ins_pipe(ialu_reg_reg_shift);
11526 %}
11527
11528 // This pattern is automatically generated from aarch64_ad.m4
11529 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11530 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11531 iRegIorL2I src1, iRegIorL2I src2,
11532 immI src3) %{
11533 match(Set dst (AndI src1 (LShiftI src2 src3)));
11534
11535 ins_cost(1.9 * INSN_COST);
11536 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11537
11538 ins_encode %{
11539 __ andw(as_Register($dst$$reg),
11540 as_Register($src1$$reg),
11541 as_Register($src2$$reg),
11542 Assembler::LSL,
11543 $src3$$constant & 0x1f);
11544 %}
11545
11546 ins_pipe(ialu_reg_reg_shift);
11547 %}
11548
11549 // This pattern is automatically generated from aarch64_ad.m4
11550 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11551 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11552 iRegL src1, iRegL src2,
11553 immI src3) %{
11554 match(Set dst (AndL src1 (LShiftL src2 src3)));
11555
11556 ins_cost(1.9 * INSN_COST);
11557 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11558
11559 ins_encode %{
11560 __ andr(as_Register($dst$$reg),
11561 as_Register($src1$$reg),
11562 as_Register($src2$$reg),
11563 Assembler::LSL,
11564 $src3$$constant & 0x3f);
11565 %}
11566
11567 ins_pipe(ialu_reg_reg_shift);
11568 %}
11569
11570 // This pattern is automatically generated from aarch64_ad.m4
11571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11572 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11573 iRegIorL2I src1, iRegIorL2I src2,
11574 immI src3) %{
11575 match(Set dst (AndI src1 (RotateRight src2 src3)));
11576
11577 ins_cost(1.9 * INSN_COST);
11578 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11579
11580 ins_encode %{
11581 __ andw(as_Register($dst$$reg),
11582 as_Register($src1$$reg),
11583 as_Register($src2$$reg),
11584 Assembler::ROR,
11585 $src3$$constant & 0x1f);
11586 %}
11587
11588 ins_pipe(ialu_reg_reg_shift);
11589 %}
11590
11591 // This pattern is automatically generated from aarch64_ad.m4
11592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11593 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11594 iRegL src1, iRegL src2,
11595 immI src3) %{
11596 match(Set dst (AndL src1 (RotateRight src2 src3)));
11597
11598 ins_cost(1.9 * INSN_COST);
11599 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11600
11601 ins_encode %{
11602 __ andr(as_Register($dst$$reg),
11603 as_Register($src1$$reg),
11604 as_Register($src2$$reg),
11605 Assembler::ROR,
11606 $src3$$constant & 0x3f);
11607 %}
11608
11609 ins_pipe(ialu_reg_reg_shift);
11610 %}
11611
11612 // This pattern is automatically generated from aarch64_ad.m4
11613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11614 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11615 iRegIorL2I src1, iRegIorL2I src2,
11616 immI src3) %{
11617 match(Set dst (XorI src1 (URShiftI src2 src3)));
11618
11619 ins_cost(1.9 * INSN_COST);
11620 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11621
11622 ins_encode %{
11623 __ eorw(as_Register($dst$$reg),
11624 as_Register($src1$$reg),
11625 as_Register($src2$$reg),
11626 Assembler::LSR,
11627 $src3$$constant & 0x1f);
11628 %}
11629
11630 ins_pipe(ialu_reg_reg_shift);
11631 %}
11632
11633 // This pattern is automatically generated from aarch64_ad.m4
11634 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11635 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11636 iRegL src1, iRegL src2,
11637 immI src3) %{
11638 match(Set dst (XorL src1 (URShiftL src2 src3)));
11639
11640 ins_cost(1.9 * INSN_COST);
11641 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11642
11643 ins_encode %{
11644 __ eor(as_Register($dst$$reg),
11645 as_Register($src1$$reg),
11646 as_Register($src2$$reg),
11647 Assembler::LSR,
11648 $src3$$constant & 0x3f);
11649 %}
11650
11651 ins_pipe(ialu_reg_reg_shift);
11652 %}
11653
11654 // This pattern is automatically generated from aarch64_ad.m4
11655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11656 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11657 iRegIorL2I src1, iRegIorL2I src2,
11658 immI src3) %{
11659 match(Set dst (XorI src1 (RShiftI src2 src3)));
11660
11661 ins_cost(1.9 * INSN_COST);
11662 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11663
11664 ins_encode %{
11665 __ eorw(as_Register($dst$$reg),
11666 as_Register($src1$$reg),
11667 as_Register($src2$$reg),
11668 Assembler::ASR,
11669 $src3$$constant & 0x1f);
11670 %}
11671
11672 ins_pipe(ialu_reg_reg_shift);
11673 %}
11674
11675 // This pattern is automatically generated from aarch64_ad.m4
11676 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11677 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11678 iRegL src1, iRegL src2,
11679 immI src3) %{
11680 match(Set dst (XorL src1 (RShiftL src2 src3)));
11681
11682 ins_cost(1.9 * INSN_COST);
11683 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11684
11685 ins_encode %{
11686 __ eor(as_Register($dst$$reg),
11687 as_Register($src1$$reg),
11688 as_Register($src2$$reg),
11689 Assembler::ASR,
11690 $src3$$constant & 0x3f);
11691 %}
11692
11693 ins_pipe(ialu_reg_reg_shift);
11694 %}
11695
11696 // This pattern is automatically generated from aarch64_ad.m4
11697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11698 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11699 iRegIorL2I src1, iRegIorL2I src2,
11700 immI src3) %{
11701 match(Set dst (XorI src1 (LShiftI src2 src3)));
11702
11703 ins_cost(1.9 * INSN_COST);
11704 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11705
11706 ins_encode %{
11707 __ eorw(as_Register($dst$$reg),
11708 as_Register($src1$$reg),
11709 as_Register($src2$$reg),
11710 Assembler::LSL,
11711 $src3$$constant & 0x1f);
11712 %}
11713
11714 ins_pipe(ialu_reg_reg_shift);
11715 %}
11716
11717 // This pattern is automatically generated from aarch64_ad.m4
11718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11719 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11720 iRegL src1, iRegL src2,
11721 immI src3) %{
11722 match(Set dst (XorL src1 (LShiftL src2 src3)));
11723
11724 ins_cost(1.9 * INSN_COST);
11725 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11726
11727 ins_encode %{
11728 __ eor(as_Register($dst$$reg),
11729 as_Register($src1$$reg),
11730 as_Register($src2$$reg),
11731 Assembler::LSL,
11732 $src3$$constant & 0x3f);
11733 %}
11734
11735 ins_pipe(ialu_reg_reg_shift);
11736 %}
11737
11738 // This pattern is automatically generated from aarch64_ad.m4
11739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11740 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11741 iRegIorL2I src1, iRegIorL2I src2,
11742 immI src3) %{
11743 match(Set dst (XorI src1 (RotateRight src2 src3)));
11744
11745 ins_cost(1.9 * INSN_COST);
11746 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11747
11748 ins_encode %{
11749 __ eorw(as_Register($dst$$reg),
11750 as_Register($src1$$reg),
11751 as_Register($src2$$reg),
11752 Assembler::ROR,
11753 $src3$$constant & 0x1f);
11754 %}
11755
11756 ins_pipe(ialu_reg_reg_shift);
11757 %}
11758
11759 // This pattern is automatically generated from aarch64_ad.m4
11760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11761 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11762 iRegL src1, iRegL src2,
11763 immI src3) %{
11764 match(Set dst (XorL src1 (RotateRight src2 src3)));
11765
11766 ins_cost(1.9 * INSN_COST);
11767 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11768
11769 ins_encode %{
11770 __ eor(as_Register($dst$$reg),
11771 as_Register($src1$$reg),
11772 as_Register($src2$$reg),
11773 Assembler::ROR,
11774 $src3$$constant & 0x3f);
11775 %}
11776
11777 ins_pipe(ialu_reg_reg_shift);
11778 %}
11779
11780 // This pattern is automatically generated from aarch64_ad.m4
11781 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11782 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11783 iRegIorL2I src1, iRegIorL2I src2,
11784 immI src3) %{
11785 match(Set dst (OrI src1 (URShiftI src2 src3)));
11786
11787 ins_cost(1.9 * INSN_COST);
11788 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11789
11790 ins_encode %{
11791 __ orrw(as_Register($dst$$reg),
11792 as_Register($src1$$reg),
11793 as_Register($src2$$reg),
11794 Assembler::LSR,
11795 $src3$$constant & 0x1f);
11796 %}
11797
11798 ins_pipe(ialu_reg_reg_shift);
11799 %}
11800
11801 // This pattern is automatically generated from aarch64_ad.m4
11802 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11803 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11804 iRegL src1, iRegL src2,
11805 immI src3) %{
11806 match(Set dst (OrL src1 (URShiftL src2 src3)));
11807
11808 ins_cost(1.9 * INSN_COST);
11809 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11810
11811 ins_encode %{
11812 __ orr(as_Register($dst$$reg),
11813 as_Register($src1$$reg),
11814 as_Register($src2$$reg),
11815 Assembler::LSR,
11816 $src3$$constant & 0x3f);
11817 %}
11818
11819 ins_pipe(ialu_reg_reg_shift);
11820 %}
11821
11822 // This pattern is automatically generated from aarch64_ad.m4
11823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11824 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11825 iRegIorL2I src1, iRegIorL2I src2,
11826 immI src3) %{
11827 match(Set dst (OrI src1 (RShiftI src2 src3)));
11828
11829 ins_cost(1.9 * INSN_COST);
11830 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11831
11832 ins_encode %{
11833 __ orrw(as_Register($dst$$reg),
11834 as_Register($src1$$reg),
11835 as_Register($src2$$reg),
11836 Assembler::ASR,
11837 $src3$$constant & 0x1f);
11838 %}
11839
11840 ins_pipe(ialu_reg_reg_shift);
11841 %}
11842
11843 // This pattern is automatically generated from aarch64_ad.m4
11844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11845 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11846 iRegL src1, iRegL src2,
11847 immI src3) %{
11848 match(Set dst (OrL src1 (RShiftL src2 src3)));
11849
11850 ins_cost(1.9 * INSN_COST);
11851 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11852
11853 ins_encode %{
11854 __ orr(as_Register($dst$$reg),
11855 as_Register($src1$$reg),
11856 as_Register($src2$$reg),
11857 Assembler::ASR,
11858 $src3$$constant & 0x3f);
11859 %}
11860
11861 ins_pipe(ialu_reg_reg_shift);
11862 %}
11863
11864 // This pattern is automatically generated from aarch64_ad.m4
11865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11866 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11867 iRegIorL2I src1, iRegIorL2I src2,
11868 immI src3) %{
11869 match(Set dst (OrI src1 (LShiftI src2 src3)));
11870
11871 ins_cost(1.9 * INSN_COST);
11872 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11873
11874 ins_encode %{
11875 __ orrw(as_Register($dst$$reg),
11876 as_Register($src1$$reg),
11877 as_Register($src2$$reg),
11878 Assembler::LSL,
11879 $src3$$constant & 0x1f);
11880 %}
11881
11882 ins_pipe(ialu_reg_reg_shift);
11883 %}
11884
11885 // This pattern is automatically generated from aarch64_ad.m4
11886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11887 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11888 iRegL src1, iRegL src2,
11889 immI src3) %{
11890 match(Set dst (OrL src1 (LShiftL src2 src3)));
11891
11892 ins_cost(1.9 * INSN_COST);
11893 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11894
11895 ins_encode %{
11896 __ orr(as_Register($dst$$reg),
11897 as_Register($src1$$reg),
11898 as_Register($src2$$reg),
11899 Assembler::LSL,
11900 $src3$$constant & 0x3f);
11901 %}
11902
11903 ins_pipe(ialu_reg_reg_shift);
11904 %}
11905
11906 // This pattern is automatically generated from aarch64_ad.m4
11907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11908 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11909 iRegIorL2I src1, iRegIorL2I src2,
11910 immI src3) %{
11911 match(Set dst (OrI src1 (RotateRight src2 src3)));
11912
11913 ins_cost(1.9 * INSN_COST);
11914 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11915
11916 ins_encode %{
11917 __ orrw(as_Register($dst$$reg),
11918 as_Register($src1$$reg),
11919 as_Register($src2$$reg),
11920 Assembler::ROR,
11921 $src3$$constant & 0x1f);
11922 %}
11923
11924 ins_pipe(ialu_reg_reg_shift);
11925 %}
11926
11927 // This pattern is automatically generated from aarch64_ad.m4
11928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11929 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11930 iRegL src1, iRegL src2,
11931 immI src3) %{
11932 match(Set dst (OrL src1 (RotateRight src2 src3)));
11933
11934 ins_cost(1.9 * INSN_COST);
11935 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11936
11937 ins_encode %{
11938 __ orr(as_Register($dst$$reg),
11939 as_Register($src1$$reg),
11940 as_Register($src2$$reg),
11941 Assembler::ROR,
11942 $src3$$constant & 0x3f);
11943 %}
11944
11945 ins_pipe(ialu_reg_reg_shift);
11946 %}
11947
11948 // This pattern is automatically generated from aarch64_ad.m4
11949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11950 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11951 iRegIorL2I src1, iRegIorL2I src2,
11952 immI src3) %{
11953 match(Set dst (AddI src1 (URShiftI src2 src3)));
11954
11955 ins_cost(1.9 * INSN_COST);
11956 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11957
11958 ins_encode %{
11959 __ addw(as_Register($dst$$reg),
11960 as_Register($src1$$reg),
11961 as_Register($src2$$reg),
11962 Assembler::LSR,
11963 $src3$$constant & 0x1f);
11964 %}
11965
11966 ins_pipe(ialu_reg_reg_shift);
11967 %}
11968
11969 // This pattern is automatically generated from aarch64_ad.m4
11970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11971 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11972 iRegL src1, iRegL src2,
11973 immI src3) %{
11974 match(Set dst (AddL src1 (URShiftL src2 src3)));
11975
11976 ins_cost(1.9 * INSN_COST);
11977 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11978
11979 ins_encode %{
11980 __ add(as_Register($dst$$reg),
11981 as_Register($src1$$reg),
11982 as_Register($src2$$reg),
11983 Assembler::LSR,
11984 $src3$$constant & 0x3f);
11985 %}
11986
11987 ins_pipe(ialu_reg_reg_shift);
11988 %}
11989
11990 // This pattern is automatically generated from aarch64_ad.m4
11991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11992 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11993 iRegIorL2I src1, iRegIorL2I src2,
11994 immI src3) %{
11995 match(Set dst (AddI src1 (RShiftI src2 src3)));
11996
11997 ins_cost(1.9 * INSN_COST);
11998 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11999
12000 ins_encode %{
12001 __ addw(as_Register($dst$$reg),
12002 as_Register($src1$$reg),
12003 as_Register($src2$$reg),
12004 Assembler::ASR,
12005 $src3$$constant & 0x1f);
12006 %}
12007
12008 ins_pipe(ialu_reg_reg_shift);
12009 %}
12010
12011 // This pattern is automatically generated from aarch64_ad.m4
12012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12013 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12014 iRegL src1, iRegL src2,
12015 immI src3) %{
12016 match(Set dst (AddL src1 (RShiftL src2 src3)));
12017
12018 ins_cost(1.9 * INSN_COST);
12019 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12020
12021 ins_encode %{
12022 __ add(as_Register($dst$$reg),
12023 as_Register($src1$$reg),
12024 as_Register($src2$$reg),
12025 Assembler::ASR,
12026 $src3$$constant & 0x3f);
12027 %}
12028
12029 ins_pipe(ialu_reg_reg_shift);
12030 %}
12031
12032 // This pattern is automatically generated from aarch64_ad.m4
12033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12034 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12035 iRegIorL2I src1, iRegIorL2I src2,
12036 immI src3) %{
12037 match(Set dst (AddI src1 (LShiftI src2 src3)));
12038
12039 ins_cost(1.9 * INSN_COST);
12040 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12041
12042 ins_encode %{
12043 __ addw(as_Register($dst$$reg),
12044 as_Register($src1$$reg),
12045 as_Register($src2$$reg),
12046 Assembler::LSL,
12047 $src3$$constant & 0x1f);
12048 %}
12049
12050 ins_pipe(ialu_reg_reg_shift);
12051 %}
12052
12053 // This pattern is automatically generated from aarch64_ad.m4
12054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12055 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12056 iRegL src1, iRegL src2,
12057 immI src3) %{
12058 match(Set dst (AddL src1 (LShiftL src2 src3)));
12059
12060 ins_cost(1.9 * INSN_COST);
12061 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12062
12063 ins_encode %{
12064 __ add(as_Register($dst$$reg),
12065 as_Register($src1$$reg),
12066 as_Register($src2$$reg),
12067 Assembler::LSL,
12068 $src3$$constant & 0x3f);
12069 %}
12070
12071 ins_pipe(ialu_reg_reg_shift);
12072 %}
12073
12074 // This pattern is automatically generated from aarch64_ad.m4
12075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12076 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12077 iRegIorL2I src1, iRegIorL2I src2,
12078 immI src3) %{
12079 match(Set dst (SubI src1 (URShiftI src2 src3)));
12080
12081 ins_cost(1.9 * INSN_COST);
12082 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12083
12084 ins_encode %{
12085 __ subw(as_Register($dst$$reg),
12086 as_Register($src1$$reg),
12087 as_Register($src2$$reg),
12088 Assembler::LSR,
12089 $src3$$constant & 0x1f);
12090 %}
12091
12092 ins_pipe(ialu_reg_reg_shift);
12093 %}
12094
12095 // This pattern is automatically generated from aarch64_ad.m4
12096 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12097 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12098 iRegL src1, iRegL src2,
12099 immI src3) %{
12100 match(Set dst (SubL src1 (URShiftL src2 src3)));
12101
12102 ins_cost(1.9 * INSN_COST);
12103 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12104
12105 ins_encode %{
12106 __ sub(as_Register($dst$$reg),
12107 as_Register($src1$$reg),
12108 as_Register($src2$$reg),
12109 Assembler::LSR,
12110 $src3$$constant & 0x3f);
12111 %}
12112
12113 ins_pipe(ialu_reg_reg_shift);
12114 %}
12115
12116 // This pattern is automatically generated from aarch64_ad.m4
12117 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12118 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12119 iRegIorL2I src1, iRegIorL2I src2,
12120 immI src3) %{
12121 match(Set dst (SubI src1 (RShiftI src2 src3)));
12122
12123 ins_cost(1.9 * INSN_COST);
12124 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12125
12126 ins_encode %{
12127 __ subw(as_Register($dst$$reg),
12128 as_Register($src1$$reg),
12129 as_Register($src2$$reg),
12130 Assembler::ASR,
12131 $src3$$constant & 0x1f);
12132 %}
12133
12134 ins_pipe(ialu_reg_reg_shift);
12135 %}
12136
12137 // This pattern is automatically generated from aarch64_ad.m4
12138 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12139 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12140 iRegL src1, iRegL src2,
12141 immI src3) %{
12142 match(Set dst (SubL src1 (RShiftL src2 src3)));
12143
12144 ins_cost(1.9 * INSN_COST);
12145 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12146
12147 ins_encode %{
12148 __ sub(as_Register($dst$$reg),
12149 as_Register($src1$$reg),
12150 as_Register($src2$$reg),
12151 Assembler::ASR,
12152 $src3$$constant & 0x3f);
12153 %}
12154
12155 ins_pipe(ialu_reg_reg_shift);
12156 %}
12157
12158 // This pattern is automatically generated from aarch64_ad.m4
12159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12160 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12161 iRegIorL2I src1, iRegIorL2I src2,
12162 immI src3) %{
12163 match(Set dst (SubI src1 (LShiftI src2 src3)));
12164
12165 ins_cost(1.9 * INSN_COST);
12166 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12167
12168 ins_encode %{
12169 __ subw(as_Register($dst$$reg),
12170 as_Register($src1$$reg),
12171 as_Register($src2$$reg),
12172 Assembler::LSL,
12173 $src3$$constant & 0x1f);
12174 %}
12175
12176 ins_pipe(ialu_reg_reg_shift);
12177 %}
12178
12179 // This pattern is automatically generated from aarch64_ad.m4
12180 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12181 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12182 iRegL src1, iRegL src2,
12183 immI src3) %{
12184 match(Set dst (SubL src1 (LShiftL src2 src3)));
12185
12186 ins_cost(1.9 * INSN_COST);
12187 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12188
12189 ins_encode %{
12190 __ sub(as_Register($dst$$reg),
12191 as_Register($src1$$reg),
12192 as_Register($src2$$reg),
12193 Assembler::LSL,
12194 $src3$$constant & 0x3f);
12195 %}
12196
12197 ins_pipe(ialu_reg_reg_shift);
12198 %}
12199
12200 // This pattern is automatically generated from aarch64_ad.m4
12201 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12202
12203 // Shift Left followed by Shift Right.
12204 // This idiom is used by the compiler for the i2b bytecode etc.
12205 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12206 %{
12207 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12208 ins_cost(INSN_COST * 2);
12209 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12210 ins_encode %{
12211 int lshift = $lshift_count$$constant & 63;
12212 int rshift = $rshift_count$$constant & 63;
12213 int s = 63 - lshift;
12214 int r = (rshift - lshift) & 63;
12215 __ sbfm(as_Register($dst$$reg),
12216 as_Register($src$$reg),
12217 r, s);
12218 %}
12219
12220 ins_pipe(ialu_reg_shift);
12221 %}
12222
12223 // This pattern is automatically generated from aarch64_ad.m4
12224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12225
12226 // Shift Left followed by Shift Right.
12227 // This idiom is used by the compiler for the i2b bytecode etc.
12228 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12229 %{
12230 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12231 ins_cost(INSN_COST * 2);
12232 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12233 ins_encode %{
12234 int lshift = $lshift_count$$constant & 31;
12235 int rshift = $rshift_count$$constant & 31;
12236 int s = 31 - lshift;
12237 int r = (rshift - lshift) & 31;
12238 __ sbfmw(as_Register($dst$$reg),
12239 as_Register($src$$reg),
12240 r, s);
12241 %}
12242
12243 ins_pipe(ialu_reg_shift);
12244 %}
12245
12246 // This pattern is automatically generated from aarch64_ad.m4
12247 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12248
12249 // Shift Left followed by Shift Right.
12250 // This idiom is used by the compiler for the i2b bytecode etc.
12251 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12252 %{
12253 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12254 ins_cost(INSN_COST * 2);
12255 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12256 ins_encode %{
12257 int lshift = $lshift_count$$constant & 63;
12258 int rshift = $rshift_count$$constant & 63;
12259 int s = 63 - lshift;
12260 int r = (rshift - lshift) & 63;
12261 __ ubfm(as_Register($dst$$reg),
12262 as_Register($src$$reg),
12263 r, s);
12264 %}
12265
12266 ins_pipe(ialu_reg_shift);
12267 %}
12268
12269 // This pattern is automatically generated from aarch64_ad.m4
12270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12271
12272 // Shift Left followed by Shift Right.
12273 // This idiom is used by the compiler for the i2b bytecode etc.
12274 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12275 %{
12276 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12277 ins_cost(INSN_COST * 2);
12278 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12279 ins_encode %{
12280 int lshift = $lshift_count$$constant & 31;
12281 int rshift = $rshift_count$$constant & 31;
12282 int s = 31 - lshift;
12283 int r = (rshift - lshift) & 31;
12284 __ ubfmw(as_Register($dst$$reg),
12285 as_Register($src$$reg),
12286 r, s);
12287 %}
12288
12289 ins_pipe(ialu_reg_shift);
12290 %}
12291
12292 // Bitfield extract with shift & mask
12293
12294 // This pattern is automatically generated from aarch64_ad.m4
12295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12296 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12297 %{
12298 match(Set dst (AndI (URShiftI src rshift) mask));
12299 // Make sure we are not going to exceed what ubfxw can do.
12300 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12301
12302 ins_cost(INSN_COST);
12303 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12304 ins_encode %{
12305 int rshift = $rshift$$constant & 31;
12306 intptr_t mask = $mask$$constant;
12307 int width = exact_log2(mask+1);
12308 __ ubfxw(as_Register($dst$$reg),
12309 as_Register($src$$reg), rshift, width);
12310 %}
12311 ins_pipe(ialu_reg_shift);
12312 %}
12313
12314 // This pattern is automatically generated from aarch64_ad.m4
12315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12316 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12317 %{
12318 match(Set dst (AndL (URShiftL src rshift) mask));
12319 // Make sure we are not going to exceed what ubfx can do.
12320 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12321
12322 ins_cost(INSN_COST);
12323 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12324 ins_encode %{
12325 int rshift = $rshift$$constant & 63;
12326 intptr_t mask = $mask$$constant;
12327 int width = exact_log2_long(mask+1);
12328 __ ubfx(as_Register($dst$$reg),
12329 as_Register($src$$reg), rshift, width);
12330 %}
12331 ins_pipe(ialu_reg_shift);
12332 %}
12333
12334
12335 // This pattern is automatically generated from aarch64_ad.m4
12336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12337
12338 // We can use ubfx when extending an And with a mask when we know mask
12339 // is positive. We know that because immI_bitmask guarantees it.
12340 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12341 %{
12342 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12343 // Make sure we are not going to exceed what ubfxw can do.
12344 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12345
12346 ins_cost(INSN_COST * 2);
12347 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12348 ins_encode %{
12349 int rshift = $rshift$$constant & 31;
12350 intptr_t mask = $mask$$constant;
12351 int width = exact_log2(mask+1);
12352 __ ubfx(as_Register($dst$$reg),
12353 as_Register($src$$reg), rshift, width);
12354 %}
12355 ins_pipe(ialu_reg_shift);
12356 %}
12357
12358
12359 // This pattern is automatically generated from aarch64_ad.m4
12360 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12361
12362 // We can use ubfiz when masking by a positive number and then left shifting the result.
12363 // We know that the mask is positive because immI_bitmask guarantees it.
12364 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12365 %{
12366 match(Set dst (LShiftI (AndI src mask) lshift));
12367 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12368
12369 ins_cost(INSN_COST);
12370 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12371 ins_encode %{
12372 int lshift = $lshift$$constant & 31;
12373 intptr_t mask = $mask$$constant;
12374 int width = exact_log2(mask+1);
12375 __ ubfizw(as_Register($dst$$reg),
12376 as_Register($src$$reg), lshift, width);
12377 %}
12378 ins_pipe(ialu_reg_shift);
12379 %}
12380
12381 // This pattern is automatically generated from aarch64_ad.m4
12382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12383
12384 // We can use ubfiz when masking by a positive number and then left shifting the result.
12385 // We know that the mask is positive because immL_bitmask guarantees it.
12386 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12387 %{
12388 match(Set dst (LShiftL (AndL src mask) lshift));
12389 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12390
12391 ins_cost(INSN_COST);
12392 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12393 ins_encode %{
12394 int lshift = $lshift$$constant & 63;
12395 intptr_t mask = $mask$$constant;
12396 int width = exact_log2_long(mask+1);
12397 __ ubfiz(as_Register($dst$$reg),
12398 as_Register($src$$reg), lshift, width);
12399 %}
12400 ins_pipe(ialu_reg_shift);
12401 %}
12402
12403 // This pattern is automatically generated from aarch64_ad.m4
12404 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12405
12406 // We can use ubfiz when masking by a positive number and then left shifting the result.
12407 // We know that the mask is positive because immI_bitmask guarantees it.
12408 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12409 %{
12410 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12411 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12412
12413 ins_cost(INSN_COST);
12414 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12415 ins_encode %{
12416 int lshift = $lshift$$constant & 31;
12417 intptr_t mask = $mask$$constant;
12418 int width = exact_log2(mask+1);
12419 __ ubfizw(as_Register($dst$$reg),
12420 as_Register($src$$reg), lshift, width);
12421 %}
12422 ins_pipe(ialu_reg_shift);
12423 %}
12424
12425 // This pattern is automatically generated from aarch64_ad.m4
12426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12427
12428 // We can use ubfiz when masking by a positive number and then left shifting the result.
12429 // We know that the mask is positive because immL_bitmask guarantees it.
12430 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12431 %{
12432 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12433 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12434
12435 ins_cost(INSN_COST);
12436 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12437 ins_encode %{
12438 int lshift = $lshift$$constant & 63;
12439 intptr_t mask = $mask$$constant;
12440 int width = exact_log2_long(mask+1);
12441 __ ubfiz(as_Register($dst$$reg),
12442 as_Register($src$$reg), lshift, width);
12443 %}
12444 ins_pipe(ialu_reg_shift);
12445 %}
12446
12447
12448 // This pattern is automatically generated from aarch64_ad.m4
12449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12450
12451 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12452 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12453 %{
12454 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12455 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12456
12457 ins_cost(INSN_COST);
12458 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12459 ins_encode %{
12460 int lshift = $lshift$$constant & 63;
12461 intptr_t mask = $mask$$constant;
12462 int width = exact_log2(mask+1);
12463 __ ubfiz(as_Register($dst$$reg),
12464 as_Register($src$$reg), lshift, width);
12465 %}
12466 ins_pipe(ialu_reg_shift);
12467 %}
12468
12469 // This pattern is automatically generated from aarch64_ad.m4
12470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12471
12472 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12473 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12474 %{
12475 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12476 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12477
12478 ins_cost(INSN_COST);
12479 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12480 ins_encode %{
12481 int lshift = $lshift$$constant & 31;
12482 intptr_t mask = $mask$$constant;
12483 int width = exact_log2(mask+1);
12484 __ ubfiz(as_Register($dst$$reg),
12485 as_Register($src$$reg), lshift, width);
12486 %}
12487 ins_pipe(ialu_reg_shift);
12488 %}
12489
12490 // This pattern is automatically generated from aarch64_ad.m4
12491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12492
12493 // Can skip int2long conversions after AND with small bitmask
12494 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12495 %{
12496 match(Set dst (ConvI2L (AndI src msk)));
12497 ins_cost(INSN_COST);
12498 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12499 ins_encode %{
12500 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12501 %}
12502 ins_pipe(ialu_reg_shift);
12503 %}
12504
12505
12506 // Rotations
12507
12508 // This pattern is automatically generated from aarch64_ad.m4
12509 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12510 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12511 %{
12512 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12513 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12514
12515 ins_cost(INSN_COST);
12516 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12517
12518 ins_encode %{
12519 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12520 $rshift$$constant & 63);
12521 %}
12522 ins_pipe(ialu_reg_reg_extr);
12523 %}
12524
12525
12526 // This pattern is automatically generated from aarch64_ad.m4
12527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12528 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12529 %{
12530 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12531 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12532
12533 ins_cost(INSN_COST);
12534 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12535
12536 ins_encode %{
12537 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12538 $rshift$$constant & 31);
12539 %}
12540 ins_pipe(ialu_reg_reg_extr);
12541 %}
12542
12543
12544 // This pattern is automatically generated from aarch64_ad.m4
12545 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12546 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12547 %{
12548 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12549 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12550
12551 ins_cost(INSN_COST);
12552 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12553
12554 ins_encode %{
12555 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12556 $rshift$$constant & 63);
12557 %}
12558 ins_pipe(ialu_reg_reg_extr);
12559 %}
12560
12561
12562 // This pattern is automatically generated from aarch64_ad.m4
12563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12564 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12565 %{
12566 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12567 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12568
12569 ins_cost(INSN_COST);
12570 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12571
12572 ins_encode %{
12573 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12574 $rshift$$constant & 31);
12575 %}
12576 ins_pipe(ialu_reg_reg_extr);
12577 %}
12578
12579 // This pattern is automatically generated from aarch64_ad.m4
12580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12581 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12582 %{
12583 match(Set dst (RotateRight src shift));
12584
12585 ins_cost(INSN_COST);
12586 format %{ "ror $dst, $src, $shift" %}
12587
12588 ins_encode %{
12589 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12590 $shift$$constant & 0x1f);
12591 %}
12592 ins_pipe(ialu_reg_reg_vshift);
12593 %}
12594
12595 // This pattern is automatically generated from aarch64_ad.m4
12596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12597 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12598 %{
12599 match(Set dst (RotateRight src shift));
12600
12601 ins_cost(INSN_COST);
12602 format %{ "ror $dst, $src, $shift" %}
12603
12604 ins_encode %{
12605 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12606 $shift$$constant & 0x3f);
12607 %}
12608 ins_pipe(ialu_reg_reg_vshift);
12609 %}
12610
12611 // This pattern is automatically generated from aarch64_ad.m4
12612 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12613 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12614 %{
12615 match(Set dst (RotateRight src shift));
12616
12617 ins_cost(INSN_COST);
12618 format %{ "ror $dst, $src, $shift" %}
12619
12620 ins_encode %{
12621 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12622 %}
12623 ins_pipe(ialu_reg_reg_vshift);
12624 %}
12625
12626 // This pattern is automatically generated from aarch64_ad.m4
12627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12628 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12629 %{
12630 match(Set dst (RotateRight src shift));
12631
12632 ins_cost(INSN_COST);
12633 format %{ "ror $dst, $src, $shift" %}
12634
12635 ins_encode %{
12636 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12637 %}
12638 ins_pipe(ialu_reg_reg_vshift);
12639 %}
12640
12641 // This pattern is automatically generated from aarch64_ad.m4
12642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12643 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12644 %{
12645 match(Set dst (RotateLeft src shift));
12646
12647 ins_cost(INSN_COST);
12648 format %{ "rol $dst, $src, $shift" %}
12649
12650 ins_encode %{
12651 __ subw(rscratch1, zr, as_Register($shift$$reg));
12652 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12653 %}
12654 ins_pipe(ialu_reg_reg_vshift);
12655 %}
12656
12657 // This pattern is automatically generated from aarch64_ad.m4
12658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12659 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12660 %{
12661 match(Set dst (RotateLeft src shift));
12662
12663 ins_cost(INSN_COST);
12664 format %{ "rol $dst, $src, $shift" %}
12665
12666 ins_encode %{
12667 __ subw(rscratch1, zr, as_Register($shift$$reg));
12668 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12669 %}
12670 ins_pipe(ialu_reg_reg_vshift);
12671 %}
12672
12673
12674 // Add/subtract (extended)
12675
12676 // This pattern is automatically generated from aarch64_ad.m4
12677 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12678 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12679 %{
12680 match(Set dst (AddL src1 (ConvI2L src2)));
12681 ins_cost(INSN_COST);
12682 format %{ "add $dst, $src1, $src2, sxtw" %}
12683
12684 ins_encode %{
12685 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12686 as_Register($src2$$reg), ext::sxtw);
12687 %}
12688 ins_pipe(ialu_reg_reg);
12689 %}
12690
12691 // This pattern is automatically generated from aarch64_ad.m4
12692 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12693 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12694 %{
12695 match(Set dst (SubL src1 (ConvI2L src2)));
12696 ins_cost(INSN_COST);
12697 format %{ "sub $dst, $src1, $src2, sxtw" %}
12698
12699 ins_encode %{
12700 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12701 as_Register($src2$$reg), ext::sxtw);
12702 %}
12703 ins_pipe(ialu_reg_reg);
12704 %}
12705
12706 // This pattern is automatically generated from aarch64_ad.m4
12707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12708 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12709 %{
12710 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12711 ins_cost(INSN_COST);
12712 format %{ "add $dst, $src1, $src2, sxth" %}
12713
12714 ins_encode %{
12715 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12716 as_Register($src2$$reg), ext::sxth);
12717 %}
12718 ins_pipe(ialu_reg_reg);
12719 %}
12720
12721 // This pattern is automatically generated from aarch64_ad.m4
12722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12723 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12724 %{
12725 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12726 ins_cost(INSN_COST);
12727 format %{ "add $dst, $src1, $src2, sxtb" %}
12728
12729 ins_encode %{
12730 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12731 as_Register($src2$$reg), ext::sxtb);
12732 %}
12733 ins_pipe(ialu_reg_reg);
12734 %}
12735
12736 // This pattern is automatically generated from aarch64_ad.m4
12737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12738 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12739 %{
12740 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12741 ins_cost(INSN_COST);
12742 format %{ "add $dst, $src1, $src2, uxtb" %}
12743
12744 ins_encode %{
12745 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12746 as_Register($src2$$reg), ext::uxtb);
12747 %}
12748 ins_pipe(ialu_reg_reg);
12749 %}
12750
12751 // This pattern is automatically generated from aarch64_ad.m4
12752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12753 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12754 %{
12755 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12756 ins_cost(INSN_COST);
12757 format %{ "add $dst, $src1, $src2, sxth" %}
12758
12759 ins_encode %{
12760 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12761 as_Register($src2$$reg), ext::sxth);
12762 %}
12763 ins_pipe(ialu_reg_reg);
12764 %}
12765
12766 // This pattern is automatically generated from aarch64_ad.m4
12767 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12768 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12769 %{
12770 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12771 ins_cost(INSN_COST);
12772 format %{ "add $dst, $src1, $src2, sxtw" %}
12773
12774 ins_encode %{
12775 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12776 as_Register($src2$$reg), ext::sxtw);
12777 %}
12778 ins_pipe(ialu_reg_reg);
12779 %}
12780
12781 // This pattern is automatically generated from aarch64_ad.m4
12782 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12783 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12784 %{
12785 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12786 ins_cost(INSN_COST);
12787 format %{ "add $dst, $src1, $src2, sxtb" %}
12788
12789 ins_encode %{
12790 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12791 as_Register($src2$$reg), ext::sxtb);
12792 %}
12793 ins_pipe(ialu_reg_reg);
12794 %}
12795
12796 // This pattern is automatically generated from aarch64_ad.m4
12797 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12798 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12799 %{
12800 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12801 ins_cost(INSN_COST);
12802 format %{ "add $dst, $src1, $src2, uxtb" %}
12803
12804 ins_encode %{
12805 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12806 as_Register($src2$$reg), ext::uxtb);
12807 %}
12808 ins_pipe(ialu_reg_reg);
12809 %}
12810
12811 // This pattern is automatically generated from aarch64_ad.m4
12812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12813 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12814 %{
12815 match(Set dst (AddI src1 (AndI src2 mask)));
12816 ins_cost(INSN_COST);
12817 format %{ "addw $dst, $src1, $src2, uxtb" %}
12818
12819 ins_encode %{
12820 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12821 as_Register($src2$$reg), ext::uxtb);
12822 %}
12823 ins_pipe(ialu_reg_reg);
12824 %}
12825
12826 // This pattern is automatically generated from aarch64_ad.m4
12827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12828 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12829 %{
12830 match(Set dst (AddI src1 (AndI src2 mask)));
12831 ins_cost(INSN_COST);
12832 format %{ "addw $dst, $src1, $src2, uxth" %}
12833
12834 ins_encode %{
12835 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12836 as_Register($src2$$reg), ext::uxth);
12837 %}
12838 ins_pipe(ialu_reg_reg);
12839 %}
12840
12841 // This pattern is automatically generated from aarch64_ad.m4
12842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12843 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12844 %{
12845 match(Set dst (AddL src1 (AndL src2 mask)));
12846 ins_cost(INSN_COST);
12847 format %{ "add $dst, $src1, $src2, uxtb" %}
12848
12849 ins_encode %{
12850 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12851 as_Register($src2$$reg), ext::uxtb);
12852 %}
12853 ins_pipe(ialu_reg_reg);
12854 %}
12855
12856 // This pattern is automatically generated from aarch64_ad.m4
12857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12858 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12859 %{
12860 match(Set dst (AddL src1 (AndL src2 mask)));
12861 ins_cost(INSN_COST);
12862 format %{ "add $dst, $src1, $src2, uxth" %}
12863
12864 ins_encode %{
12865 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12866 as_Register($src2$$reg), ext::uxth);
12867 %}
12868 ins_pipe(ialu_reg_reg);
12869 %}
12870
12871 // This pattern is automatically generated from aarch64_ad.m4
12872 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12873 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12874 %{
12875 match(Set dst (AddL src1 (AndL src2 mask)));
12876 ins_cost(INSN_COST);
12877 format %{ "add $dst, $src1, $src2, uxtw" %}
12878
12879 ins_encode %{
12880 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12881 as_Register($src2$$reg), ext::uxtw);
12882 %}
12883 ins_pipe(ialu_reg_reg);
12884 %}
12885
12886 // This pattern is automatically generated from aarch64_ad.m4
12887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12888 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12889 %{
12890 match(Set dst (SubI src1 (AndI src2 mask)));
12891 ins_cost(INSN_COST);
12892 format %{ "subw $dst, $src1, $src2, uxtb" %}
12893
12894 ins_encode %{
12895 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12896 as_Register($src2$$reg), ext::uxtb);
12897 %}
12898 ins_pipe(ialu_reg_reg);
12899 %}
12900
12901 // This pattern is automatically generated from aarch64_ad.m4
12902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12903 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12904 %{
12905 match(Set dst (SubI src1 (AndI src2 mask)));
12906 ins_cost(INSN_COST);
12907 format %{ "subw $dst, $src1, $src2, uxth" %}
12908
12909 ins_encode %{
12910 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12911 as_Register($src2$$reg), ext::uxth);
12912 %}
12913 ins_pipe(ialu_reg_reg);
12914 %}
12915
12916 // This pattern is automatically generated from aarch64_ad.m4
12917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12918 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12919 %{
12920 match(Set dst (SubL src1 (AndL src2 mask)));
12921 ins_cost(INSN_COST);
12922 format %{ "sub $dst, $src1, $src2, uxtb" %}
12923
12924 ins_encode %{
12925 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12926 as_Register($src2$$reg), ext::uxtb);
12927 %}
12928 ins_pipe(ialu_reg_reg);
12929 %}
12930
12931 // This pattern is automatically generated from aarch64_ad.m4
12932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12933 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12934 %{
12935 match(Set dst (SubL src1 (AndL src2 mask)));
12936 ins_cost(INSN_COST);
12937 format %{ "sub $dst, $src1, $src2, uxth" %}
12938
12939 ins_encode %{
12940 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12941 as_Register($src2$$reg), ext::uxth);
12942 %}
12943 ins_pipe(ialu_reg_reg);
12944 %}
12945
12946 // This pattern is automatically generated from aarch64_ad.m4
12947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12948 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12949 %{
12950 match(Set dst (SubL src1 (AndL src2 mask)));
12951 ins_cost(INSN_COST);
12952 format %{ "sub $dst, $src1, $src2, uxtw" %}
12953
12954 ins_encode %{
12955 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12956 as_Register($src2$$reg), ext::uxtw);
12957 %}
12958 ins_pipe(ialu_reg_reg);
12959 %}
12960
12961
12962 // This pattern is automatically generated from aarch64_ad.m4
12963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12964 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12965 %{
12966 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12967 ins_cost(1.9 * INSN_COST);
12968 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12969
12970 ins_encode %{
12971 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12972 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12973 %}
12974 ins_pipe(ialu_reg_reg_shift);
12975 %}
12976
12977 // This pattern is automatically generated from aarch64_ad.m4
12978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12979 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12980 %{
12981 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12982 ins_cost(1.9 * INSN_COST);
12983 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12984
12985 ins_encode %{
12986 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12987 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12988 %}
12989 ins_pipe(ialu_reg_reg_shift);
12990 %}
12991
12992 // This pattern is automatically generated from aarch64_ad.m4
12993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12994 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12995 %{
12996 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12997 ins_cost(1.9 * INSN_COST);
12998 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12999
13000 ins_encode %{
13001 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13002 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13003 %}
13004 ins_pipe(ialu_reg_reg_shift);
13005 %}
13006
13007 // This pattern is automatically generated from aarch64_ad.m4
13008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13009 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13010 %{
13011 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13012 ins_cost(1.9 * INSN_COST);
13013 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13014
13015 ins_encode %{
13016 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13017 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13018 %}
13019 ins_pipe(ialu_reg_reg_shift);
13020 %}
13021
13022 // This pattern is automatically generated from aarch64_ad.m4
13023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13024 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13025 %{
13026 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13027 ins_cost(1.9 * INSN_COST);
13028 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13029
13030 ins_encode %{
13031 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13032 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13033 %}
13034 ins_pipe(ialu_reg_reg_shift);
13035 %}
13036
13037 // This pattern is automatically generated from aarch64_ad.m4
13038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13039 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13040 %{
13041 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13042 ins_cost(1.9 * INSN_COST);
13043 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13044
13045 ins_encode %{
13046 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13047 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13048 %}
13049 ins_pipe(ialu_reg_reg_shift);
13050 %}
13051
13052 // This pattern is automatically generated from aarch64_ad.m4
13053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13054 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13055 %{
13056 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13057 ins_cost(1.9 * INSN_COST);
13058 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13059
13060 ins_encode %{
13061 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13062 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13063 %}
13064 ins_pipe(ialu_reg_reg_shift);
13065 %}
13066
13067 // This pattern is automatically generated from aarch64_ad.m4
13068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13069 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13070 %{
13071 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13072 ins_cost(1.9 * INSN_COST);
13073 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13074
13075 ins_encode %{
13076 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13077 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13078 %}
13079 ins_pipe(ialu_reg_reg_shift);
13080 %}
13081
13082 // This pattern is automatically generated from aarch64_ad.m4
13083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13084 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13085 %{
13086 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13087 ins_cost(1.9 * INSN_COST);
13088 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13089
13090 ins_encode %{
13091 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13092 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13093 %}
13094 ins_pipe(ialu_reg_reg_shift);
13095 %}
13096
13097 // This pattern is automatically generated from aarch64_ad.m4
13098 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13099 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13100 %{
13101 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13102 ins_cost(1.9 * INSN_COST);
13103 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13104
13105 ins_encode %{
13106 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13107 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13108 %}
13109 ins_pipe(ialu_reg_reg_shift);
13110 %}
13111
13112 // This pattern is automatically generated from aarch64_ad.m4
13113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13114 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13115 %{
13116 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13117 ins_cost(1.9 * INSN_COST);
13118 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13119
13120 ins_encode %{
13121 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13122 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13123 %}
13124 ins_pipe(ialu_reg_reg_shift);
13125 %}
13126
13127 // This pattern is automatically generated from aarch64_ad.m4
13128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13129 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13130 %{
13131 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13132 ins_cost(1.9 * INSN_COST);
13133 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13134
13135 ins_encode %{
13136 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13137 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13138 %}
13139 ins_pipe(ialu_reg_reg_shift);
13140 %}
13141
13142 // This pattern is automatically generated from aarch64_ad.m4
13143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13144 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13145 %{
13146 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13147 ins_cost(1.9 * INSN_COST);
13148 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13149
13150 ins_encode %{
13151 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13152 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13153 %}
13154 ins_pipe(ialu_reg_reg_shift);
13155 %}
13156
13157 // This pattern is automatically generated from aarch64_ad.m4
13158 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13159 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13160 %{
13161 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13162 ins_cost(1.9 * INSN_COST);
13163 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13164
13165 ins_encode %{
13166 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13167 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13168 %}
13169 ins_pipe(ialu_reg_reg_shift);
13170 %}
13171
13172 // This pattern is automatically generated from aarch64_ad.m4
13173 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13174 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13175 %{
13176 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13177 ins_cost(1.9 * INSN_COST);
13178 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13179
13180 ins_encode %{
13181 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13182 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13183 %}
13184 ins_pipe(ialu_reg_reg_shift);
13185 %}
13186
13187 // This pattern is automatically generated from aarch64_ad.m4
13188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13189 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13190 %{
13191 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13192 ins_cost(1.9 * INSN_COST);
13193 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13194
13195 ins_encode %{
13196 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13197 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13198 %}
13199 ins_pipe(ialu_reg_reg_shift);
13200 %}
13201
13202 // This pattern is automatically generated from aarch64_ad.m4
13203 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13204 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13205 %{
13206 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13207 ins_cost(1.9 * INSN_COST);
13208 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13209
13210 ins_encode %{
13211 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13212 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13213 %}
13214 ins_pipe(ialu_reg_reg_shift);
13215 %}
13216
13217 // This pattern is automatically generated from aarch64_ad.m4
13218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13219 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13220 %{
13221 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13222 ins_cost(1.9 * INSN_COST);
13223 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13224
13225 ins_encode %{
13226 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13227 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13228 %}
13229 ins_pipe(ialu_reg_reg_shift);
13230 %}
13231
13232 // This pattern is automatically generated from aarch64_ad.m4
13233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13234 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13235 %{
13236 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13237 ins_cost(1.9 * INSN_COST);
13238 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13239
13240 ins_encode %{
13241 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13242 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13243 %}
13244 ins_pipe(ialu_reg_reg_shift);
13245 %}
13246
13247 // This pattern is automatically generated from aarch64_ad.m4
13248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13249 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13250 %{
13251 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13252 ins_cost(1.9 * INSN_COST);
13253 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13254
13255 ins_encode %{
13256 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13257 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13258 %}
13259 ins_pipe(ialu_reg_reg_shift);
13260 %}
13261
13262 // This pattern is automatically generated from aarch64_ad.m4
13263 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13264 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13265 %{
13266 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13267 ins_cost(1.9 * INSN_COST);
13268 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13269
13270 ins_encode %{
13271 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13272 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13273 %}
13274 ins_pipe(ialu_reg_reg_shift);
13275 %}
13276
13277 // This pattern is automatically generated from aarch64_ad.m4
13278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13279 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13280 %{
13281 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13282 ins_cost(1.9 * INSN_COST);
13283 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13284
13285 ins_encode %{
13286 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13287 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13288 %}
13289 ins_pipe(ialu_reg_reg_shift);
13290 %}
13291
13292 // This pattern is automatically generated from aarch64_ad.m4
13293 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13294 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13295 %{
13296 effect(DEF dst, USE src1, USE src2, USE cr);
13297 ins_cost(INSN_COST * 2);
13298 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13299
13300 ins_encode %{
13301 __ cselw($dst$$Register,
13302 $src1$$Register,
13303 $src2$$Register,
13304 Assembler::LT);
13305 %}
13306 ins_pipe(icond_reg_reg);
13307 %}
13308
13309 // This pattern is automatically generated from aarch64_ad.m4
13310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13311 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13312 %{
13313 effect(DEF dst, USE src1, USE src2, USE cr);
13314 ins_cost(INSN_COST * 2);
13315 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13316
13317 ins_encode %{
13318 __ cselw($dst$$Register,
13319 $src1$$Register,
13320 $src2$$Register,
13321 Assembler::GT);
13322 %}
13323 ins_pipe(icond_reg_reg);
13324 %}
13325
13326 // This pattern is automatically generated from aarch64_ad.m4
13327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13328 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13329 %{
13330 effect(DEF dst, USE src1, USE cr);
13331 ins_cost(INSN_COST * 2);
13332 format %{ "cselw $dst, $src1, zr lt\t" %}
13333
13334 ins_encode %{
13335 __ cselw($dst$$Register,
13336 $src1$$Register,
13337 zr,
13338 Assembler::LT);
13339 %}
13340 ins_pipe(icond_reg);
13341 %}
13342
13343 // This pattern is automatically generated from aarch64_ad.m4
13344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13345 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13346 %{
13347 effect(DEF dst, USE src1, USE cr);
13348 ins_cost(INSN_COST * 2);
13349 format %{ "cselw $dst, $src1, zr gt\t" %}
13350
13351 ins_encode %{
13352 __ cselw($dst$$Register,
13353 $src1$$Register,
13354 zr,
13355 Assembler::GT);
13356 %}
13357 ins_pipe(icond_reg);
13358 %}
13359
13360 // This pattern is automatically generated from aarch64_ad.m4
13361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13362 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13363 %{
13364 effect(DEF dst, USE src1, USE cr);
13365 ins_cost(INSN_COST * 2);
13366 format %{ "csincw $dst, $src1, zr le\t" %}
13367
13368 ins_encode %{
13369 __ csincw($dst$$Register,
13370 $src1$$Register,
13371 zr,
13372 Assembler::LE);
13373 %}
13374 ins_pipe(icond_reg);
13375 %}
13376
13377 // This pattern is automatically generated from aarch64_ad.m4
13378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13379 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13380 %{
13381 effect(DEF dst, USE src1, USE cr);
13382 ins_cost(INSN_COST * 2);
13383 format %{ "csincw $dst, $src1, zr gt\t" %}
13384
13385 ins_encode %{
13386 __ csincw($dst$$Register,
13387 $src1$$Register,
13388 zr,
13389 Assembler::GT);
13390 %}
13391 ins_pipe(icond_reg);
13392 %}
13393
13394 // This pattern is automatically generated from aarch64_ad.m4
13395 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13396 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13397 %{
13398 effect(DEF dst, USE src1, USE cr);
13399 ins_cost(INSN_COST * 2);
13400 format %{ "csinvw $dst, $src1, zr lt\t" %}
13401
13402 ins_encode %{
13403 __ csinvw($dst$$Register,
13404 $src1$$Register,
13405 zr,
13406 Assembler::LT);
13407 %}
13408 ins_pipe(icond_reg);
13409 %}
13410
13411 // This pattern is automatically generated from aarch64_ad.m4
13412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13413 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13414 %{
13415 effect(DEF dst, USE src1, USE cr);
13416 ins_cost(INSN_COST * 2);
13417 format %{ "csinvw $dst, $src1, zr ge\t" %}
13418
13419 ins_encode %{
13420 __ csinvw($dst$$Register,
13421 $src1$$Register,
13422 zr,
13423 Assembler::GE);
13424 %}
13425 ins_pipe(icond_reg);
13426 %}
13427
13428 // This pattern is automatically generated from aarch64_ad.m4
13429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13430 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13431 %{
13432 match(Set dst (MinI src imm));
13433 ins_cost(INSN_COST * 3);
13434 expand %{
13435 rFlagsReg cr;
13436 compI_reg_imm0(cr, src);
13437 cmovI_reg_imm0_lt(dst, src, cr);
13438 %}
13439 %}
13440
13441 // This pattern is automatically generated from aarch64_ad.m4
13442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13443 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13444 %{
13445 match(Set dst (MinI imm src));
13446 ins_cost(INSN_COST * 3);
13447 expand %{
13448 rFlagsReg cr;
13449 compI_reg_imm0(cr, src);
13450 cmovI_reg_imm0_lt(dst, src, cr);
13451 %}
13452 %}
13453
13454 // This pattern is automatically generated from aarch64_ad.m4
13455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13456 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13457 %{
13458 match(Set dst (MinI src imm));
13459 ins_cost(INSN_COST * 3);
13460 expand %{
13461 rFlagsReg cr;
13462 compI_reg_imm0(cr, src);
13463 cmovI_reg_imm1_le(dst, src, cr);
13464 %}
13465 %}
13466
13467 // This pattern is automatically generated from aarch64_ad.m4
13468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13469 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13470 %{
13471 match(Set dst (MinI imm src));
13472 ins_cost(INSN_COST * 3);
13473 expand %{
13474 rFlagsReg cr;
13475 compI_reg_imm0(cr, src);
13476 cmovI_reg_imm1_le(dst, src, cr);
13477 %}
13478 %}
13479
13480 // This pattern is automatically generated from aarch64_ad.m4
13481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13482 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13483 %{
13484 match(Set dst (MinI src imm));
13485 ins_cost(INSN_COST * 3);
13486 expand %{
13487 rFlagsReg cr;
13488 compI_reg_imm0(cr, src);
13489 cmovI_reg_immM1_lt(dst, src, cr);
13490 %}
13491 %}
13492
13493 // This pattern is automatically generated from aarch64_ad.m4
13494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13495 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13496 %{
13497 match(Set dst (MinI imm src));
13498 ins_cost(INSN_COST * 3);
13499 expand %{
13500 rFlagsReg cr;
13501 compI_reg_imm0(cr, src);
13502 cmovI_reg_immM1_lt(dst, src, cr);
13503 %}
13504 %}
13505
13506 // This pattern is automatically generated from aarch64_ad.m4
13507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13508 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13509 %{
13510 match(Set dst (MaxI src imm));
13511 ins_cost(INSN_COST * 3);
13512 expand %{
13513 rFlagsReg cr;
13514 compI_reg_imm0(cr, src);
13515 cmovI_reg_imm0_gt(dst, src, cr);
13516 %}
13517 %}
13518
13519 // This pattern is automatically generated from aarch64_ad.m4
13520 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13521 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13522 %{
13523 match(Set dst (MaxI imm src));
13524 ins_cost(INSN_COST * 3);
13525 expand %{
13526 rFlagsReg cr;
13527 compI_reg_imm0(cr, src);
13528 cmovI_reg_imm0_gt(dst, src, cr);
13529 %}
13530 %}
13531
13532 // This pattern is automatically generated from aarch64_ad.m4
13533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13534 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13535 %{
13536 match(Set dst (MaxI src imm));
13537 ins_cost(INSN_COST * 3);
13538 expand %{
13539 rFlagsReg cr;
13540 compI_reg_imm0(cr, src);
13541 cmovI_reg_imm1_gt(dst, src, cr);
13542 %}
13543 %}
13544
13545 // This pattern is automatically generated from aarch64_ad.m4
13546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13547 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13548 %{
13549 match(Set dst (MaxI imm src));
13550 ins_cost(INSN_COST * 3);
13551 expand %{
13552 rFlagsReg cr;
13553 compI_reg_imm0(cr, src);
13554 cmovI_reg_imm1_gt(dst, src, cr);
13555 %}
13556 %}
13557
13558 // This pattern is automatically generated from aarch64_ad.m4
13559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13560 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13561 %{
13562 match(Set dst (MaxI src imm));
13563 ins_cost(INSN_COST * 3);
13564 expand %{
13565 rFlagsReg cr;
13566 compI_reg_imm0(cr, src);
13567 cmovI_reg_immM1_ge(dst, src, cr);
13568 %}
13569 %}
13570
13571 // This pattern is automatically generated from aarch64_ad.m4
13572 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13573 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13574 %{
13575 match(Set dst (MaxI imm src));
13576 ins_cost(INSN_COST * 3);
13577 expand %{
13578 rFlagsReg cr;
13579 compI_reg_imm0(cr, src);
13580 cmovI_reg_immM1_ge(dst, src, cr);
13581 %}
13582 %}
13583
13584 // This pattern is automatically generated from aarch64_ad.m4
13585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13586 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13587 %{
13588 match(Set dst (ReverseI src));
13589 ins_cost(INSN_COST);
13590 format %{ "rbitw $dst, $src" %}
13591 ins_encode %{
13592 __ rbitw($dst$$Register, $src$$Register);
13593 %}
13594 ins_pipe(ialu_reg);
13595 %}
13596
13597 // This pattern is automatically generated from aarch64_ad.m4
13598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13599 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13600 %{
13601 match(Set dst (ReverseL src));
13602 ins_cost(INSN_COST);
13603 format %{ "rbit $dst, $src" %}
13604 ins_encode %{
13605 __ rbit($dst$$Register, $src$$Register);
13606 %}
13607 ins_pipe(ialu_reg);
13608 %}
13609
13610
13611 // END This section of the file is automatically generated. Do not edit --------------
13612
13613
13614 // ============================================================================
13615 // Floating Point Arithmetic Instructions
13616
13617 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13618 match(Set dst (AddF src1 src2));
13619
13620 ins_cost(INSN_COST * 5);
13621 format %{ "fadds $dst, $src1, $src2" %}
13622
13623 ins_encode %{
13624 __ fadds(as_FloatRegister($dst$$reg),
13625 as_FloatRegister($src1$$reg),
13626 as_FloatRegister($src2$$reg));
13627 %}
13628
13629 ins_pipe(fp_dop_reg_reg_s);
13630 %}
13631
13632 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13633 match(Set dst (AddD src1 src2));
13634
13635 ins_cost(INSN_COST * 5);
13636 format %{ "faddd $dst, $src1, $src2" %}
13637
13638 ins_encode %{
13639 __ faddd(as_FloatRegister($dst$$reg),
13640 as_FloatRegister($src1$$reg),
13641 as_FloatRegister($src2$$reg));
13642 %}
13643
13644 ins_pipe(fp_dop_reg_reg_d);
13645 %}
13646
13647 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13648 match(Set dst (SubF src1 src2));
13649
13650 ins_cost(INSN_COST * 5);
13651 format %{ "fsubs $dst, $src1, $src2" %}
13652
13653 ins_encode %{
13654 __ fsubs(as_FloatRegister($dst$$reg),
13655 as_FloatRegister($src1$$reg),
13656 as_FloatRegister($src2$$reg));
13657 %}
13658
13659 ins_pipe(fp_dop_reg_reg_s);
13660 %}
13661
13662 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13663 match(Set dst (SubD src1 src2));
13664
13665 ins_cost(INSN_COST * 5);
13666 format %{ "fsubd $dst, $src1, $src2" %}
13667
13668 ins_encode %{
13669 __ fsubd(as_FloatRegister($dst$$reg),
13670 as_FloatRegister($src1$$reg),
13671 as_FloatRegister($src2$$reg));
13672 %}
13673
13674 ins_pipe(fp_dop_reg_reg_d);
13675 %}
13676
13677 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13678 match(Set dst (MulF src1 src2));
13679
13680 ins_cost(INSN_COST * 6);
13681 format %{ "fmuls $dst, $src1, $src2" %}
13682
13683 ins_encode %{
13684 __ fmuls(as_FloatRegister($dst$$reg),
13685 as_FloatRegister($src1$$reg),
13686 as_FloatRegister($src2$$reg));
13687 %}
13688
13689 ins_pipe(fp_dop_reg_reg_s);
13690 %}
13691
13692 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13693 match(Set dst (MulD src1 src2));
13694
13695 ins_cost(INSN_COST * 6);
13696 format %{ "fmuld $dst, $src1, $src2" %}
13697
13698 ins_encode %{
13699 __ fmuld(as_FloatRegister($dst$$reg),
13700 as_FloatRegister($src1$$reg),
13701 as_FloatRegister($src2$$reg));
13702 %}
13703
13704 ins_pipe(fp_dop_reg_reg_d);
13705 %}
13706
13707 // src1 * src2 + src3
13708 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13709 match(Set dst (FmaF src3 (Binary src1 src2)));
13710
13711 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13712
13713 ins_encode %{
13714 assert(UseFMA, "Needs FMA instructions support.");
13715 __ fmadds(as_FloatRegister($dst$$reg),
13716 as_FloatRegister($src1$$reg),
13717 as_FloatRegister($src2$$reg),
13718 as_FloatRegister($src3$$reg));
13719 %}
13720
13721 ins_pipe(pipe_class_default);
13722 %}
13723
13724 // src1 * src2 + src3
13725 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13726 match(Set dst (FmaD src3 (Binary src1 src2)));
13727
13728 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13729
13730 ins_encode %{
13731 assert(UseFMA, "Needs FMA instructions support.");
13732 __ fmaddd(as_FloatRegister($dst$$reg),
13733 as_FloatRegister($src1$$reg),
13734 as_FloatRegister($src2$$reg),
13735 as_FloatRegister($src3$$reg));
13736 %}
13737
13738 ins_pipe(pipe_class_default);
13739 %}
13740
13741 // src1 * (-src2) + src3
13742 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13743 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13744 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13745
13746 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13747
13748 ins_encode %{
13749 assert(UseFMA, "Needs FMA instructions support.");
13750 __ fmsubs(as_FloatRegister($dst$$reg),
13751 as_FloatRegister($src1$$reg),
13752 as_FloatRegister($src2$$reg),
13753 as_FloatRegister($src3$$reg));
13754 %}
13755
13756 ins_pipe(pipe_class_default);
13757 %}
13758
13759 // src1 * (-src2) + src3
13760 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13761 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13762 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13763
13764 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13765
13766 ins_encode %{
13767 assert(UseFMA, "Needs FMA instructions support.");
13768 __ fmsubd(as_FloatRegister($dst$$reg),
13769 as_FloatRegister($src1$$reg),
13770 as_FloatRegister($src2$$reg),
13771 as_FloatRegister($src3$$reg));
13772 %}
13773
13774 ins_pipe(pipe_class_default);
13775 %}
13776
13777 // src1 * (-src2) - src3
13778 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13779 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13780 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13781
13782 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13783
13784 ins_encode %{
13785 assert(UseFMA, "Needs FMA instructions support.");
13786 __ fnmadds(as_FloatRegister($dst$$reg),
13787 as_FloatRegister($src1$$reg),
13788 as_FloatRegister($src2$$reg),
13789 as_FloatRegister($src3$$reg));
13790 %}
13791
13792 ins_pipe(pipe_class_default);
13793 %}
13794
13795 // src1 * (-src2) - src3
13796 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13797 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13798 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13799
13800 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13801
13802 ins_encode %{
13803 assert(UseFMA, "Needs FMA instructions support.");
13804 __ fnmaddd(as_FloatRegister($dst$$reg),
13805 as_FloatRegister($src1$$reg),
13806 as_FloatRegister($src2$$reg),
13807 as_FloatRegister($src3$$reg));
13808 %}
13809
13810 ins_pipe(pipe_class_default);
13811 %}
13812
13813 // src1 * src2 - src3
13814 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13815 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13816
13817 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13818
13819 ins_encode %{
13820 assert(UseFMA, "Needs FMA instructions support.");
13821 __ fnmsubs(as_FloatRegister($dst$$reg),
13822 as_FloatRegister($src1$$reg),
13823 as_FloatRegister($src2$$reg),
13824 as_FloatRegister($src3$$reg));
13825 %}
13826
13827 ins_pipe(pipe_class_default);
13828 %}
13829
13830 // src1 * src2 - src3
13831 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13832 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13833
13834 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13835
13836 ins_encode %{
13837 assert(UseFMA, "Needs FMA instructions support.");
13838 // n.b. insn name should be fnmsubd
13839 __ fnmsub(as_FloatRegister($dst$$reg),
13840 as_FloatRegister($src1$$reg),
13841 as_FloatRegister($src2$$reg),
13842 as_FloatRegister($src3$$reg));
13843 %}
13844
13845 ins_pipe(pipe_class_default);
13846 %}
13847
13848
13849 // Math.max(FF)F
13850 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13851 match(Set dst (MaxF src1 src2));
13852
13853 format %{ "fmaxs $dst, $src1, $src2" %}
13854 ins_encode %{
13855 __ fmaxs(as_FloatRegister($dst$$reg),
13856 as_FloatRegister($src1$$reg),
13857 as_FloatRegister($src2$$reg));
13858 %}
13859
13860 ins_pipe(fp_dop_reg_reg_s);
13861 %}
13862
13863 // Math.min(FF)F
13864 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13865 match(Set dst (MinF src1 src2));
13866
13867 format %{ "fmins $dst, $src1, $src2" %}
13868 ins_encode %{
13869 __ fmins(as_FloatRegister($dst$$reg),
13870 as_FloatRegister($src1$$reg),
13871 as_FloatRegister($src2$$reg));
13872 %}
13873
13874 ins_pipe(fp_dop_reg_reg_s);
13875 %}
13876
13877 // Math.max(DD)D
13878 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13879 match(Set dst (MaxD src1 src2));
13880
13881 format %{ "fmaxd $dst, $src1, $src2" %}
13882 ins_encode %{
13883 __ fmaxd(as_FloatRegister($dst$$reg),
13884 as_FloatRegister($src1$$reg),
13885 as_FloatRegister($src2$$reg));
13886 %}
13887
13888 ins_pipe(fp_dop_reg_reg_d);
13889 %}
13890
13891 // Math.min(DD)D
13892 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13893 match(Set dst (MinD src1 src2));
13894
13895 format %{ "fmind $dst, $src1, $src2" %}
13896 ins_encode %{
13897 __ fmind(as_FloatRegister($dst$$reg),
13898 as_FloatRegister($src1$$reg),
13899 as_FloatRegister($src2$$reg));
13900 %}
13901
13902 ins_pipe(fp_dop_reg_reg_d);
13903 %}
13904
13905
13906 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13907 match(Set dst (DivF src1 src2));
13908
13909 ins_cost(INSN_COST * 18);
13910 format %{ "fdivs $dst, $src1, $src2" %}
13911
13912 ins_encode %{
13913 __ fdivs(as_FloatRegister($dst$$reg),
13914 as_FloatRegister($src1$$reg),
13915 as_FloatRegister($src2$$reg));
13916 %}
13917
13918 ins_pipe(fp_div_s);
13919 %}
13920
13921 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13922 match(Set dst (DivD src1 src2));
13923
13924 ins_cost(INSN_COST * 32);
13925 format %{ "fdivd $dst, $src1, $src2" %}
13926
13927 ins_encode %{
13928 __ fdivd(as_FloatRegister($dst$$reg),
13929 as_FloatRegister($src1$$reg),
13930 as_FloatRegister($src2$$reg));
13931 %}
13932
13933 ins_pipe(fp_div_d);
13934 %}
13935
13936 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13937 match(Set dst (NegF src));
13938
13939 ins_cost(INSN_COST * 3);
13940 format %{ "fneg $dst, $src" %}
13941
13942 ins_encode %{
13943 __ fnegs(as_FloatRegister($dst$$reg),
13944 as_FloatRegister($src$$reg));
13945 %}
13946
13947 ins_pipe(fp_uop_s);
13948 %}
13949
13950 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13951 match(Set dst (NegD src));
13952
13953 ins_cost(INSN_COST * 3);
13954 format %{ "fnegd $dst, $src" %}
13955
13956 ins_encode %{
13957 __ fnegd(as_FloatRegister($dst$$reg),
13958 as_FloatRegister($src$$reg));
13959 %}
13960
13961 ins_pipe(fp_uop_d);
13962 %}
13963
13964 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13965 %{
13966 match(Set dst (AbsI src));
13967
13968 effect(KILL cr);
13969 ins_cost(INSN_COST * 2);
13970 format %{ "cmpw $src, zr\n\t"
13971 "cnegw $dst, $src, Assembler::LT\t# int abs"
13972 %}
13973
13974 ins_encode %{
13975 __ cmpw(as_Register($src$$reg), zr);
13976 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13977 %}
13978 ins_pipe(pipe_class_default);
13979 %}
13980
13981 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13982 %{
13983 match(Set dst (AbsL src));
13984
13985 effect(KILL cr);
13986 ins_cost(INSN_COST * 2);
13987 format %{ "cmp $src, zr\n\t"
13988 "cneg $dst, $src, Assembler::LT\t# long abs"
13989 %}
13990
13991 ins_encode %{
13992 __ cmp(as_Register($src$$reg), zr);
13993 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13994 %}
13995 ins_pipe(pipe_class_default);
13996 %}
13997
13998 instruct absF_reg(vRegF dst, vRegF src) %{
13999 match(Set dst (AbsF src));
14000
14001 ins_cost(INSN_COST * 3);
14002 format %{ "fabss $dst, $src" %}
14003 ins_encode %{
14004 __ fabss(as_FloatRegister($dst$$reg),
14005 as_FloatRegister($src$$reg));
14006 %}
14007
14008 ins_pipe(fp_uop_s);
14009 %}
14010
14011 instruct absD_reg(vRegD dst, vRegD src) %{
14012 match(Set dst (AbsD src));
14013
14014 ins_cost(INSN_COST * 3);
14015 format %{ "fabsd $dst, $src" %}
14016 ins_encode %{
14017 __ fabsd(as_FloatRegister($dst$$reg),
14018 as_FloatRegister($src$$reg));
14019 %}
14020
14021 ins_pipe(fp_uop_d);
14022 %}
14023
14024 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14025 match(Set dst (AbsF (SubF src1 src2)));
14026
14027 ins_cost(INSN_COST * 3);
14028 format %{ "fabds $dst, $src1, $src2" %}
14029 ins_encode %{
14030 __ fabds(as_FloatRegister($dst$$reg),
14031 as_FloatRegister($src1$$reg),
14032 as_FloatRegister($src2$$reg));
14033 %}
14034
14035 ins_pipe(fp_uop_s);
14036 %}
14037
14038 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14039 match(Set dst (AbsD (SubD src1 src2)));
14040
14041 ins_cost(INSN_COST * 3);
14042 format %{ "fabdd $dst, $src1, $src2" %}
14043 ins_encode %{
14044 __ fabdd(as_FloatRegister($dst$$reg),
14045 as_FloatRegister($src1$$reg),
14046 as_FloatRegister($src2$$reg));
14047 %}
14048
14049 ins_pipe(fp_uop_d);
14050 %}
14051
14052 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14053 match(Set dst (SqrtD src));
14054
14055 ins_cost(INSN_COST * 50);
14056 format %{ "fsqrtd $dst, $src" %}
14057 ins_encode %{
14058 __ fsqrtd(as_FloatRegister($dst$$reg),
14059 as_FloatRegister($src$$reg));
14060 %}
14061
14062 ins_pipe(fp_div_s);
14063 %}
14064
14065 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14066 match(Set dst (SqrtF src));
14067
14068 ins_cost(INSN_COST * 50);
14069 format %{ "fsqrts $dst, $src" %}
14070 ins_encode %{
14071 __ fsqrts(as_FloatRegister($dst$$reg),
14072 as_FloatRegister($src$$reg));
14073 %}
14074
14075 ins_pipe(fp_div_d);
14076 %}
14077
14078 // Math.rint, floor, ceil
14079 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14080 match(Set dst (RoundDoubleMode src rmode));
14081 format %{ "frint $dst, $src, $rmode" %}
14082 ins_encode %{
14083 switch ($rmode$$constant) {
14084 case RoundDoubleModeNode::rmode_rint:
14085 __ frintnd(as_FloatRegister($dst$$reg),
14086 as_FloatRegister($src$$reg));
14087 break;
14088 case RoundDoubleModeNode::rmode_floor:
14089 __ frintmd(as_FloatRegister($dst$$reg),
14090 as_FloatRegister($src$$reg));
14091 break;
14092 case RoundDoubleModeNode::rmode_ceil:
14093 __ frintpd(as_FloatRegister($dst$$reg),
14094 as_FloatRegister($src$$reg));
14095 break;
14096 }
14097 %}
14098 ins_pipe(fp_uop_d);
14099 %}
14100
14101 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14102 match(Set dst (CopySignD src1 (Binary src2 zero)));
14103 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14104 format %{ "CopySignD $dst $src1 $src2" %}
14105 ins_encode %{
14106 FloatRegister dst = as_FloatRegister($dst$$reg),
14107 src1 = as_FloatRegister($src1$$reg),
14108 src2 = as_FloatRegister($src2$$reg),
14109 zero = as_FloatRegister($zero$$reg);
14110 __ fnegd(dst, zero);
14111 __ bsl(dst, __ T8B, src2, src1);
14112 %}
14113 ins_pipe(fp_uop_d);
14114 %}
14115
14116 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14117 match(Set dst (CopySignF src1 src2));
14118 effect(TEMP_DEF dst, USE src1, USE src2);
14119 format %{ "CopySignF $dst $src1 $src2" %}
14120 ins_encode %{
14121 FloatRegister dst = as_FloatRegister($dst$$reg),
14122 src1 = as_FloatRegister($src1$$reg),
14123 src2 = as_FloatRegister($src2$$reg);
14124 __ movi(dst, __ T2S, 0x80, 24);
14125 __ bsl(dst, __ T8B, src2, src1);
14126 %}
14127 ins_pipe(fp_uop_d);
14128 %}
14129
14130 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14131 match(Set dst (SignumD src (Binary zero one)));
14132 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14133 format %{ "signumD $dst, $src" %}
14134 ins_encode %{
14135 FloatRegister src = as_FloatRegister($src$$reg),
14136 dst = as_FloatRegister($dst$$reg),
14137 zero = as_FloatRegister($zero$$reg),
14138 one = as_FloatRegister($one$$reg);
14139 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14140 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14141 // Bit selection instruction gets bit from "one" for each enabled bit in
14142 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14143 // NaN the whole "src" will be copied because "dst" is zero. For all other
14144 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14145 // from "src", and all other bits are copied from 1.0.
14146 __ bsl(dst, __ T8B, one, src);
14147 %}
14148 ins_pipe(fp_uop_d);
14149 %}
14150
14151 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14152 match(Set dst (SignumF src (Binary zero one)));
14153 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14154 format %{ "signumF $dst, $src" %}
14155 ins_encode %{
14156 FloatRegister src = as_FloatRegister($src$$reg),
14157 dst = as_FloatRegister($dst$$reg),
14158 zero = as_FloatRegister($zero$$reg),
14159 one = as_FloatRegister($one$$reg);
14160 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14161 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14162 // Bit selection instruction gets bit from "one" for each enabled bit in
14163 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14164 // NaN the whole "src" will be copied because "dst" is zero. For all other
14165 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14166 // from "src", and all other bits are copied from 1.0.
14167 __ bsl(dst, __ T8B, one, src);
14168 %}
14169 ins_pipe(fp_uop_d);
14170 %}
14171
14172 instruct onspinwait() %{
14173 match(OnSpinWait);
14174 ins_cost(INSN_COST);
14175
14176 format %{ "onspinwait" %}
14177
14178 ins_encode %{
14179 __ spin_wait();
14180 %}
14181 ins_pipe(pipe_class_empty);
14182 %}
14183
14184 // ============================================================================
14185 // Logical Instructions
14186
14187 // Integer Logical Instructions
14188
14189 // And Instructions
14190
14191
14192 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14193 match(Set dst (AndI src1 src2));
14194
14195 format %{ "andw $dst, $src1, $src2\t# int" %}
14196
14197 ins_cost(INSN_COST);
14198 ins_encode %{
14199 __ andw(as_Register($dst$$reg),
14200 as_Register($src1$$reg),
14201 as_Register($src2$$reg));
14202 %}
14203
14204 ins_pipe(ialu_reg_reg);
14205 %}
14206
14207 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14208 match(Set dst (AndI src1 src2));
14209
14210 format %{ "andsw $dst, $src1, $src2\t# int" %}
14211
14212 ins_cost(INSN_COST);
14213 ins_encode %{
14214 __ andw(as_Register($dst$$reg),
14215 as_Register($src1$$reg),
14216 (uint64_t)($src2$$constant));
14217 %}
14218
14219 ins_pipe(ialu_reg_imm);
14220 %}
14221
14222 // Or Instructions
14223
14224 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14225 match(Set dst (OrI src1 src2));
14226
14227 format %{ "orrw $dst, $src1, $src2\t# int" %}
14228
14229 ins_cost(INSN_COST);
14230 ins_encode %{
14231 __ orrw(as_Register($dst$$reg),
14232 as_Register($src1$$reg),
14233 as_Register($src2$$reg));
14234 %}
14235
14236 ins_pipe(ialu_reg_reg);
14237 %}
14238
14239 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14240 match(Set dst (OrI src1 src2));
14241
14242 format %{ "orrw $dst, $src1, $src2\t# int" %}
14243
14244 ins_cost(INSN_COST);
14245 ins_encode %{
14246 __ orrw(as_Register($dst$$reg),
14247 as_Register($src1$$reg),
14248 (uint64_t)($src2$$constant));
14249 %}
14250
14251 ins_pipe(ialu_reg_imm);
14252 %}
14253
14254 // Xor Instructions
14255
14256 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14257 match(Set dst (XorI src1 src2));
14258
14259 format %{ "eorw $dst, $src1, $src2\t# int" %}
14260
14261 ins_cost(INSN_COST);
14262 ins_encode %{
14263 __ eorw(as_Register($dst$$reg),
14264 as_Register($src1$$reg),
14265 as_Register($src2$$reg));
14266 %}
14267
14268 ins_pipe(ialu_reg_reg);
14269 %}
14270
14271 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14272 match(Set dst (XorI src1 src2));
14273
14274 format %{ "eorw $dst, $src1, $src2\t# int" %}
14275
14276 ins_cost(INSN_COST);
14277 ins_encode %{
14278 __ eorw(as_Register($dst$$reg),
14279 as_Register($src1$$reg),
14280 (uint64_t)($src2$$constant));
14281 %}
14282
14283 ins_pipe(ialu_reg_imm);
14284 %}
14285
14286 // Long Logical Instructions
14287 // TODO
14288
14289 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14290 match(Set dst (AndL src1 src2));
14291
14292 format %{ "and $dst, $src1, $src2\t# int" %}
14293
14294 ins_cost(INSN_COST);
14295 ins_encode %{
14296 __ andr(as_Register($dst$$reg),
14297 as_Register($src1$$reg),
14298 as_Register($src2$$reg));
14299 %}
14300
14301 ins_pipe(ialu_reg_reg);
14302 %}
14303
14304 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14305 match(Set dst (AndL src1 src2));
14306
14307 format %{ "and $dst, $src1, $src2\t# int" %}
14308
14309 ins_cost(INSN_COST);
14310 ins_encode %{
14311 __ andr(as_Register($dst$$reg),
14312 as_Register($src1$$reg),
14313 (uint64_t)($src2$$constant));
14314 %}
14315
14316 ins_pipe(ialu_reg_imm);
14317 %}
14318
14319 // Or Instructions
14320
14321 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14322 match(Set dst (OrL src1 src2));
14323
14324 format %{ "orr $dst, $src1, $src2\t# int" %}
14325
14326 ins_cost(INSN_COST);
14327 ins_encode %{
14328 __ orr(as_Register($dst$$reg),
14329 as_Register($src1$$reg),
14330 as_Register($src2$$reg));
14331 %}
14332
14333 ins_pipe(ialu_reg_reg);
14334 %}
14335
14336 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14337 match(Set dst (OrL src1 src2));
14338
14339 format %{ "orr $dst, $src1, $src2\t# int" %}
14340
14341 ins_cost(INSN_COST);
14342 ins_encode %{
14343 __ orr(as_Register($dst$$reg),
14344 as_Register($src1$$reg),
14345 (uint64_t)($src2$$constant));
14346 %}
14347
14348 ins_pipe(ialu_reg_imm);
14349 %}
14350
14351 // Xor Instructions
14352
14353 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14354 match(Set dst (XorL src1 src2));
14355
14356 format %{ "eor $dst, $src1, $src2\t# int" %}
14357
14358 ins_cost(INSN_COST);
14359 ins_encode %{
14360 __ eor(as_Register($dst$$reg),
14361 as_Register($src1$$reg),
14362 as_Register($src2$$reg));
14363 %}
14364
14365 ins_pipe(ialu_reg_reg);
14366 %}
14367
14368 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14369 match(Set dst (XorL src1 src2));
14370
14371 ins_cost(INSN_COST);
14372 format %{ "eor $dst, $src1, $src2\t# int" %}
14373
14374 ins_encode %{
14375 __ eor(as_Register($dst$$reg),
14376 as_Register($src1$$reg),
14377 (uint64_t)($src2$$constant));
14378 %}
14379
14380 ins_pipe(ialu_reg_imm);
14381 %}
14382
14383 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14384 %{
14385 match(Set dst (ConvI2L src));
14386
14387 ins_cost(INSN_COST);
14388 format %{ "sxtw $dst, $src\t# i2l" %}
14389 ins_encode %{
14390 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14391 %}
14392 ins_pipe(ialu_reg_shift);
14393 %}
14394
14395 // this pattern occurs in bigmath arithmetic
14396 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14397 %{
14398 match(Set dst (AndL (ConvI2L src) mask));
14399
14400 ins_cost(INSN_COST);
14401 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14402 ins_encode %{
14403 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14404 %}
14405
14406 ins_pipe(ialu_reg_shift);
14407 %}
14408
14409 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14410 match(Set dst (ConvL2I src));
14411
14412 ins_cost(INSN_COST);
14413 format %{ "movw $dst, $src \t// l2i" %}
14414
14415 ins_encode %{
14416 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14417 %}
14418
14419 ins_pipe(ialu_reg);
14420 %}
14421
14422 instruct convD2F_reg(vRegF dst, vRegD src) %{
14423 match(Set dst (ConvD2F src));
14424
14425 ins_cost(INSN_COST * 5);
14426 format %{ "fcvtd $dst, $src \t// d2f" %}
14427
14428 ins_encode %{
14429 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14430 %}
14431
14432 ins_pipe(fp_d2f);
14433 %}
14434
14435 instruct convF2D_reg(vRegD dst, vRegF src) %{
14436 match(Set dst (ConvF2D src));
14437
14438 ins_cost(INSN_COST * 5);
14439 format %{ "fcvts $dst, $src \t// f2d" %}
14440
14441 ins_encode %{
14442 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14443 %}
14444
14445 ins_pipe(fp_f2d);
14446 %}
14447
14448 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14449 match(Set dst (ConvF2I src));
14450
14451 ins_cost(INSN_COST * 5);
14452 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14453
14454 ins_encode %{
14455 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14456 %}
14457
14458 ins_pipe(fp_f2i);
14459 %}
14460
14461 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14462 match(Set dst (ConvF2L src));
14463
14464 ins_cost(INSN_COST * 5);
14465 format %{ "fcvtzs $dst, $src \t// f2l" %}
14466
14467 ins_encode %{
14468 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14469 %}
14470
14471 ins_pipe(fp_f2l);
14472 %}
14473
14474 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14475 match(Set dst (ConvF2HF src));
14476 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14477 "smov $dst, $tmp\t# move result from $tmp to $dst"
14478 %}
14479 effect(TEMP tmp);
14480 ins_encode %{
14481 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14482 %}
14483 ins_pipe(pipe_slow);
14484 %}
14485
14486 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14487 match(Set dst (ConvHF2F src));
14488 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14489 "fcvt $dst, $tmp\t# convert half to single precision"
14490 %}
14491 effect(TEMP tmp);
14492 ins_encode %{
14493 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14494 %}
14495 ins_pipe(pipe_slow);
14496 %}
14497
14498 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14499 match(Set dst (ConvI2F src));
14500
14501 ins_cost(INSN_COST * 5);
14502 format %{ "scvtfws $dst, $src \t// i2f" %}
14503
14504 ins_encode %{
14505 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14506 %}
14507
14508 ins_pipe(fp_i2f);
14509 %}
14510
14511 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14512 match(Set dst (ConvL2F src));
14513
14514 ins_cost(INSN_COST * 5);
14515 format %{ "scvtfs $dst, $src \t// l2f" %}
14516
14517 ins_encode %{
14518 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14519 %}
14520
14521 ins_pipe(fp_l2f);
14522 %}
14523
14524 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14525 match(Set dst (ConvD2I src));
14526
14527 ins_cost(INSN_COST * 5);
14528 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14529
14530 ins_encode %{
14531 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14532 %}
14533
14534 ins_pipe(fp_d2i);
14535 %}
14536
14537 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14538 match(Set dst (ConvD2L src));
14539
14540 ins_cost(INSN_COST * 5);
14541 format %{ "fcvtzd $dst, $src \t// d2l" %}
14542
14543 ins_encode %{
14544 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14545 %}
14546
14547 ins_pipe(fp_d2l);
14548 %}
14549
14550 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14551 match(Set dst (ConvI2D src));
14552
14553 ins_cost(INSN_COST * 5);
14554 format %{ "scvtfwd $dst, $src \t// i2d" %}
14555
14556 ins_encode %{
14557 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14558 %}
14559
14560 ins_pipe(fp_i2d);
14561 %}
14562
14563 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14564 match(Set dst (ConvL2D src));
14565
14566 ins_cost(INSN_COST * 5);
14567 format %{ "scvtfd $dst, $src \t// l2d" %}
14568
14569 ins_encode %{
14570 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14571 %}
14572
14573 ins_pipe(fp_l2d);
14574 %}
14575
14576 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14577 %{
14578 match(Set dst (RoundD src));
14579 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14580 format %{ "java_round_double $dst,$src"%}
14581 ins_encode %{
14582 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14583 as_FloatRegister($ftmp$$reg));
14584 %}
14585 ins_pipe(pipe_slow);
14586 %}
14587
14588 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14589 %{
14590 match(Set dst (RoundF src));
14591 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14592 format %{ "java_round_float $dst,$src"%}
14593 ins_encode %{
14594 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14595 as_FloatRegister($ftmp$$reg));
14596 %}
14597 ins_pipe(pipe_slow);
14598 %}
14599
14600 // stack <-> reg and reg <-> reg shuffles with no conversion
14601
14602 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14603
14604 match(Set dst (MoveF2I src));
14605
14606 effect(DEF dst, USE src);
14607
14608 ins_cost(4 * INSN_COST);
14609
14610 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14611
14612 ins_encode %{
14613 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14614 %}
14615
14616 ins_pipe(iload_reg_reg);
14617
14618 %}
14619
14620 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14621
14622 match(Set dst (MoveI2F src));
14623
14624 effect(DEF dst, USE src);
14625
14626 ins_cost(4 * INSN_COST);
14627
14628 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14629
14630 ins_encode %{
14631 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14632 %}
14633
14634 ins_pipe(pipe_class_memory);
14635
14636 %}
14637
14638 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14639
14640 match(Set dst (MoveD2L src));
14641
14642 effect(DEF dst, USE src);
14643
14644 ins_cost(4 * INSN_COST);
14645
14646 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14647
14648 ins_encode %{
14649 __ ldr($dst$$Register, Address(sp, $src$$disp));
14650 %}
14651
14652 ins_pipe(iload_reg_reg);
14653
14654 %}
14655
14656 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14657
14658 match(Set dst (MoveL2D src));
14659
14660 effect(DEF dst, USE src);
14661
14662 ins_cost(4 * INSN_COST);
14663
14664 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14665
14666 ins_encode %{
14667 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14668 %}
14669
14670 ins_pipe(pipe_class_memory);
14671
14672 %}
14673
14674 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14675
14676 match(Set dst (MoveF2I src));
14677
14678 effect(DEF dst, USE src);
14679
14680 ins_cost(INSN_COST);
14681
14682 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14683
14684 ins_encode %{
14685 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14686 %}
14687
14688 ins_pipe(pipe_class_memory);
14689
14690 %}
14691
14692 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14693
14694 match(Set dst (MoveI2F src));
14695
14696 effect(DEF dst, USE src);
14697
14698 ins_cost(INSN_COST);
14699
14700 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14701
14702 ins_encode %{
14703 __ strw($src$$Register, Address(sp, $dst$$disp));
14704 %}
14705
14706 ins_pipe(istore_reg_reg);
14707
14708 %}
14709
14710 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14711
14712 match(Set dst (MoveD2L src));
14713
14714 effect(DEF dst, USE src);
14715
14716 ins_cost(INSN_COST);
14717
14718 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14719
14720 ins_encode %{
14721 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14722 %}
14723
14724 ins_pipe(pipe_class_memory);
14725
14726 %}
14727
14728 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14729
14730 match(Set dst (MoveL2D src));
14731
14732 effect(DEF dst, USE src);
14733
14734 ins_cost(INSN_COST);
14735
14736 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14737
14738 ins_encode %{
14739 __ str($src$$Register, Address(sp, $dst$$disp));
14740 %}
14741
14742 ins_pipe(istore_reg_reg);
14743
14744 %}
14745
14746 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14747
14748 match(Set dst (MoveF2I src));
14749
14750 effect(DEF dst, USE src);
14751
14752 ins_cost(INSN_COST);
14753
14754 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14755
14756 ins_encode %{
14757 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14758 %}
14759
14760 ins_pipe(fp_f2i);
14761
14762 %}
14763
14764 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14765
14766 match(Set dst (MoveI2F src));
14767
14768 effect(DEF dst, USE src);
14769
14770 ins_cost(INSN_COST);
14771
14772 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14773
14774 ins_encode %{
14775 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14776 %}
14777
14778 ins_pipe(fp_i2f);
14779
14780 %}
14781
14782 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14783
14784 match(Set dst (MoveD2L src));
14785
14786 effect(DEF dst, USE src);
14787
14788 ins_cost(INSN_COST);
14789
14790 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14791
14792 ins_encode %{
14793 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14794 %}
14795
14796 ins_pipe(fp_d2l);
14797
14798 %}
14799
14800 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14801
14802 match(Set dst (MoveL2D src));
14803
14804 effect(DEF dst, USE src);
14805
14806 ins_cost(INSN_COST);
14807
14808 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14809
14810 ins_encode %{
14811 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14812 %}
14813
14814 ins_pipe(fp_l2d);
14815
14816 %}
14817
14818 // ============================================================================
14819 // clearing of an array
14820
14821 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14822 %{
14823 match(Set dummy (ClearArray cnt base));
14824 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14825
14826 ins_cost(4 * INSN_COST);
14827 format %{ "ClearArray $cnt, $base" %}
14828
14829 ins_encode %{
14830 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14831 if (tpc == nullptr) {
14832 ciEnv::current()->record_failure("CodeCache is full");
14833 return;
14834 }
14835 %}
14836
14837 ins_pipe(pipe_class_memory);
14838 %}
14839
14840 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14841 %{
14842 predicate((uint64_t)n->in(2)->get_long()
14843 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14844 match(Set dummy (ClearArray cnt base));
14845 effect(TEMP temp, USE_KILL base, KILL cr);
14846
14847 ins_cost(4 * INSN_COST);
14848 format %{ "ClearArray $cnt, $base" %}
14849
14850 ins_encode %{
14851 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14852 if (tpc == nullptr) {
14853 ciEnv::current()->record_failure("CodeCache is full");
14854 return;
14855 }
14856 %}
14857
14858 ins_pipe(pipe_class_memory);
14859 %}
14860
14861 // ============================================================================
14862 // Overflow Math Instructions
14863
14864 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14865 %{
14866 match(Set cr (OverflowAddI op1 op2));
14867
14868 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14869 ins_cost(INSN_COST);
14870 ins_encode %{
14871 __ cmnw($op1$$Register, $op2$$Register);
14872 %}
14873
14874 ins_pipe(icmp_reg_reg);
14875 %}
14876
14877 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14878 %{
14879 match(Set cr (OverflowAddI op1 op2));
14880
14881 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14882 ins_cost(INSN_COST);
14883 ins_encode %{
14884 __ cmnw($op1$$Register, $op2$$constant);
14885 %}
14886
14887 ins_pipe(icmp_reg_imm);
14888 %}
14889
14890 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14891 %{
14892 match(Set cr (OverflowAddL op1 op2));
14893
14894 format %{ "cmn $op1, $op2\t# overflow check long" %}
14895 ins_cost(INSN_COST);
14896 ins_encode %{
14897 __ cmn($op1$$Register, $op2$$Register);
14898 %}
14899
14900 ins_pipe(icmp_reg_reg);
14901 %}
14902
14903 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14904 %{
14905 match(Set cr (OverflowAddL op1 op2));
14906
14907 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14908 ins_cost(INSN_COST);
14909 ins_encode %{
14910 __ adds(zr, $op1$$Register, $op2$$constant);
14911 %}
14912
14913 ins_pipe(icmp_reg_imm);
14914 %}
14915
14916 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14917 %{
14918 match(Set cr (OverflowSubI op1 op2));
14919
14920 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14921 ins_cost(INSN_COST);
14922 ins_encode %{
14923 __ cmpw($op1$$Register, $op2$$Register);
14924 %}
14925
14926 ins_pipe(icmp_reg_reg);
14927 %}
14928
14929 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14930 %{
14931 match(Set cr (OverflowSubI op1 op2));
14932
14933 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14934 ins_cost(INSN_COST);
14935 ins_encode %{
14936 __ cmpw($op1$$Register, $op2$$constant);
14937 %}
14938
14939 ins_pipe(icmp_reg_imm);
14940 %}
14941
14942 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14943 %{
14944 match(Set cr (OverflowSubL op1 op2));
14945
14946 format %{ "cmp $op1, $op2\t# overflow check long" %}
14947 ins_cost(INSN_COST);
14948 ins_encode %{
14949 __ cmp($op1$$Register, $op2$$Register);
14950 %}
14951
14952 ins_pipe(icmp_reg_reg);
14953 %}
14954
14955 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14956 %{
14957 match(Set cr (OverflowSubL op1 op2));
14958
14959 format %{ "cmp $op1, $op2\t# overflow check long" %}
14960 ins_cost(INSN_COST);
14961 ins_encode %{
14962 __ subs(zr, $op1$$Register, $op2$$constant);
14963 %}
14964
14965 ins_pipe(icmp_reg_imm);
14966 %}
14967
14968 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14969 %{
14970 match(Set cr (OverflowSubI zero op1));
14971
14972 format %{ "cmpw zr, $op1\t# overflow check int" %}
14973 ins_cost(INSN_COST);
14974 ins_encode %{
14975 __ cmpw(zr, $op1$$Register);
14976 %}
14977
14978 ins_pipe(icmp_reg_imm);
14979 %}
14980
14981 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14982 %{
14983 match(Set cr (OverflowSubL zero op1));
14984
14985 format %{ "cmp zr, $op1\t# overflow check long" %}
14986 ins_cost(INSN_COST);
14987 ins_encode %{
14988 __ cmp(zr, $op1$$Register);
14989 %}
14990
14991 ins_pipe(icmp_reg_imm);
14992 %}
14993
14994 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14995 %{
14996 match(Set cr (OverflowMulI op1 op2));
14997
14998 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14999 "cmp rscratch1, rscratch1, sxtw\n\t"
15000 "movw rscratch1, #0x80000000\n\t"
15001 "cselw rscratch1, rscratch1, zr, NE\n\t"
15002 "cmpw rscratch1, #1" %}
15003 ins_cost(5 * INSN_COST);
15004 ins_encode %{
15005 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15006 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15007 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15008 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15009 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15010 %}
15011
15012 ins_pipe(pipe_slow);
15013 %}
15014
15015 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15016 %{
15017 match(If cmp (OverflowMulI op1 op2));
15018 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15019 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15020 effect(USE labl, KILL cr);
15021
15022 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15023 "cmp rscratch1, rscratch1, sxtw\n\t"
15024 "b$cmp $labl" %}
15025 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15026 ins_encode %{
15027 Label* L = $labl$$label;
15028 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15029 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15030 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15031 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15032 %}
15033
15034 ins_pipe(pipe_serial);
15035 %}
15036
15037 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15038 %{
15039 match(Set cr (OverflowMulL op1 op2));
15040
15041 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15042 "smulh rscratch2, $op1, $op2\n\t"
15043 "cmp rscratch2, rscratch1, ASR #63\n\t"
15044 "movw rscratch1, #0x80000000\n\t"
15045 "cselw rscratch1, rscratch1, zr, NE\n\t"
15046 "cmpw rscratch1, #1" %}
15047 ins_cost(6 * INSN_COST);
15048 ins_encode %{
15049 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15050 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15051 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15052 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15053 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15054 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15055 %}
15056
15057 ins_pipe(pipe_slow);
15058 %}
15059
15060 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15061 %{
15062 match(If cmp (OverflowMulL op1 op2));
15063 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15064 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15065 effect(USE labl, KILL cr);
15066
15067 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15068 "smulh rscratch2, $op1, $op2\n\t"
15069 "cmp rscratch2, rscratch1, ASR #63\n\t"
15070 "b$cmp $labl" %}
15071 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15072 ins_encode %{
15073 Label* L = $labl$$label;
15074 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15075 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15076 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15077 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15078 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15079 %}
15080
15081 ins_pipe(pipe_serial);
15082 %}
15083
15084 // ============================================================================
15085 // Compare Instructions
15086
15087 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15088 %{
15089 match(Set cr (CmpI op1 op2));
15090
15091 effect(DEF cr, USE op1, USE op2);
15092
15093 ins_cost(INSN_COST);
15094 format %{ "cmpw $op1, $op2" %}
15095
15096 ins_encode(aarch64_enc_cmpw(op1, op2));
15097
15098 ins_pipe(icmp_reg_reg);
15099 %}
15100
15101 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15102 %{
15103 match(Set cr (CmpI op1 zero));
15104
15105 effect(DEF cr, USE op1);
15106
15107 ins_cost(INSN_COST);
15108 format %{ "cmpw $op1, 0" %}
15109
15110 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15111
15112 ins_pipe(icmp_reg_imm);
15113 %}
15114
15115 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15116 %{
15117 match(Set cr (CmpI op1 op2));
15118
15119 effect(DEF cr, USE op1);
15120
15121 ins_cost(INSN_COST);
15122 format %{ "cmpw $op1, $op2" %}
15123
15124 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15125
15126 ins_pipe(icmp_reg_imm);
15127 %}
15128
15129 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15130 %{
15131 match(Set cr (CmpI op1 op2));
15132
15133 effect(DEF cr, USE op1);
15134
15135 ins_cost(INSN_COST * 2);
15136 format %{ "cmpw $op1, $op2" %}
15137
15138 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15139
15140 ins_pipe(icmp_reg_imm);
15141 %}
15142
15143 // Unsigned compare Instructions; really, same as signed compare
15144 // except it should only be used to feed an If or a CMovI which takes a
15145 // cmpOpU.
15146
15147 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15148 %{
15149 match(Set cr (CmpU op1 op2));
15150
15151 effect(DEF cr, USE op1, USE op2);
15152
15153 ins_cost(INSN_COST);
15154 format %{ "cmpw $op1, $op2\t# unsigned" %}
15155
15156 ins_encode(aarch64_enc_cmpw(op1, op2));
15157
15158 ins_pipe(icmp_reg_reg);
15159 %}
15160
15161 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15162 %{
15163 match(Set cr (CmpU op1 zero));
15164
15165 effect(DEF cr, USE op1);
15166
15167 ins_cost(INSN_COST);
15168 format %{ "cmpw $op1, #0\t# unsigned" %}
15169
15170 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15171
15172 ins_pipe(icmp_reg_imm);
15173 %}
15174
15175 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15176 %{
15177 match(Set cr (CmpU op1 op2));
15178
15179 effect(DEF cr, USE op1);
15180
15181 ins_cost(INSN_COST);
15182 format %{ "cmpw $op1, $op2\t# unsigned" %}
15183
15184 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15185
15186 ins_pipe(icmp_reg_imm);
15187 %}
15188
15189 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15190 %{
15191 match(Set cr (CmpU op1 op2));
15192
15193 effect(DEF cr, USE op1);
15194
15195 ins_cost(INSN_COST * 2);
15196 format %{ "cmpw $op1, $op2\t# unsigned" %}
15197
15198 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15199
15200 ins_pipe(icmp_reg_imm);
15201 %}
15202
15203 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15204 %{
15205 match(Set cr (CmpL op1 op2));
15206
15207 effect(DEF cr, USE op1, USE op2);
15208
15209 ins_cost(INSN_COST);
15210 format %{ "cmp $op1, $op2" %}
15211
15212 ins_encode(aarch64_enc_cmp(op1, op2));
15213
15214 ins_pipe(icmp_reg_reg);
15215 %}
15216
15217 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15218 %{
15219 match(Set cr (CmpL op1 zero));
15220
15221 effect(DEF cr, USE op1);
15222
15223 ins_cost(INSN_COST);
15224 format %{ "tst $op1" %}
15225
15226 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15227
15228 ins_pipe(icmp_reg_imm);
15229 %}
15230
15231 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15232 %{
15233 match(Set cr (CmpL op1 op2));
15234
15235 effect(DEF cr, USE op1);
15236
15237 ins_cost(INSN_COST);
15238 format %{ "cmp $op1, $op2" %}
15239
15240 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15241
15242 ins_pipe(icmp_reg_imm);
15243 %}
15244
15245 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15246 %{
15247 match(Set cr (CmpL op1 op2));
15248
15249 effect(DEF cr, USE op1);
15250
15251 ins_cost(INSN_COST * 2);
15252 format %{ "cmp $op1, $op2" %}
15253
15254 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15255
15256 ins_pipe(icmp_reg_imm);
15257 %}
15258
15259 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15260 %{
15261 match(Set cr (CmpUL op1 op2));
15262
15263 effect(DEF cr, USE op1, USE op2);
15264
15265 ins_cost(INSN_COST);
15266 format %{ "cmp $op1, $op2" %}
15267
15268 ins_encode(aarch64_enc_cmp(op1, op2));
15269
15270 ins_pipe(icmp_reg_reg);
15271 %}
15272
15273 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15274 %{
15275 match(Set cr (CmpUL op1 zero));
15276
15277 effect(DEF cr, USE op1);
15278
15279 ins_cost(INSN_COST);
15280 format %{ "tst $op1" %}
15281
15282 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15283
15284 ins_pipe(icmp_reg_imm);
15285 %}
15286
15287 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15288 %{
15289 match(Set cr (CmpUL op1 op2));
15290
15291 effect(DEF cr, USE op1);
15292
15293 ins_cost(INSN_COST);
15294 format %{ "cmp $op1, $op2" %}
15295
15296 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15297
15298 ins_pipe(icmp_reg_imm);
15299 %}
15300
15301 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15302 %{
15303 match(Set cr (CmpUL op1 op2));
15304
15305 effect(DEF cr, USE op1);
15306
15307 ins_cost(INSN_COST * 2);
15308 format %{ "cmp $op1, $op2" %}
15309
15310 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15311
15312 ins_pipe(icmp_reg_imm);
15313 %}
15314
15315 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15316 %{
15317 match(Set cr (CmpP op1 op2));
15318
15319 effect(DEF cr, USE op1, USE op2);
15320
15321 ins_cost(INSN_COST);
15322 format %{ "cmp $op1, $op2\t // ptr" %}
15323
15324 ins_encode(aarch64_enc_cmpp(op1, op2));
15325
15326 ins_pipe(icmp_reg_reg);
15327 %}
15328
15329 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15330 %{
15331 match(Set cr (CmpN op1 op2));
15332
15333 effect(DEF cr, USE op1, USE op2);
15334
15335 ins_cost(INSN_COST);
15336 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15337
15338 ins_encode(aarch64_enc_cmpn(op1, op2));
15339
15340 ins_pipe(icmp_reg_reg);
15341 %}
15342
15343 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15344 %{
15345 match(Set cr (CmpP op1 zero));
15346
15347 effect(DEF cr, USE op1, USE zero);
15348
15349 ins_cost(INSN_COST);
15350 format %{ "cmp $op1, 0\t // ptr" %}
15351
15352 ins_encode(aarch64_enc_testp(op1));
15353
15354 ins_pipe(icmp_reg_imm);
15355 %}
15356
15357 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15358 %{
15359 match(Set cr (CmpN op1 zero));
15360
15361 effect(DEF cr, USE op1, USE zero);
15362
15363 ins_cost(INSN_COST);
15364 format %{ "cmp $op1, 0\t // compressed ptr" %}
15365
15366 ins_encode(aarch64_enc_testn(op1));
15367
15368 ins_pipe(icmp_reg_imm);
15369 %}
15370
15371 // FP comparisons
15372 //
15373 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15374 // using normal cmpOp. See declaration of rFlagsReg for details.
15375
15376 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15377 %{
15378 match(Set cr (CmpF src1 src2));
15379
15380 ins_cost(3 * INSN_COST);
15381 format %{ "fcmps $src1, $src2" %}
15382
15383 ins_encode %{
15384 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15385 %}
15386
15387 ins_pipe(pipe_class_compare);
15388 %}
15389
15390 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15391 %{
15392 match(Set cr (CmpF src1 src2));
15393
15394 ins_cost(3 * INSN_COST);
15395 format %{ "fcmps $src1, 0.0" %}
15396
15397 ins_encode %{
15398 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15399 %}
15400
15401 ins_pipe(pipe_class_compare);
15402 %}
15403 // FROM HERE
15404
15405 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15406 %{
15407 match(Set cr (CmpD src1 src2));
15408
15409 ins_cost(3 * INSN_COST);
15410 format %{ "fcmpd $src1, $src2" %}
15411
15412 ins_encode %{
15413 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15414 %}
15415
15416 ins_pipe(pipe_class_compare);
15417 %}
15418
15419 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15420 %{
15421 match(Set cr (CmpD src1 src2));
15422
15423 ins_cost(3 * INSN_COST);
15424 format %{ "fcmpd $src1, 0.0" %}
15425
15426 ins_encode %{
15427 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15428 %}
15429
15430 ins_pipe(pipe_class_compare);
15431 %}
15432
15433 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15434 %{
15435 match(Set dst (CmpF3 src1 src2));
15436 effect(KILL cr);
15437
15438 ins_cost(5 * INSN_COST);
15439 format %{ "fcmps $src1, $src2\n\t"
15440 "csinvw($dst, zr, zr, eq\n\t"
15441 "csnegw($dst, $dst, $dst, lt)"
15442 %}
15443
15444 ins_encode %{
15445 Label done;
15446 FloatRegister s1 = as_FloatRegister($src1$$reg);
15447 FloatRegister s2 = as_FloatRegister($src2$$reg);
15448 Register d = as_Register($dst$$reg);
15449 __ fcmps(s1, s2);
15450 // installs 0 if EQ else -1
15451 __ csinvw(d, zr, zr, Assembler::EQ);
15452 // keeps -1 if less or unordered else installs 1
15453 __ csnegw(d, d, d, Assembler::LT);
15454 __ bind(done);
15455 %}
15456
15457 ins_pipe(pipe_class_default);
15458
15459 %}
15460
15461 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15462 %{
15463 match(Set dst (CmpD3 src1 src2));
15464 effect(KILL cr);
15465
15466 ins_cost(5 * INSN_COST);
15467 format %{ "fcmpd $src1, $src2\n\t"
15468 "csinvw($dst, zr, zr, eq\n\t"
15469 "csnegw($dst, $dst, $dst, lt)"
15470 %}
15471
15472 ins_encode %{
15473 Label done;
15474 FloatRegister s1 = as_FloatRegister($src1$$reg);
15475 FloatRegister s2 = as_FloatRegister($src2$$reg);
15476 Register d = as_Register($dst$$reg);
15477 __ fcmpd(s1, s2);
15478 // installs 0 if EQ else -1
15479 __ csinvw(d, zr, zr, Assembler::EQ);
15480 // keeps -1 if less or unordered else installs 1
15481 __ csnegw(d, d, d, Assembler::LT);
15482 __ bind(done);
15483 %}
15484 ins_pipe(pipe_class_default);
15485
15486 %}
15487
15488 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15489 %{
15490 match(Set dst (CmpF3 src1 zero));
15491 effect(KILL cr);
15492
15493 ins_cost(5 * INSN_COST);
15494 format %{ "fcmps $src1, 0.0\n\t"
15495 "csinvw($dst, zr, zr, eq\n\t"
15496 "csnegw($dst, $dst, $dst, lt)"
15497 %}
15498
15499 ins_encode %{
15500 Label done;
15501 FloatRegister s1 = as_FloatRegister($src1$$reg);
15502 Register d = as_Register($dst$$reg);
15503 __ fcmps(s1, 0.0);
15504 // installs 0 if EQ else -1
15505 __ csinvw(d, zr, zr, Assembler::EQ);
15506 // keeps -1 if less or unordered else installs 1
15507 __ csnegw(d, d, d, Assembler::LT);
15508 __ bind(done);
15509 %}
15510
15511 ins_pipe(pipe_class_default);
15512
15513 %}
15514
15515 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15516 %{
15517 match(Set dst (CmpD3 src1 zero));
15518 effect(KILL cr);
15519
15520 ins_cost(5 * INSN_COST);
15521 format %{ "fcmpd $src1, 0.0\n\t"
15522 "csinvw($dst, zr, zr, eq\n\t"
15523 "csnegw($dst, $dst, $dst, lt)"
15524 %}
15525
15526 ins_encode %{
15527 Label done;
15528 FloatRegister s1 = as_FloatRegister($src1$$reg);
15529 Register d = as_Register($dst$$reg);
15530 __ fcmpd(s1, 0.0);
15531 // installs 0 if EQ else -1
15532 __ csinvw(d, zr, zr, Assembler::EQ);
15533 // keeps -1 if less or unordered else installs 1
15534 __ csnegw(d, d, d, Assembler::LT);
15535 __ bind(done);
15536 %}
15537 ins_pipe(pipe_class_default);
15538
15539 %}
15540
15541 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15542 %{
15543 match(Set dst (CmpLTMask p q));
15544 effect(KILL cr);
15545
15546 ins_cost(3 * INSN_COST);
15547
15548 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15549 "csetw $dst, lt\n\t"
15550 "subw $dst, zr, $dst"
15551 %}
15552
15553 ins_encode %{
15554 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15555 __ csetw(as_Register($dst$$reg), Assembler::LT);
15556 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15557 %}
15558
15559 ins_pipe(ialu_reg_reg);
15560 %}
15561
15562 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15563 %{
15564 match(Set dst (CmpLTMask src zero));
15565 effect(KILL cr);
15566
15567 ins_cost(INSN_COST);
15568
15569 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15570
15571 ins_encode %{
15572 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15573 %}
15574
15575 ins_pipe(ialu_reg_shift);
15576 %}
15577
15578 // ============================================================================
15579 // Max and Min
15580
15581 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15582
15583 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15584 %{
15585 effect(DEF cr, USE src);
15586 ins_cost(INSN_COST);
15587 format %{ "cmpw $src, 0" %}
15588
15589 ins_encode %{
15590 __ cmpw($src$$Register, 0);
15591 %}
15592 ins_pipe(icmp_reg_imm);
15593 %}
15594
15595 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15596 %{
15597 match(Set dst (MinI src1 src2));
15598 ins_cost(INSN_COST * 3);
15599
15600 expand %{
15601 rFlagsReg cr;
15602 compI_reg_reg(cr, src1, src2);
15603 cmovI_reg_reg_lt(dst, src1, src2, cr);
15604 %}
15605 %}
15606
15607 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15608 %{
15609 match(Set dst (MaxI src1 src2));
15610 ins_cost(INSN_COST * 3);
15611
15612 expand %{
15613 rFlagsReg cr;
15614 compI_reg_reg(cr, src1, src2);
15615 cmovI_reg_reg_gt(dst, src1, src2, cr);
15616 %}
15617 %}
15618
15619
15620 // ============================================================================
15621 // Branch Instructions
15622
15623 // Direct Branch.
15624 instruct branch(label lbl)
15625 %{
15626 match(Goto);
15627
15628 effect(USE lbl);
15629
15630 ins_cost(BRANCH_COST);
15631 format %{ "b $lbl" %}
15632
15633 ins_encode(aarch64_enc_b(lbl));
15634
15635 ins_pipe(pipe_branch);
15636 %}
15637
15638 // Conditional Near Branch
15639 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15640 %{
15641 // Same match rule as `branchConFar'.
15642 match(If cmp cr);
15643
15644 effect(USE lbl);
15645
15646 ins_cost(BRANCH_COST);
15647 // If set to 1 this indicates that the current instruction is a
15648 // short variant of a long branch. This avoids using this
15649 // instruction in first-pass matching. It will then only be used in
15650 // the `Shorten_branches' pass.
15651 // ins_short_branch(1);
15652 format %{ "b$cmp $lbl" %}
15653
15654 ins_encode(aarch64_enc_br_con(cmp, lbl));
15655
15656 ins_pipe(pipe_branch_cond);
15657 %}
15658
15659 // Conditional Near Branch Unsigned
15660 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15661 %{
15662 // Same match rule as `branchConFar'.
15663 match(If cmp cr);
15664
15665 effect(USE lbl);
15666
15667 ins_cost(BRANCH_COST);
15668 // If set to 1 this indicates that the current instruction is a
15669 // short variant of a long branch. This avoids using this
15670 // instruction in first-pass matching. It will then only be used in
15671 // the `Shorten_branches' pass.
15672 // ins_short_branch(1);
15673 format %{ "b$cmp $lbl\t# unsigned" %}
15674
15675 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15676
15677 ins_pipe(pipe_branch_cond);
15678 %}
15679
15680 // Make use of CBZ and CBNZ. These instructions, as well as being
15681 // shorter than (cmp; branch), have the additional benefit of not
15682 // killing the flags.
15683
15684 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15685 match(If cmp (CmpI op1 op2));
15686 effect(USE labl);
15687
15688 ins_cost(BRANCH_COST);
15689 format %{ "cbw$cmp $op1, $labl" %}
15690 ins_encode %{
15691 Label* L = $labl$$label;
15692 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15693 if (cond == Assembler::EQ)
15694 __ cbzw($op1$$Register, *L);
15695 else
15696 __ cbnzw($op1$$Register, *L);
15697 %}
15698 ins_pipe(pipe_cmp_branch);
15699 %}
15700
15701 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15702 match(If cmp (CmpL op1 op2));
15703 effect(USE labl);
15704
15705 ins_cost(BRANCH_COST);
15706 format %{ "cb$cmp $op1, $labl" %}
15707 ins_encode %{
15708 Label* L = $labl$$label;
15709 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15710 if (cond == Assembler::EQ)
15711 __ cbz($op1$$Register, *L);
15712 else
15713 __ cbnz($op1$$Register, *L);
15714 %}
15715 ins_pipe(pipe_cmp_branch);
15716 %}
15717
15718 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15719 match(If cmp (CmpP op1 op2));
15720 effect(USE labl);
15721
15722 ins_cost(BRANCH_COST);
15723 format %{ "cb$cmp $op1, $labl" %}
15724 ins_encode %{
15725 Label* L = $labl$$label;
15726 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15727 if (cond == Assembler::EQ)
15728 __ cbz($op1$$Register, *L);
15729 else
15730 __ cbnz($op1$$Register, *L);
15731 %}
15732 ins_pipe(pipe_cmp_branch);
15733 %}
15734
15735 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15736 match(If cmp (CmpN op1 op2));
15737 effect(USE labl);
15738
15739 ins_cost(BRANCH_COST);
15740 format %{ "cbw$cmp $op1, $labl" %}
15741 ins_encode %{
15742 Label* L = $labl$$label;
15743 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15744 if (cond == Assembler::EQ)
15745 __ cbzw($op1$$Register, *L);
15746 else
15747 __ cbnzw($op1$$Register, *L);
15748 %}
15749 ins_pipe(pipe_cmp_branch);
15750 %}
15751
15752 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15753 match(If cmp (CmpP (DecodeN oop) zero));
15754 effect(USE labl);
15755
15756 ins_cost(BRANCH_COST);
15757 format %{ "cb$cmp $oop, $labl" %}
15758 ins_encode %{
15759 Label* L = $labl$$label;
15760 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15761 if (cond == Assembler::EQ)
15762 __ cbzw($oop$$Register, *L);
15763 else
15764 __ cbnzw($oop$$Register, *L);
15765 %}
15766 ins_pipe(pipe_cmp_branch);
15767 %}
15768
15769 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15770 match(If cmp (CmpU op1 op2));
15771 effect(USE labl);
15772
15773 ins_cost(BRANCH_COST);
15774 format %{ "cbw$cmp $op1, $labl" %}
15775 ins_encode %{
15776 Label* L = $labl$$label;
15777 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15778 if (cond == Assembler::EQ || cond == Assembler::LS) {
15779 __ cbzw($op1$$Register, *L);
15780 } else {
15781 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15782 __ cbnzw($op1$$Register, *L);
15783 }
15784 %}
15785 ins_pipe(pipe_cmp_branch);
15786 %}
15787
15788 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15789 match(If cmp (CmpUL op1 op2));
15790 effect(USE labl);
15791
15792 ins_cost(BRANCH_COST);
15793 format %{ "cb$cmp $op1, $labl" %}
15794 ins_encode %{
15795 Label* L = $labl$$label;
15796 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15797 if (cond == Assembler::EQ || cond == Assembler::LS) {
15798 __ cbz($op1$$Register, *L);
15799 } else {
15800 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15801 __ cbnz($op1$$Register, *L);
15802 }
15803 %}
15804 ins_pipe(pipe_cmp_branch);
15805 %}
15806
15807 // Test bit and Branch
15808
15809 // Patterns for short (< 32KiB) variants
15810 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15811 match(If cmp (CmpL op1 op2));
15812 effect(USE labl);
15813
15814 ins_cost(BRANCH_COST);
15815 format %{ "cb$cmp $op1, $labl # long" %}
15816 ins_encode %{
15817 Label* L = $labl$$label;
15818 Assembler::Condition cond =
15819 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15820 __ tbr(cond, $op1$$Register, 63, *L);
15821 %}
15822 ins_pipe(pipe_cmp_branch);
15823 ins_short_branch(1);
15824 %}
15825
15826 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15827 match(If cmp (CmpI op1 op2));
15828 effect(USE labl);
15829
15830 ins_cost(BRANCH_COST);
15831 format %{ "cb$cmp $op1, $labl # int" %}
15832 ins_encode %{
15833 Label* L = $labl$$label;
15834 Assembler::Condition cond =
15835 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15836 __ tbr(cond, $op1$$Register, 31, *L);
15837 %}
15838 ins_pipe(pipe_cmp_branch);
15839 ins_short_branch(1);
15840 %}
15841
15842 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15843 match(If cmp (CmpL (AndL op1 op2) op3));
15844 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15845 effect(USE labl);
15846
15847 ins_cost(BRANCH_COST);
15848 format %{ "tb$cmp $op1, $op2, $labl" %}
15849 ins_encode %{
15850 Label* L = $labl$$label;
15851 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15852 int bit = exact_log2_long($op2$$constant);
15853 __ tbr(cond, $op1$$Register, bit, *L);
15854 %}
15855 ins_pipe(pipe_cmp_branch);
15856 ins_short_branch(1);
15857 %}
15858
15859 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15860 match(If cmp (CmpI (AndI op1 op2) op3));
15861 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15862 effect(USE labl);
15863
15864 ins_cost(BRANCH_COST);
15865 format %{ "tb$cmp $op1, $op2, $labl" %}
15866 ins_encode %{
15867 Label* L = $labl$$label;
15868 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15869 int bit = exact_log2((juint)$op2$$constant);
15870 __ tbr(cond, $op1$$Register, bit, *L);
15871 %}
15872 ins_pipe(pipe_cmp_branch);
15873 ins_short_branch(1);
15874 %}
15875
15876 // And far variants
15877 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15878 match(If cmp (CmpL op1 op2));
15879 effect(USE labl);
15880
15881 ins_cost(BRANCH_COST);
15882 format %{ "cb$cmp $op1, $labl # long" %}
15883 ins_encode %{
15884 Label* L = $labl$$label;
15885 Assembler::Condition cond =
15886 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15887 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15888 %}
15889 ins_pipe(pipe_cmp_branch);
15890 %}
15891
15892 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15893 match(If cmp (CmpI op1 op2));
15894 effect(USE labl);
15895
15896 ins_cost(BRANCH_COST);
15897 format %{ "cb$cmp $op1, $labl # int" %}
15898 ins_encode %{
15899 Label* L = $labl$$label;
15900 Assembler::Condition cond =
15901 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15902 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15903 %}
15904 ins_pipe(pipe_cmp_branch);
15905 %}
15906
15907 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15908 match(If cmp (CmpL (AndL op1 op2) op3));
15909 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15910 effect(USE labl);
15911
15912 ins_cost(BRANCH_COST);
15913 format %{ "tb$cmp $op1, $op2, $labl" %}
15914 ins_encode %{
15915 Label* L = $labl$$label;
15916 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15917 int bit = exact_log2_long($op2$$constant);
15918 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15919 %}
15920 ins_pipe(pipe_cmp_branch);
15921 %}
15922
15923 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15924 match(If cmp (CmpI (AndI op1 op2) op3));
15925 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15926 effect(USE labl);
15927
15928 ins_cost(BRANCH_COST);
15929 format %{ "tb$cmp $op1, $op2, $labl" %}
15930 ins_encode %{
15931 Label* L = $labl$$label;
15932 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15933 int bit = exact_log2((juint)$op2$$constant);
15934 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15935 %}
15936 ins_pipe(pipe_cmp_branch);
15937 %}
15938
15939 // Test bits
15940
15941 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15942 match(Set cr (CmpL (AndL op1 op2) op3));
15943 predicate(Assembler::operand_valid_for_logical_immediate
15944 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15945
15946 ins_cost(INSN_COST);
15947 format %{ "tst $op1, $op2 # long" %}
15948 ins_encode %{
15949 __ tst($op1$$Register, $op2$$constant);
15950 %}
15951 ins_pipe(ialu_reg_reg);
15952 %}
15953
15954 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15955 match(Set cr (CmpI (AndI op1 op2) op3));
15956 predicate(Assembler::operand_valid_for_logical_immediate
15957 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15958
15959 ins_cost(INSN_COST);
15960 format %{ "tst $op1, $op2 # int" %}
15961 ins_encode %{
15962 __ tstw($op1$$Register, $op2$$constant);
15963 %}
15964 ins_pipe(ialu_reg_reg);
15965 %}
15966
15967 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15968 match(Set cr (CmpL (AndL op1 op2) op3));
15969
15970 ins_cost(INSN_COST);
15971 format %{ "tst $op1, $op2 # long" %}
15972 ins_encode %{
15973 __ tst($op1$$Register, $op2$$Register);
15974 %}
15975 ins_pipe(ialu_reg_reg);
15976 %}
15977
15978 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15979 match(Set cr (CmpI (AndI op1 op2) op3));
15980
15981 ins_cost(INSN_COST);
15982 format %{ "tstw $op1, $op2 # int" %}
15983 ins_encode %{
15984 __ tstw($op1$$Register, $op2$$Register);
15985 %}
15986 ins_pipe(ialu_reg_reg);
15987 %}
15988
15989
15990 // Conditional Far Branch
15991 // Conditional Far Branch Unsigned
15992 // TODO: fixme
15993
15994 // counted loop end branch near
15995 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15996 %{
15997 match(CountedLoopEnd cmp cr);
15998
15999 effect(USE lbl);
16000
16001 ins_cost(BRANCH_COST);
16002 // short variant.
16003 // ins_short_branch(1);
16004 format %{ "b$cmp $lbl \t// counted loop end" %}
16005
16006 ins_encode(aarch64_enc_br_con(cmp, lbl));
16007
16008 ins_pipe(pipe_branch);
16009 %}
16010
16011 // counted loop end branch far
16012 // TODO: fixme
16013
16014 // ============================================================================
16015 // inlined locking and unlocking
16016
16017 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16018 %{
16019 predicate(LockingMode != LM_LIGHTWEIGHT);
16020 match(Set cr (FastLock object box));
16021 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16022
16023 ins_cost(5 * INSN_COST);
16024 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16025
16026 ins_encode %{
16027 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16028 %}
16029
16030 ins_pipe(pipe_serial);
16031 %}
16032
16033 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16034 %{
16035 predicate(LockingMode != LM_LIGHTWEIGHT);
16036 match(Set cr (FastUnlock object box));
16037 effect(TEMP tmp, TEMP tmp2);
16038
16039 ins_cost(5 * INSN_COST);
16040 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16041
16042 ins_encode %{
16043 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16044 %}
16045
16046 ins_pipe(pipe_serial);
16047 %}
16048
16049 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16050 %{
16051 predicate(LockingMode == LM_LIGHTWEIGHT);
16052 match(Set cr (FastLock object box));
16053 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16054
16055 ins_cost(5 * INSN_COST);
16056 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16057
16058 ins_encode %{
16059 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16060 %}
16061
16062 ins_pipe(pipe_serial);
16063 %}
16064
16065 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16066 %{
16067 predicate(LockingMode == LM_LIGHTWEIGHT);
16068 match(Set cr (FastUnlock object box));
16069 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16070
16071 ins_cost(5 * INSN_COST);
16072 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16073
16074 ins_encode %{
16075 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16076 %}
16077
16078 ins_pipe(pipe_serial);
16079 %}
16080
16081 // ============================================================================
16082 // Safepoint Instructions
16083
16084 // TODO
16085 // provide a near and far version of this code
16086
16087 instruct safePoint(rFlagsReg cr, iRegP poll)
16088 %{
16089 match(SafePoint poll);
16090 effect(KILL cr);
16091
16092 format %{
16093 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16094 %}
16095 ins_encode %{
16096 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16097 %}
16098 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16099 %}
16100
16101
16102 // ============================================================================
16103 // Procedure Call/Return Instructions
16104
16105 // Call Java Static Instruction
16106
16107 instruct CallStaticJavaDirect(method meth)
16108 %{
16109 match(CallStaticJava);
16110
16111 effect(USE meth);
16112
16113 ins_cost(CALL_COST);
16114
16115 format %{ "call,static $meth \t// ==> " %}
16116
16117 ins_encode(aarch64_enc_java_static_call(meth),
16118 aarch64_enc_call_epilog);
16119
16120 ins_pipe(pipe_class_call);
16121 %}
16122
16123 // TO HERE
16124
16125 // Call Java Dynamic Instruction
16126 instruct CallDynamicJavaDirect(method meth)
16127 %{
16128 match(CallDynamicJava);
16129
16130 effect(USE meth);
16131
16132 ins_cost(CALL_COST);
16133
16134 format %{ "CALL,dynamic $meth \t// ==> " %}
16135
16136 ins_encode(aarch64_enc_java_dynamic_call(meth),
16137 aarch64_enc_call_epilog);
16138
16139 ins_pipe(pipe_class_call);
16140 %}
16141
16142 // Call Runtime Instruction
16143
16144 instruct CallRuntimeDirect(method meth)
16145 %{
16146 match(CallRuntime);
16147
16148 effect(USE meth);
16149
16150 ins_cost(CALL_COST);
16151
16152 format %{ "CALL, runtime $meth" %}
16153
16154 ins_encode( aarch64_enc_java_to_runtime(meth) );
16155
16156 ins_pipe(pipe_class_call);
16157 %}
16158
16159 // Call Runtime Instruction
16160
16161 instruct CallLeafDirect(method meth)
16162 %{
16163 match(CallLeaf);
16164
16165 effect(USE meth);
16166
16167 ins_cost(CALL_COST);
16168
16169 format %{ "CALL, runtime leaf $meth" %}
16170
16171 ins_encode( aarch64_enc_java_to_runtime(meth) );
16172
16173 ins_pipe(pipe_class_call);
16174 %}
16175
16176 // Call Runtime Instruction without safepoint and with vector arguments
16177 instruct CallLeafDirectVector(method meth)
16178 %{
16179 match(CallLeafVector);
16180
16181 effect(USE meth);
16182
16183 ins_cost(CALL_COST);
16184
16185 format %{ "CALL, runtime leaf vector $meth" %}
16186
16187 ins_encode(aarch64_enc_java_to_runtime(meth));
16188
16189 ins_pipe(pipe_class_call);
16190 %}
16191
16192 // Call Runtime Instruction
16193
16194 instruct CallLeafNoFPDirect(method meth)
16195 %{
16196 match(CallLeafNoFP);
16197
16198 effect(USE meth);
16199
16200 ins_cost(CALL_COST);
16201
16202 format %{ "CALL, runtime leaf nofp $meth" %}
16203
16204 ins_encode( aarch64_enc_java_to_runtime(meth) );
16205
16206 ins_pipe(pipe_class_call);
16207 %}
16208
16209 // Tail Call; Jump from runtime stub to Java code.
16210 // Also known as an 'interprocedural jump'.
16211 // Target of jump will eventually return to caller.
16212 // TailJump below removes the return address.
16213 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16214 // emitted just above the TailCall which has reset rfp to the caller state.
16215 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16216 %{
16217 match(TailCall jump_target method_ptr);
16218
16219 ins_cost(CALL_COST);
16220
16221 format %{ "br $jump_target\t# $method_ptr holds method" %}
16222
16223 ins_encode(aarch64_enc_tail_call(jump_target));
16224
16225 ins_pipe(pipe_class_call);
16226 %}
16227
16228 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16229 %{
16230 match(TailJump jump_target ex_oop);
16231
16232 ins_cost(CALL_COST);
16233
16234 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16235
16236 ins_encode(aarch64_enc_tail_jmp(jump_target));
16237
16238 ins_pipe(pipe_class_call);
16239 %}
16240
16241 // Forward exception.
16242 instruct ForwardExceptionjmp()
16243 %{
16244 match(ForwardException);
16245 ins_cost(CALL_COST);
16246
16247 format %{ "b forward_exception_stub" %}
16248 ins_encode %{
16249 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16250 %}
16251 ins_pipe(pipe_class_call);
16252 %}
16253
16254 // Create exception oop: created by stack-crawling runtime code.
16255 // Created exception is now available to this handler, and is setup
16256 // just prior to jumping to this handler. No code emitted.
16257 // TODO check
16258 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16259 instruct CreateException(iRegP_R0 ex_oop)
16260 %{
16261 match(Set ex_oop (CreateEx));
16262
16263 format %{ " -- \t// exception oop; no code emitted" %}
16264
16265 size(0);
16266
16267 ins_encode( /*empty*/ );
16268
16269 ins_pipe(pipe_class_empty);
16270 %}
16271
16272 // Rethrow exception: The exception oop will come in the first
16273 // argument position. Then JUMP (not call) to the rethrow stub code.
16274 instruct RethrowException() %{
16275 match(Rethrow);
16276 ins_cost(CALL_COST);
16277
16278 format %{ "b rethrow_stub" %}
16279
16280 ins_encode( aarch64_enc_rethrow() );
16281
16282 ins_pipe(pipe_class_call);
16283 %}
16284
16285
16286 // Return Instruction
16287 // epilog node loads ret address into lr as part of frame pop
16288 instruct Ret()
16289 %{
16290 match(Return);
16291
16292 format %{ "ret\t// return register" %}
16293
16294 ins_encode( aarch64_enc_ret() );
16295
16296 ins_pipe(pipe_branch);
16297 %}
16298
16299 // Die now.
16300 instruct ShouldNotReachHere() %{
16301 match(Halt);
16302
16303 ins_cost(CALL_COST);
16304 format %{ "ShouldNotReachHere" %}
16305
16306 ins_encode %{
16307 if (is_reachable()) {
16308 __ stop(_halt_reason);
16309 }
16310 %}
16311
16312 ins_pipe(pipe_class_default);
16313 %}
16314
16315 // ============================================================================
16316 // Partial Subtype Check
16317 //
16318 // superklass array for an instance of the superklass. Set a hidden
16319 // internal cache on a hit (cache is checked with exposed code in
16320 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16321 // encoding ALSO sets flags.
16322
16323 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16324 %{
16325 match(Set result (PartialSubtypeCheck sub super));
16326 predicate(!UseSecondarySupersTable);
16327 effect(KILL cr, KILL temp);
16328
16329 ins_cost(20 * INSN_COST); // slightly larger than the next version
16330 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16331
16332 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16333
16334 opcode(0x1); // Force zero of result reg on hit
16335
16336 ins_pipe(pipe_class_memory);
16337 %}
16338
16339 // Two versions of partialSubtypeCheck, both used when we need to
16340 // search for a super class in the secondary supers array. The first
16341 // is used when we don't know _a priori_ the class being searched
16342 // for. The second, far more common, is used when we do know: this is
16343 // used for instanceof, checkcast, and any case where C2 can determine
16344 // it by constant propagation.
16345
16346 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16347 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16348 rFlagsReg cr)
16349 %{
16350 match(Set result (PartialSubtypeCheck sub super));
16351 predicate(UseSecondarySupersTable);
16352 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16353
16354 ins_cost(10 * INSN_COST); // slightly larger than the next version
16355 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16356
16357 ins_encode %{
16358 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16359 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16360 $vtemp$$FloatRegister,
16361 $result$$Register, /*L_success*/nullptr);
16362 %}
16363
16364 ins_pipe(pipe_class_memory);
16365 %}
16366
16367 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16368 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16369 rFlagsReg cr)
16370 %{
16371 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16372 predicate(UseSecondarySupersTable);
16373 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16374
16375 ins_cost(5 * INSN_COST); // smaller than the next version
16376 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16377
16378 ins_encode %{
16379 bool success = false;
16380 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16381 if (InlineSecondarySupersTest) {
16382 success =
16383 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16384 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16385 $vtemp$$FloatRegister,
16386 $result$$Register,
16387 super_klass_slot);
16388 } else {
16389 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16390 success = (call != nullptr);
16391 }
16392 if (!success) {
16393 ciEnv::current()->record_failure("CodeCache is full");
16394 return;
16395 }
16396 %}
16397
16398 ins_pipe(pipe_class_memory);
16399 %}
16400
16401 // Intrisics for String.compareTo()
16402
16403 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16404 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16405 %{
16406 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16407 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16408 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16409
16410 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16411 ins_encode %{
16412 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16413 __ string_compare($str1$$Register, $str2$$Register,
16414 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16415 $tmp1$$Register, $tmp2$$Register,
16416 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16417 %}
16418 ins_pipe(pipe_class_memory);
16419 %}
16420
16421 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16422 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16423 %{
16424 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16425 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16426 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16427
16428 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16429 ins_encode %{
16430 __ string_compare($str1$$Register, $str2$$Register,
16431 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16432 $tmp1$$Register, $tmp2$$Register,
16433 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16434 %}
16435 ins_pipe(pipe_class_memory);
16436 %}
16437
16438 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16439 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16440 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16441 %{
16442 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16443 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16444 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16445 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16446
16447 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16448 ins_encode %{
16449 __ string_compare($str1$$Register, $str2$$Register,
16450 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16451 $tmp1$$Register, $tmp2$$Register,
16452 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16453 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16454 %}
16455 ins_pipe(pipe_class_memory);
16456 %}
16457
16458 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16459 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16460 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16461 %{
16462 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16463 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16464 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16465 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16466
16467 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16468 ins_encode %{
16469 __ string_compare($str1$$Register, $str2$$Register,
16470 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16471 $tmp1$$Register, $tmp2$$Register,
16472 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16473 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16474 %}
16475 ins_pipe(pipe_class_memory);
16476 %}
16477
16478 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16479 // these string_compare variants as NEON register type for convenience so that the prototype of
16480 // string_compare can be shared with all variants.
16481
16482 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16483 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16484 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16485 pRegGov_P1 pgtmp2, rFlagsReg cr)
16486 %{
16487 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16488 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16489 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16490 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16491
16492 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16493 ins_encode %{
16494 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16495 __ string_compare($str1$$Register, $str2$$Register,
16496 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16497 $tmp1$$Register, $tmp2$$Register,
16498 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16499 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16500 StrIntrinsicNode::LL);
16501 %}
16502 ins_pipe(pipe_class_memory);
16503 %}
16504
16505 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16506 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16507 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16508 pRegGov_P1 pgtmp2, rFlagsReg cr)
16509 %{
16510 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16511 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16512 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16513 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16514
16515 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16516 ins_encode %{
16517 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16518 __ string_compare($str1$$Register, $str2$$Register,
16519 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16520 $tmp1$$Register, $tmp2$$Register,
16521 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16522 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16523 StrIntrinsicNode::LU);
16524 %}
16525 ins_pipe(pipe_class_memory);
16526 %}
16527
16528 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16529 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16530 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16531 pRegGov_P1 pgtmp2, rFlagsReg cr)
16532 %{
16533 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16534 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16535 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16536 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16537
16538 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16539 ins_encode %{
16540 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16541 __ string_compare($str1$$Register, $str2$$Register,
16542 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16543 $tmp1$$Register, $tmp2$$Register,
16544 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16545 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16546 StrIntrinsicNode::UL);
16547 %}
16548 ins_pipe(pipe_class_memory);
16549 %}
16550
16551 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16552 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16553 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16554 pRegGov_P1 pgtmp2, rFlagsReg cr)
16555 %{
16556 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16557 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16558 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16559 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16560
16561 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16562 ins_encode %{
16563 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16564 __ string_compare($str1$$Register, $str2$$Register,
16565 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16566 $tmp1$$Register, $tmp2$$Register,
16567 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16568 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16569 StrIntrinsicNode::UU);
16570 %}
16571 ins_pipe(pipe_class_memory);
16572 %}
16573
16574 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16575 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16576 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16577 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16578 %{
16579 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16580 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16581 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16582 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16583 TEMP vtmp0, TEMP vtmp1, KILL cr);
16584 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16585 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16586
16587 ins_encode %{
16588 __ string_indexof($str1$$Register, $str2$$Register,
16589 $cnt1$$Register, $cnt2$$Register,
16590 $tmp1$$Register, $tmp2$$Register,
16591 $tmp3$$Register, $tmp4$$Register,
16592 $tmp5$$Register, $tmp6$$Register,
16593 -1, $result$$Register, StrIntrinsicNode::UU);
16594 %}
16595 ins_pipe(pipe_class_memory);
16596 %}
16597
16598 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16599 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16600 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16601 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16602 %{
16603 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16604 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16605 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16606 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16607 TEMP vtmp0, TEMP vtmp1, KILL cr);
16608 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16609 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16610
16611 ins_encode %{
16612 __ string_indexof($str1$$Register, $str2$$Register,
16613 $cnt1$$Register, $cnt2$$Register,
16614 $tmp1$$Register, $tmp2$$Register,
16615 $tmp3$$Register, $tmp4$$Register,
16616 $tmp5$$Register, $tmp6$$Register,
16617 -1, $result$$Register, StrIntrinsicNode::LL);
16618 %}
16619 ins_pipe(pipe_class_memory);
16620 %}
16621
16622 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16623 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16624 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16625 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16626 %{
16627 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16628 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16629 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16630 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16631 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16632 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16633 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16634
16635 ins_encode %{
16636 __ string_indexof($str1$$Register, $str2$$Register,
16637 $cnt1$$Register, $cnt2$$Register,
16638 $tmp1$$Register, $tmp2$$Register,
16639 $tmp3$$Register, $tmp4$$Register,
16640 $tmp5$$Register, $tmp6$$Register,
16641 -1, $result$$Register, StrIntrinsicNode::UL);
16642 %}
16643 ins_pipe(pipe_class_memory);
16644 %}
16645
16646 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16647 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16648 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16649 %{
16650 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16651 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16652 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16653 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16654 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16655 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16656
16657 ins_encode %{
16658 int icnt2 = (int)$int_cnt2$$constant;
16659 __ string_indexof($str1$$Register, $str2$$Register,
16660 $cnt1$$Register, zr,
16661 $tmp1$$Register, $tmp2$$Register,
16662 $tmp3$$Register, $tmp4$$Register, zr, zr,
16663 icnt2, $result$$Register, StrIntrinsicNode::UU);
16664 %}
16665 ins_pipe(pipe_class_memory);
16666 %}
16667
16668 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16669 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16670 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16671 %{
16672 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16673 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16674 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16675 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16676 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16677 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16678
16679 ins_encode %{
16680 int icnt2 = (int)$int_cnt2$$constant;
16681 __ string_indexof($str1$$Register, $str2$$Register,
16682 $cnt1$$Register, zr,
16683 $tmp1$$Register, $tmp2$$Register,
16684 $tmp3$$Register, $tmp4$$Register, zr, zr,
16685 icnt2, $result$$Register, StrIntrinsicNode::LL);
16686 %}
16687 ins_pipe(pipe_class_memory);
16688 %}
16689
16690 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16691 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16692 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16693 %{
16694 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16695 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16696 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16697 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16698 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16699 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16700
16701 ins_encode %{
16702 int icnt2 = (int)$int_cnt2$$constant;
16703 __ string_indexof($str1$$Register, $str2$$Register,
16704 $cnt1$$Register, zr,
16705 $tmp1$$Register, $tmp2$$Register,
16706 $tmp3$$Register, $tmp4$$Register, zr, zr,
16707 icnt2, $result$$Register, StrIntrinsicNode::UL);
16708 %}
16709 ins_pipe(pipe_class_memory);
16710 %}
16711
16712 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16713 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16714 iRegINoSp tmp3, rFlagsReg cr)
16715 %{
16716 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16717 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16718 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16719 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16720
16721 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16722
16723 ins_encode %{
16724 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16725 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16726 $tmp3$$Register);
16727 %}
16728 ins_pipe(pipe_class_memory);
16729 %}
16730
16731 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16732 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16733 iRegINoSp tmp3, rFlagsReg cr)
16734 %{
16735 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16736 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16737 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16738 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16739
16740 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16741
16742 ins_encode %{
16743 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16744 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16745 $tmp3$$Register);
16746 %}
16747 ins_pipe(pipe_class_memory);
16748 %}
16749
16750 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16751 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16752 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16753 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16754 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16755 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16756 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16757 ins_encode %{
16758 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16759 $result$$Register, $ztmp1$$FloatRegister,
16760 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16761 $ptmp$$PRegister, true /* isL */);
16762 %}
16763 ins_pipe(pipe_class_memory);
16764 %}
16765
16766 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16767 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16768 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16769 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16770 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16771 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16772 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16773 ins_encode %{
16774 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16775 $result$$Register, $ztmp1$$FloatRegister,
16776 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16777 $ptmp$$PRegister, false /* isL */);
16778 %}
16779 ins_pipe(pipe_class_memory);
16780 %}
16781
16782 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16783 iRegI_R0 result, rFlagsReg cr)
16784 %{
16785 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16786 match(Set result (StrEquals (Binary str1 str2) cnt));
16787 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16788
16789 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16790 ins_encode %{
16791 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16792 __ string_equals($str1$$Register, $str2$$Register,
16793 $result$$Register, $cnt$$Register);
16794 %}
16795 ins_pipe(pipe_class_memory);
16796 %}
16797
16798 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16799 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16800 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16801 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16802 iRegP_R10 tmp, rFlagsReg cr)
16803 %{
16804 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16805 match(Set result (AryEq ary1 ary2));
16806 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16807 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16808 TEMP vtmp6, TEMP vtmp7, KILL cr);
16809
16810 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16811 ins_encode %{
16812 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16813 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16814 $result$$Register, $tmp$$Register, 1);
16815 if (tpc == nullptr) {
16816 ciEnv::current()->record_failure("CodeCache is full");
16817 return;
16818 }
16819 %}
16820 ins_pipe(pipe_class_memory);
16821 %}
16822
16823 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16824 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16825 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16826 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16827 iRegP_R10 tmp, rFlagsReg cr)
16828 %{
16829 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16830 match(Set result (AryEq ary1 ary2));
16831 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16832 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16833 TEMP vtmp6, TEMP vtmp7, KILL cr);
16834
16835 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16836 ins_encode %{
16837 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16838 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16839 $result$$Register, $tmp$$Register, 2);
16840 if (tpc == nullptr) {
16841 ciEnv::current()->record_failure("CodeCache is full");
16842 return;
16843 }
16844 %}
16845 ins_pipe(pipe_class_memory);
16846 %}
16847
16848 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16849 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16850 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16851 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16852 %{
16853 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16854 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16855 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16856
16857 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16858 ins_encode %{
16859 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16860 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16861 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16862 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16863 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16864 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16865 (BasicType)$basic_type$$constant);
16866 if (tpc == nullptr) {
16867 ciEnv::current()->record_failure("CodeCache is full");
16868 return;
16869 }
16870 %}
16871 ins_pipe(pipe_class_memory);
16872 %}
16873
16874 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16875 %{
16876 match(Set result (CountPositives ary1 len));
16877 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16878 format %{ "count positives byte[] $ary1,$len -> $result" %}
16879 ins_encode %{
16880 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16881 if (tpc == nullptr) {
16882 ciEnv::current()->record_failure("CodeCache is full");
16883 return;
16884 }
16885 %}
16886 ins_pipe( pipe_slow );
16887 %}
16888
16889 // fast char[] to byte[] compression
16890 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16891 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16892 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16893 iRegI_R0 result, rFlagsReg cr)
16894 %{
16895 match(Set result (StrCompressedCopy src (Binary dst len)));
16896 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16897 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16898
16899 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16900 ins_encode %{
16901 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16902 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16903 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16904 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16905 %}
16906 ins_pipe(pipe_slow);
16907 %}
16908
16909 // fast byte[] to char[] inflation
16910 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16911 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16912 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16913 %{
16914 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16915 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16916 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16917 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16918
16919 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16920 ins_encode %{
16921 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16922 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16923 $vtmp2$$FloatRegister, $tmp$$Register);
16924 if (tpc == nullptr) {
16925 ciEnv::current()->record_failure("CodeCache is full");
16926 return;
16927 }
16928 %}
16929 ins_pipe(pipe_class_memory);
16930 %}
16931
16932 // encode char[] to byte[] in ISO_8859_1
16933 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16934 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16935 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16936 iRegI_R0 result, rFlagsReg cr)
16937 %{
16938 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16939 match(Set result (EncodeISOArray src (Binary dst len)));
16940 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16941 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16942
16943 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16944 ins_encode %{
16945 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16946 $result$$Register, false,
16947 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16948 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16949 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16950 %}
16951 ins_pipe(pipe_class_memory);
16952 %}
16953
16954 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16955 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16956 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16957 iRegI_R0 result, rFlagsReg cr)
16958 %{
16959 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16960 match(Set result (EncodeISOArray src (Binary dst len)));
16961 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16962 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16963
16964 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16965 ins_encode %{
16966 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16967 $result$$Register, true,
16968 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16969 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16970 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16971 %}
16972 ins_pipe(pipe_class_memory);
16973 %}
16974
16975 //----------------------------- CompressBits/ExpandBits ------------------------
16976
16977 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16978 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16979 match(Set dst (CompressBits src mask));
16980 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16981 format %{ "mov $tsrc, $src\n\t"
16982 "mov $tmask, $mask\n\t"
16983 "bext $tdst, $tsrc, $tmask\n\t"
16984 "mov $dst, $tdst"
16985 %}
16986 ins_encode %{
16987 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16988 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16989 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16990 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16991 %}
16992 ins_pipe(pipe_slow);
16993 %}
16994
16995 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16996 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16997 match(Set dst (CompressBits (LoadI mem) mask));
16998 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16999 format %{ "ldrs $tsrc, $mem\n\t"
17000 "ldrs $tmask, $mask\n\t"
17001 "bext $tdst, $tsrc, $tmask\n\t"
17002 "mov $dst, $tdst"
17003 %}
17004 ins_encode %{
17005 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17006 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17007 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17008 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17009 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17010 %}
17011 ins_pipe(pipe_slow);
17012 %}
17013
17014 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17015 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17016 match(Set dst (CompressBits src mask));
17017 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17018 format %{ "mov $tsrc, $src\n\t"
17019 "mov $tmask, $mask\n\t"
17020 "bext $tdst, $tsrc, $tmask\n\t"
17021 "mov $dst, $tdst"
17022 %}
17023 ins_encode %{
17024 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17025 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17026 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17027 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17028 %}
17029 ins_pipe(pipe_slow);
17030 %}
17031
17032 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17033 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17034 match(Set dst (CompressBits (LoadL mem) mask));
17035 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17036 format %{ "ldrd $tsrc, $mem\n\t"
17037 "ldrd $tmask, $mask\n\t"
17038 "bext $tdst, $tsrc, $tmask\n\t"
17039 "mov $dst, $tdst"
17040 %}
17041 ins_encode %{
17042 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17043 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17044 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17045 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17046 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17047 %}
17048 ins_pipe(pipe_slow);
17049 %}
17050
17051 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17052 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17053 match(Set dst (ExpandBits src mask));
17054 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17055 format %{ "mov $tsrc, $src\n\t"
17056 "mov $tmask, $mask\n\t"
17057 "bdep $tdst, $tsrc, $tmask\n\t"
17058 "mov $dst, $tdst"
17059 %}
17060 ins_encode %{
17061 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17062 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17063 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17064 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17065 %}
17066 ins_pipe(pipe_slow);
17067 %}
17068
17069 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17070 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17071 match(Set dst (ExpandBits (LoadI mem) mask));
17072 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17073 format %{ "ldrs $tsrc, $mem\n\t"
17074 "ldrs $tmask, $mask\n\t"
17075 "bdep $tdst, $tsrc, $tmask\n\t"
17076 "mov $dst, $tdst"
17077 %}
17078 ins_encode %{
17079 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17080 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17081 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17082 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17083 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17084 %}
17085 ins_pipe(pipe_slow);
17086 %}
17087
17088 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17089 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17090 match(Set dst (ExpandBits src mask));
17091 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17092 format %{ "mov $tsrc, $src\n\t"
17093 "mov $tmask, $mask\n\t"
17094 "bdep $tdst, $tsrc, $tmask\n\t"
17095 "mov $dst, $tdst"
17096 %}
17097 ins_encode %{
17098 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17099 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17100 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17101 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17102 %}
17103 ins_pipe(pipe_slow);
17104 %}
17105
17106
17107 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17108 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17109 match(Set dst (ExpandBits (LoadL mem) mask));
17110 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17111 format %{ "ldrd $tsrc, $mem\n\t"
17112 "ldrd $tmask, $mask\n\t"
17113 "bdep $tdst, $tsrc, $tmask\n\t"
17114 "mov $dst, $tdst"
17115 %}
17116 ins_encode %{
17117 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17118 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17119 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17120 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17121 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17122 %}
17123 ins_pipe(pipe_slow);
17124 %}
17125
17126 // ============================================================================
17127 // This name is KNOWN by the ADLC and cannot be changed.
17128 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17129 // for this guy.
17130 instruct tlsLoadP(thread_RegP dst)
17131 %{
17132 match(Set dst (ThreadLocal));
17133
17134 ins_cost(0);
17135
17136 format %{ " -- \t// $dst=Thread::current(), empty" %}
17137
17138 size(0);
17139
17140 ins_encode( /*empty*/ );
17141
17142 ins_pipe(pipe_class_empty);
17143 %}
17144
17145 //----------PEEPHOLE RULES-----------------------------------------------------
17146 // These must follow all instruction definitions as they use the names
17147 // defined in the instructions definitions.
17148 //
17149 // peepmatch ( root_instr_name [preceding_instruction]* );
17150 //
17151 // peepconstraint %{
17152 // (instruction_number.operand_name relational_op instruction_number.operand_name
17153 // [, ...] );
17154 // // instruction numbers are zero-based using left to right order in peepmatch
17155 //
17156 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17157 // // provide an instruction_number.operand_name for each operand that appears
17158 // // in the replacement instruction's match rule
17159 //
17160 // ---------VM FLAGS---------------------------------------------------------
17161 //
17162 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17163 //
17164 // Each peephole rule is given an identifying number starting with zero and
17165 // increasing by one in the order seen by the parser. An individual peephole
17166 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17167 // on the command-line.
17168 //
17169 // ---------CURRENT LIMITATIONS----------------------------------------------
17170 //
17171 // Only match adjacent instructions in same basic block
17172 // Only equality constraints
17173 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17174 // Only one replacement instruction
17175 //
17176 // ---------EXAMPLE----------------------------------------------------------
17177 //
17178 // // pertinent parts of existing instructions in architecture description
17179 // instruct movI(iRegINoSp dst, iRegI src)
17180 // %{
17181 // match(Set dst (CopyI src));
17182 // %}
17183 //
17184 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17185 // %{
17186 // match(Set dst (AddI dst src));
17187 // effect(KILL cr);
17188 // %}
17189 //
17190 // // Change (inc mov) to lea
17191 // peephole %{
17192 // // increment preceded by register-register move
17193 // peepmatch ( incI_iReg movI );
17194 // // require that the destination register of the increment
17195 // // match the destination register of the move
17196 // peepconstraint ( 0.dst == 1.dst );
17197 // // construct a replacement instruction that sets
17198 // // the destination to ( move's source register + one )
17199 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17200 // %}
17201 //
17202
17203 // Implementation no longer uses movX instructions since
17204 // machine-independent system no longer uses CopyX nodes.
17205 //
17206 // peephole
17207 // %{
17208 // peepmatch (incI_iReg movI);
17209 // peepconstraint (0.dst == 1.dst);
17210 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17211 // %}
17212
17213 // peephole
17214 // %{
17215 // peepmatch (decI_iReg movI);
17216 // peepconstraint (0.dst == 1.dst);
17217 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17218 // %}
17219
17220 // peephole
17221 // %{
17222 // peepmatch (addI_iReg_imm movI);
17223 // peepconstraint (0.dst == 1.dst);
17224 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17225 // %}
17226
17227 // peephole
17228 // %{
17229 // peepmatch (incL_iReg movL);
17230 // peepconstraint (0.dst == 1.dst);
17231 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17232 // %}
17233
17234 // peephole
17235 // %{
17236 // peepmatch (decL_iReg movL);
17237 // peepconstraint (0.dst == 1.dst);
17238 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17239 // %}
17240
17241 // peephole
17242 // %{
17243 // peepmatch (addL_iReg_imm movL);
17244 // peepconstraint (0.dst == 1.dst);
17245 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17246 // %}
17247
17248 // peephole
17249 // %{
17250 // peepmatch (addP_iReg_imm movP);
17251 // peepconstraint (0.dst == 1.dst);
17252 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17253 // %}
17254
17255 // // Change load of spilled value to only a spill
17256 // instruct storeI(memory mem, iRegI src)
17257 // %{
17258 // match(Set mem (StoreI mem src));
17259 // %}
17260 //
17261 // instruct loadI(iRegINoSp dst, memory mem)
17262 // %{
17263 // match(Set dst (LoadI mem));
17264 // %}
17265 //
17266
17267 //----------SMARTSPILL RULES---------------------------------------------------
17268 // These must follow all instruction definitions as they use the names
17269 // defined in the instructions definitions.
17270
17271 // Local Variables:
17272 // mode: c++
17273 // End: