1 //
2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return MacroAssembler::max_trampoline_stub_size(); // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1652 // lea(rscratch1, RuntimeAddress(addr)
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else {
1658 return 6 * NativeInstruction::instruction_size;
1659 }
1660 }
1661
1662 //=============================================================================
1663
1664 #ifndef PRODUCT
1665 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1666 st->print("BREAKPOINT");
1667 }
1668 #endif
1669
1670 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1671 __ brk(0);
1672 }
1673
1674 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1675 return MachNode::size(ra_);
1676 }
1677
1678 //=============================================================================
1679
1680 #ifndef PRODUCT
1681 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1682 st->print("nop \t# %d bytes pad for loops and calls", _count);
1683 }
1684 #endif
1685
1686 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1687 for (int i = 0; i < _count; i++) {
1688 __ nop();
1689 }
1690 }
1691
1692 uint MachNopNode::size(PhaseRegAlloc*) const {
1693 return _count * NativeInstruction::instruction_size;
1694 }
1695
1696 //=============================================================================
1697 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1698
1699 int ConstantTable::calculate_table_base_offset() const {
1700 return 0; // absolute addressing, no offset
1701 }
1702
1703 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1704 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1705 ShouldNotReachHere();
1706 }
1707
1708 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1709 // Empty encoding
1710 }
1711
1712 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1713 return 0;
1714 }
1715
1716 #ifndef PRODUCT
1717 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1718 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1719 }
1720 #endif
1721
1722 #ifndef PRODUCT
1723 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1724 Compile* C = ra_->C;
1725
1726 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1727
1728 if (C->output()->need_stack_bang(framesize))
1729 st->print("# stack bang size=%d\n\t", framesize);
1730
1731 if (VM_Version::use_rop_protection()) {
1732 st->print("ldr zr, [lr]\n\t");
1733 st->print("paciaz\n\t");
1734 }
1735 if (framesize < ((1 << 9) + 2 * wordSize)) {
1736 st->print("sub sp, sp, #%d\n\t", framesize);
1737 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1738 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1739 } else {
1740 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1741 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1742 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1743 st->print("sub sp, sp, rscratch1");
1744 }
1745 if (C->stub_function() == nullptr) {
1746 st->print("\n\t");
1747 st->print("ldr rscratch1, [guard]\n\t");
1748 st->print("dmb ishld\n\t");
1749 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1750 st->print("cmp rscratch1, rscratch2\n\t");
1751 st->print("b.eq skip");
1752 st->print("\n\t");
1753 st->print("blr #nmethod_entry_barrier_stub\n\t");
1754 st->print("b skip\n\t");
1755 st->print("guard: int\n\t");
1756 st->print("\n\t");
1757 st->print("skip:\n\t");
1758 }
1759 }
1760 #endif
1761
1762 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1763 Compile* C = ra_->C;
1764
1765 // n.b. frame size includes space for return pc and rfp
1766 const int framesize = C->output()->frame_size_in_bytes();
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 if (C->clinit_barrier_on_entry()) {
1773 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1774
1775 Label L_skip_barrier;
1776
1777 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1778 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1779 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1780 __ bind(L_skip_barrier);
1781 }
1782
1783 if (C->max_vector_size() > 0) {
1784 __ reinitialize_ptrue();
1785 }
1786
1787 int bangsize = C->output()->bang_size_in_bytes();
1788 if (C->output()->need_stack_bang(bangsize))
1789 __ generate_stack_overflow_check(bangsize);
1790
1791 __ build_frame(framesize);
1792
1793 if (C->stub_function() == nullptr) {
1794 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1795 // Dummy labels for just measuring the code size
1796 Label dummy_slow_path;
1797 Label dummy_continuation;
1798 Label dummy_guard;
1799 Label* slow_path = &dummy_slow_path;
1800 Label* continuation = &dummy_continuation;
1801 Label* guard = &dummy_guard;
1802 if (!Compile::current()->output()->in_scratch_emit_size()) {
1803 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1804 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1805 Compile::current()->output()->add_stub(stub);
1806 slow_path = &stub->entry();
1807 continuation = &stub->continuation();
1808 guard = &stub->guard();
1809 }
1810 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1811 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1812 }
1813
1814 if (VerifyStackAtCalls) {
1815 Unimplemented();
1816 }
1817
1818 C->output()->set_frame_complete(__ offset());
1819
1820 if (C->has_mach_constant_base_node()) {
1821 // NOTE: We set the table base offset here because users might be
1822 // emitted before MachConstantBaseNode.
1823 ConstantTable& constant_table = C->output()->constant_table();
1824 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1825 }
1826 }
1827
1828 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1829 {
1830 return MachNode::size(ra_); // too many variables; just compute it
1831 // the hard way
1832 }
1833
1834 int MachPrologNode::reloc() const
1835 {
1836 return 0;
1837 }
1838
1839 //=============================================================================
1840
1841 #ifndef PRODUCT
1842 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1843 Compile* C = ra_->C;
1844 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1845
1846 st->print("# pop frame %d\n\t",framesize);
1847
1848 if (framesize == 0) {
1849 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1850 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1851 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1852 st->print("add sp, sp, #%d\n\t", framesize);
1853 } else {
1854 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1855 st->print("add sp, sp, rscratch1\n\t");
1856 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1857 }
1858 if (VM_Version::use_rop_protection()) {
1859 st->print("autiaz\n\t");
1860 st->print("ldr zr, [lr]\n\t");
1861 }
1862
1863 if (do_polling() && C->is_method_compilation()) {
1864 st->print("# test polling word\n\t");
1865 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1866 st->print("cmp sp, rscratch1\n\t");
1867 st->print("bhi #slow_path");
1868 }
1869 }
1870 #endif
1871
1872 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1873 Compile* C = ra_->C;
1874 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1875
1876 __ remove_frame(framesize);
1877
1878 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1879 __ reserved_stack_check();
1880 }
1881
1882 if (do_polling() && C->is_method_compilation()) {
1883 Label dummy_label;
1884 Label* code_stub = &dummy_label;
1885 if (!C->output()->in_scratch_emit_size()) {
1886 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1887 C->output()->add_stub(stub);
1888 code_stub = &stub->entry();
1889 }
1890 __ relocate(relocInfo::poll_return_type);
1891 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1892 }
1893 }
1894
1895 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1896 // Variable size. Determine dynamically.
1897 return MachNode::size(ra_);
1898 }
1899
1900 int MachEpilogNode::reloc() const {
1901 // Return number of relocatable values contained in this instruction.
1902 return 1; // 1 for polling page.
1903 }
1904
1905 const Pipeline * MachEpilogNode::pipeline() const {
1906 return MachNode::pipeline_class();
1907 }
1908
1909 //=============================================================================
1910
1911 static enum RC rc_class(OptoReg::Name reg) {
1912
1913 if (reg == OptoReg::Bad) {
1914 return rc_bad;
1915 }
1916
1917 // we have 32 int registers * 2 halves
1918 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1919
1920 if (reg < slots_of_int_registers) {
1921 return rc_int;
1922 }
1923
1924 // we have 32 float register * 8 halves
1925 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1926 if (reg < slots_of_int_registers + slots_of_float_registers) {
1927 return rc_float;
1928 }
1929
1930 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1931 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1932 return rc_predicate;
1933 }
1934
1935 // Between predicate regs & stack is the flags.
1936 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1937
1938 return rc_stack;
1939 }
1940
1941 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1942 Compile* C = ra_->C;
1943
1944 // Get registers to move.
1945 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1946 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1947 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1948 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1949
1950 enum RC src_hi_rc = rc_class(src_hi);
1951 enum RC src_lo_rc = rc_class(src_lo);
1952 enum RC dst_hi_rc = rc_class(dst_hi);
1953 enum RC dst_lo_rc = rc_class(dst_lo);
1954
1955 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1956
1957 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1958 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1959 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1960 "expected aligned-adjacent pairs");
1961 }
1962
1963 if (src_lo == dst_lo && src_hi == dst_hi) {
1964 return 0; // Self copy, no move.
1965 }
1966
1967 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1968 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1969 int src_offset = ra_->reg2offset(src_lo);
1970 int dst_offset = ra_->reg2offset(dst_lo);
1971
1972 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1973 uint ireg = ideal_reg();
1974 if (ireg == Op_VecA && masm) {
1975 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1976 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1977 // stack->stack
1978 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1979 sve_vector_reg_size_in_bytes);
1980 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1981 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1982 sve_vector_reg_size_in_bytes);
1983 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1984 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1985 sve_vector_reg_size_in_bytes);
1986 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1987 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1988 as_FloatRegister(Matcher::_regEncode[src_lo]),
1989 as_FloatRegister(Matcher::_regEncode[src_lo]));
1990 } else {
1991 ShouldNotReachHere();
1992 }
1993 } else if (masm) {
1994 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1995 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1996 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1997 // stack->stack
1998 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1999 if (ireg == Op_VecD) {
2000 __ unspill(rscratch1, true, src_offset);
2001 __ spill(rscratch1, true, dst_offset);
2002 } else {
2003 __ spill_copy128(src_offset, dst_offset);
2004 }
2005 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2006 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2007 ireg == Op_VecD ? __ T8B : __ T16B,
2008 as_FloatRegister(Matcher::_regEncode[src_lo]));
2009 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2010 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2011 ireg == Op_VecD ? __ D : __ Q,
2012 ra_->reg2offset(dst_lo));
2013 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2014 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2015 ireg == Op_VecD ? __ D : __ Q,
2016 ra_->reg2offset(src_lo));
2017 } else {
2018 ShouldNotReachHere();
2019 }
2020 }
2021 } else if (masm) {
2022 switch (src_lo_rc) {
2023 case rc_int:
2024 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2025 if (is64) {
2026 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2027 as_Register(Matcher::_regEncode[src_lo]));
2028 } else {
2029 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2030 as_Register(Matcher::_regEncode[src_lo]));
2031 }
2032 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2033 if (is64) {
2034 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2035 as_Register(Matcher::_regEncode[src_lo]));
2036 } else {
2037 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2038 as_Register(Matcher::_regEncode[src_lo]));
2039 }
2040 } else { // gpr --> stack spill
2041 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2042 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2043 }
2044 break;
2045 case rc_float:
2046 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2047 if (is64) {
2048 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2049 as_FloatRegister(Matcher::_regEncode[src_lo]));
2050 } else {
2051 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2052 as_FloatRegister(Matcher::_regEncode[src_lo]));
2053 }
2054 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2055 if (is64) {
2056 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2057 as_FloatRegister(Matcher::_regEncode[src_lo]));
2058 } else {
2059 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2060 as_FloatRegister(Matcher::_regEncode[src_lo]));
2061 }
2062 } else { // fpr --> stack spill
2063 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2064 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2065 is64 ? __ D : __ S, dst_offset);
2066 }
2067 break;
2068 case rc_stack:
2069 if (dst_lo_rc == rc_int) { // stack --> gpr load
2070 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2071 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2072 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2073 is64 ? __ D : __ S, src_offset);
2074 } else if (dst_lo_rc == rc_predicate) {
2075 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2076 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2077 } else { // stack --> stack copy
2078 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2079 if (ideal_reg() == Op_RegVectMask) {
2080 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2081 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2082 } else {
2083 __ unspill(rscratch1, is64, src_offset);
2084 __ spill(rscratch1, is64, dst_offset);
2085 }
2086 }
2087 break;
2088 case rc_predicate:
2089 if (dst_lo_rc == rc_predicate) {
2090 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2091 } else if (dst_lo_rc == rc_stack) {
2092 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2093 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2094 } else {
2095 assert(false, "bad src and dst rc_class combination.");
2096 ShouldNotReachHere();
2097 }
2098 break;
2099 default:
2100 assert(false, "bad rc_class for spill");
2101 ShouldNotReachHere();
2102 }
2103 }
2104
2105 if (st) {
2106 st->print("spill ");
2107 if (src_lo_rc == rc_stack) {
2108 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2109 } else {
2110 st->print("%s -> ", Matcher::regName[src_lo]);
2111 }
2112 if (dst_lo_rc == rc_stack) {
2113 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2114 } else {
2115 st->print("%s", Matcher::regName[dst_lo]);
2116 }
2117 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2118 int vsize = 0;
2119 switch (ideal_reg()) {
2120 case Op_VecD:
2121 vsize = 64;
2122 break;
2123 case Op_VecX:
2124 vsize = 128;
2125 break;
2126 case Op_VecA:
2127 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2128 break;
2129 default:
2130 assert(false, "bad register type for spill");
2131 ShouldNotReachHere();
2132 }
2133 st->print("\t# vector spill size = %d", vsize);
2134 } else if (ideal_reg() == Op_RegVectMask) {
2135 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2136 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2137 st->print("\t# predicate spill size = %d", vsize);
2138 } else {
2139 st->print("\t# spill size = %d", is64 ? 64 : 32);
2140 }
2141 }
2142
2143 return 0;
2144
2145 }
2146
2147 #ifndef PRODUCT
2148 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2149 if (!ra_)
2150 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2151 else
2152 implementation(nullptr, ra_, false, st);
2153 }
2154 #endif
2155
2156 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2157 implementation(masm, ra_, false, nullptr);
2158 }
2159
2160 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2161 return MachNode::size(ra_);
2162 }
2163
2164 //=============================================================================
2165
2166 #ifndef PRODUCT
2167 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2168 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2169 int reg = ra_->get_reg_first(this);
2170 st->print("add %s, rsp, #%d]\t# box lock",
2171 Matcher::regName[reg], offset);
2172 }
2173 #endif
2174
2175 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2176 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2177 int reg = ra_->get_encode(this);
2178
2179 // This add will handle any 24-bit signed offset. 24 bits allows an
2180 // 8 megabyte stack frame.
2181 __ add(as_Register(reg), sp, offset);
2182 }
2183
2184 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2185 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2186 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2187
2188 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2189 return NativeInstruction::instruction_size;
2190 } else {
2191 return 2 * NativeInstruction::instruction_size;
2192 }
2193 }
2194
2195 //=============================================================================
2196
2197 #ifndef PRODUCT
2198 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2199 {
2200 st->print_cr("# MachUEPNode");
2201 if (UseCompressedClassPointers) {
2202 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2203 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2204 st->print_cr("\tcmpw rscratch1, r10");
2205 } else {
2206 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2207 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2208 st->print_cr("\tcmp rscratch1, r10");
2209 }
2210 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2211 }
2212 #endif
2213
2214 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2215 {
2216 __ ic_check(InteriorEntryAlignment);
2217 }
2218
2219 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2220 {
2221 return MachNode::size(ra_);
2222 }
2223
2224 // REQUIRED EMIT CODE
2225
2226 //=============================================================================
2227
2228 // Emit exception handler code.
2229 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2230 {
2231 // mov rscratch1 #exception_blob_entry_point
2232 // br rscratch1
2233 // Note that the code buffer's insts_mark is always relative to insts.
2234 // That's why we must use the macroassembler to generate a handler.
2235 address base = __ start_a_stub(size_exception_handler());
2236 if (base == nullptr) {
2237 ciEnv::current()->record_failure("CodeCache is full");
2238 return 0; // CodeBuffer::expand failed
2239 }
2240 int offset = __ offset();
2241 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2242 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2243 __ end_a_stub();
2244 return offset;
2245 }
2246
2247 // Emit deopt handler code.
2248 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2249 {
2250 // Note that the code buffer's insts_mark is always relative to insts.
2251 // That's why we must use the macroassembler to generate a handler.
2252 address base = __ start_a_stub(size_deopt_handler());
2253 if (base == nullptr) {
2254 ciEnv::current()->record_failure("CodeCache is full");
2255 return 0; // CodeBuffer::expand failed
2256 }
2257 int offset = __ offset();
2258
2259 __ adr(lr, __ pc());
2260 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2261
2262 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2263 __ end_a_stub();
2264 return offset;
2265 }
2266
2267 // REQUIRED MATCHER CODE
2268
2269 //=============================================================================
2270
2271 bool Matcher::match_rule_supported(int opcode) {
2272 if (!has_match_rule(opcode))
2273 return false;
2274
2275 switch (opcode) {
2276 case Op_OnSpinWait:
2277 return VM_Version::supports_on_spin_wait();
2278 case Op_CacheWB:
2279 case Op_CacheWBPreSync:
2280 case Op_CacheWBPostSync:
2281 if (!VM_Version::supports_data_cache_line_flush()) {
2282 return false;
2283 }
2284 break;
2285 case Op_ExpandBits:
2286 case Op_CompressBits:
2287 if (!VM_Version::supports_svebitperm()) {
2288 return false;
2289 }
2290 break;
2291 case Op_FmaF:
2292 case Op_FmaD:
2293 case Op_FmaVF:
2294 case Op_FmaVD:
2295 if (!UseFMA) {
2296 return false;
2297 }
2298 break;
2299 }
2300
2301 return true; // Per default match rules are supported.
2302 }
2303
2304 const RegMask* Matcher::predicate_reg_mask(void) {
2305 return &_PR_REG_mask;
2306 }
2307
2308 bool Matcher::supports_vector_calling_convention(void) {
2309 return EnableVectorSupport && UseVectorStubs;
2310 }
2311
2312 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2313 assert(EnableVectorSupport && UseVectorStubs, "sanity");
2314 int lo = V0_num;
2315 int hi = V0_H_num;
2316 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2317 hi = V0_K_num;
2318 }
2319 return OptoRegPair(hi, lo);
2320 }
2321
2322 // Is this branch offset short enough that a short branch can be used?
2323 //
2324 // NOTE: If the platform does not provide any short branch variants, then
2325 // this method should return false for offset 0.
2326 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2327 // The passed offset is relative to address of the branch.
2328
2329 return (-32768 <= offset && offset < 32768);
2330 }
2331
2332 // Vector width in bytes.
2333 int Matcher::vector_width_in_bytes(BasicType bt) {
2334 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2335 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2336 // Minimum 2 values in vector
2337 if (size < 2*type2aelembytes(bt)) size = 0;
2338 // But never < 4
2339 if (size < 4) size = 0;
2340 return size;
2341 }
2342
2343 // Limits on vector size (number of elements) loaded into vector.
2344 int Matcher::max_vector_size(const BasicType bt) {
2345 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2346 }
2347
2348 int Matcher::min_vector_size(const BasicType bt) {
2349 int max_size = max_vector_size(bt);
2350 // Limit the min vector size to 8 bytes.
2351 int size = 8 / type2aelembytes(bt);
2352 if (bt == T_BYTE) {
2353 // To support vector api shuffle/rearrange.
2354 size = 4;
2355 } else if (bt == T_BOOLEAN) {
2356 // To support vector api load/store mask.
2357 size = 2;
2358 }
2359 if (size < 2) size = 2;
2360 return MIN2(size, max_size);
2361 }
2362
2363 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2364 return Matcher::max_vector_size(bt);
2365 }
2366
2367 // Actual max scalable vector register length.
2368 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2369 return Matcher::max_vector_size(bt);
2370 }
2371
2372 // Vector ideal reg.
2373 uint Matcher::vector_ideal_reg(int len) {
2374 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2375 return Op_VecA;
2376 }
2377 switch(len) {
2378 // For 16-bit/32-bit mask vector, reuse VecD.
2379 case 2:
2380 case 4:
2381 case 8: return Op_VecD;
2382 case 16: return Op_VecX;
2383 }
2384 ShouldNotReachHere();
2385 return 0;
2386 }
2387
2388 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2389 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2390 switch (ideal_reg) {
2391 case Op_VecA: return new vecAOper();
2392 case Op_VecD: return new vecDOper();
2393 case Op_VecX: return new vecXOper();
2394 }
2395 ShouldNotReachHere();
2396 return nullptr;
2397 }
2398
2399 bool Matcher::is_reg2reg_move(MachNode* m) {
2400 return false;
2401 }
2402
2403 bool Matcher::is_generic_vector(MachOper* opnd) {
2404 return opnd->opcode() == VREG;
2405 }
2406
2407 // Return whether or not this register is ever used as an argument.
2408 // This function is used on startup to build the trampoline stubs in
2409 // generateOptoStub. Registers not mentioned will be killed by the VM
2410 // call in the trampoline, and arguments in those registers not be
2411 // available to the callee.
2412 bool Matcher::can_be_java_arg(int reg)
2413 {
2414 return
2415 reg == R0_num || reg == R0_H_num ||
2416 reg == R1_num || reg == R1_H_num ||
2417 reg == R2_num || reg == R2_H_num ||
2418 reg == R3_num || reg == R3_H_num ||
2419 reg == R4_num || reg == R4_H_num ||
2420 reg == R5_num || reg == R5_H_num ||
2421 reg == R6_num || reg == R6_H_num ||
2422 reg == R7_num || reg == R7_H_num ||
2423 reg == V0_num || reg == V0_H_num ||
2424 reg == V1_num || reg == V1_H_num ||
2425 reg == V2_num || reg == V2_H_num ||
2426 reg == V3_num || reg == V3_H_num ||
2427 reg == V4_num || reg == V4_H_num ||
2428 reg == V5_num || reg == V5_H_num ||
2429 reg == V6_num || reg == V6_H_num ||
2430 reg == V7_num || reg == V7_H_num;
2431 }
2432
2433 bool Matcher::is_spillable_arg(int reg)
2434 {
2435 return can_be_java_arg(reg);
2436 }
2437
2438 uint Matcher::int_pressure_limit()
2439 {
2440 // JDK-8183543: When taking the number of available registers as int
2441 // register pressure threshold, the jtreg test:
2442 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2443 // failed due to C2 compilation failure with
2444 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2445 //
2446 // A derived pointer is live at CallNode and then is flagged by RA
2447 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2448 // derived pointers and lastly fail to spill after reaching maximum
2449 // number of iterations. Lowering the default pressure threshold to
2450 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2451 // a high register pressure area of the code so that split_DEF can
2452 // generate DefinitionSpillCopy for the derived pointer.
2453 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2454 if (!PreserveFramePointer) {
2455 // When PreserveFramePointer is off, frame pointer is allocatable,
2456 // but different from other SOC registers, it is excluded from
2457 // fatproj's mask because its save type is No-Save. Decrease 1 to
2458 // ensure high pressure at fatproj when PreserveFramePointer is off.
2459 // See check_pressure_at_fatproj().
2460 default_int_pressure_threshold--;
2461 }
2462 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2463 }
2464
2465 uint Matcher::float_pressure_limit()
2466 {
2467 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2468 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2469 }
2470
2471 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2472 return false;
2473 }
2474
2475 RegMask Matcher::divI_proj_mask() {
2476 ShouldNotReachHere();
2477 return RegMask();
2478 }
2479
2480 // Register for MODI projection of divmodI.
2481 RegMask Matcher::modI_proj_mask() {
2482 ShouldNotReachHere();
2483 return RegMask();
2484 }
2485
2486 // Register for DIVL projection of divmodL.
2487 RegMask Matcher::divL_proj_mask() {
2488 ShouldNotReachHere();
2489 return RegMask();
2490 }
2491
2492 // Register for MODL projection of divmodL.
2493 RegMask Matcher::modL_proj_mask() {
2494 ShouldNotReachHere();
2495 return RegMask();
2496 }
2497
2498 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2499 return FP_REG_mask();
2500 }
2501
2502 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2503 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2504 Node* u = addp->fast_out(i);
2505 if (u->is_LoadStore()) {
2506 // On AArch64, LoadStoreNodes (i.e. compare and swap
2507 // instructions) only take register indirect as an operand, so
2508 // any attempt to use an AddPNode as an input to a LoadStoreNode
2509 // must fail.
2510 return false;
2511 }
2512 if (u->is_Mem()) {
2513 int opsize = u->as_Mem()->memory_size();
2514 assert(opsize > 0, "unexpected memory operand size");
2515 if (u->as_Mem()->memory_size() != (1<<shift)) {
2516 return false;
2517 }
2518 }
2519 }
2520 return true;
2521 }
2522
2523 // Convert BootTest condition to Assembler condition.
2524 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2525 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2526 Assembler::Condition result;
2527 switch(cond) {
2528 case BoolTest::eq:
2529 result = Assembler::EQ; break;
2530 case BoolTest::ne:
2531 result = Assembler::NE; break;
2532 case BoolTest::le:
2533 result = Assembler::LE; break;
2534 case BoolTest::ge:
2535 result = Assembler::GE; break;
2536 case BoolTest::lt:
2537 result = Assembler::LT; break;
2538 case BoolTest::gt:
2539 result = Assembler::GT; break;
2540 case BoolTest::ule:
2541 result = Assembler::LS; break;
2542 case BoolTest::uge:
2543 result = Assembler::HS; break;
2544 case BoolTest::ult:
2545 result = Assembler::LO; break;
2546 case BoolTest::ugt:
2547 result = Assembler::HI; break;
2548 case BoolTest::overflow:
2549 result = Assembler::VS; break;
2550 case BoolTest::no_overflow:
2551 result = Assembler::VC; break;
2552 default:
2553 ShouldNotReachHere();
2554 return Assembler::Condition(-1);
2555 }
2556
2557 // Check conversion
2558 if (cond & BoolTest::unsigned_compare) {
2559 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2560 } else {
2561 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2562 }
2563
2564 return result;
2565 }
2566
2567 // Binary src (Replicate con)
2568 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2569 if (n == nullptr || m == nullptr) {
2570 return false;
2571 }
2572
2573 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2574 return false;
2575 }
2576
2577 Node* imm_node = m->in(1);
2578 if (!imm_node->is_Con()) {
2579 return false;
2580 }
2581
2582 const Type* t = imm_node->bottom_type();
2583 if (!(t->isa_int() || t->isa_long())) {
2584 return false;
2585 }
2586
2587 switch (n->Opcode()) {
2588 case Op_AndV:
2589 case Op_OrV:
2590 case Op_XorV: {
2591 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2592 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2593 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2594 }
2595 case Op_AddVB:
2596 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2597 case Op_AddVS:
2598 case Op_AddVI:
2599 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2600 case Op_AddVL:
2601 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2602 default:
2603 return false;
2604 }
2605 }
2606
2607 // (XorV src (Replicate m1))
2608 // (XorVMask src (MaskAll m1))
2609 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2610 if (n != nullptr && m != nullptr) {
2611 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2612 VectorNode::is_all_ones_vector(m);
2613 }
2614 return false;
2615 }
2616
2617 // Should the matcher clone input 'm' of node 'n'?
2618 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2619 if (is_vshift_con_pattern(n, m) ||
2620 is_vector_bitwise_not_pattern(n, m) ||
2621 is_valid_sve_arith_imm_pattern(n, m) ||
2622 is_encode_and_store_pattern(n, m)) {
2623 mstack.push(m, Visit);
2624 return true;
2625 }
2626 return false;
2627 }
2628
2629 // Should the Matcher clone shifts on addressing modes, expecting them
2630 // to be subsumed into complex addressing expressions or compute them
2631 // into registers?
2632 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2633
2634 // Loads and stores with indirect memory input (e.g., volatile loads and
2635 // stores) do not subsume the input into complex addressing expressions. If
2636 // the addressing expression is input to at least one such load or store, do
2637 // not clone the addressing expression. Query needs_acquiring_load and
2638 // needs_releasing_store as a proxy for indirect memory input, as it is not
2639 // possible to directly query for indirect memory input at this stage.
2640 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2641 Node* n = m->fast_out(i);
2642 if (n->is_Load() && needs_acquiring_load(n)) {
2643 return false;
2644 }
2645 if (n->is_Store() && needs_releasing_store(n)) {
2646 return false;
2647 }
2648 }
2649
2650 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2651 return true;
2652 }
2653
2654 Node *off = m->in(AddPNode::Offset);
2655 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2656 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2657 // Are there other uses besides address expressions?
2658 !is_visited(off)) {
2659 address_visited.set(off->_idx); // Flag as address_visited
2660 mstack.push(off->in(2), Visit);
2661 Node *conv = off->in(1);
2662 if (conv->Opcode() == Op_ConvI2L &&
2663 // Are there other uses besides address expressions?
2664 !is_visited(conv)) {
2665 address_visited.set(conv->_idx); // Flag as address_visited
2666 mstack.push(conv->in(1), Pre_Visit);
2667 } else {
2668 mstack.push(conv, Pre_Visit);
2669 }
2670 address_visited.test_set(m->_idx); // Flag as address_visited
2671 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2672 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2673 return true;
2674 } else if (off->Opcode() == Op_ConvI2L &&
2675 // Are there other uses besides address expressions?
2676 !is_visited(off)) {
2677 address_visited.test_set(m->_idx); // Flag as address_visited
2678 address_visited.set(off->_idx); // Flag as address_visited
2679 mstack.push(off->in(1), Pre_Visit);
2680 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2681 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2682 return true;
2683 }
2684 return false;
2685 }
2686
2687 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2688 { \
2689 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2690 guarantee(DISP == 0, "mode not permitted for volatile"); \
2691 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2692 __ INSN(REG, as_Register(BASE)); \
2693 }
2694
2695
2696 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2697 {
2698 Address::extend scale;
2699
2700 // Hooboy, this is fugly. We need a way to communicate to the
2701 // encoder that the index needs to be sign extended, so we have to
2702 // enumerate all the cases.
2703 switch (opcode) {
2704 case INDINDEXSCALEDI2L:
2705 case INDINDEXSCALEDI2LN:
2706 case INDINDEXI2L:
2707 case INDINDEXI2LN:
2708 scale = Address::sxtw(size);
2709 break;
2710 default:
2711 scale = Address::lsl(size);
2712 }
2713
2714 if (index == -1) {
2715 return Address(base, disp);
2716 } else {
2717 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2718 return Address(base, as_Register(index), scale);
2719 }
2720 }
2721
2722
2723 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2724 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2725 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2726 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2727 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2728
2729 // Used for all non-volatile memory accesses. The use of
2730 // $mem->opcode() to discover whether this pattern uses sign-extended
2731 // offsets is something of a kludge.
2732 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2733 Register reg, int opcode,
2734 Register base, int index, int scale, int disp,
2735 int size_in_memory)
2736 {
2737 Address addr = mem2address(opcode, base, index, scale, disp);
2738 if (addr.getMode() == Address::base_plus_offset) {
2739 /* Fix up any out-of-range offsets. */
2740 assert_different_registers(rscratch1, base);
2741 assert_different_registers(rscratch1, reg);
2742 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2743 }
2744 (masm->*insn)(reg, addr);
2745 }
2746
2747 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2748 FloatRegister reg, int opcode,
2749 Register base, int index, int size, int disp,
2750 int size_in_memory)
2751 {
2752 Address::extend scale;
2753
2754 switch (opcode) {
2755 case INDINDEXSCALEDI2L:
2756 case INDINDEXSCALEDI2LN:
2757 scale = Address::sxtw(size);
2758 break;
2759 default:
2760 scale = Address::lsl(size);
2761 }
2762
2763 if (index == -1) {
2764 // Fix up any out-of-range offsets.
2765 assert_different_registers(rscratch1, base);
2766 Address addr = Address(base, disp);
2767 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2768 (masm->*insn)(reg, addr);
2769 } else {
2770 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2771 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2772 }
2773 }
2774
2775 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2776 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2777 int opcode, Register base, int index, int size, int disp)
2778 {
2779 if (index == -1) {
2780 (masm->*insn)(reg, T, Address(base, disp));
2781 } else {
2782 assert(disp == 0, "unsupported address mode");
2783 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2784 }
2785 }
2786
2787 %}
2788
2789
2790
2791 //----------ENCODING BLOCK-----------------------------------------------------
2792 // This block specifies the encoding classes used by the compiler to
2793 // output byte streams. Encoding classes are parameterized macros
2794 // used by Machine Instruction Nodes in order to generate the bit
2795 // encoding of the instruction. Operands specify their base encoding
2796 // interface with the interface keyword. There are currently
2797 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2798 // COND_INTER. REG_INTER causes an operand to generate a function
2799 // which returns its register number when queried. CONST_INTER causes
2800 // an operand to generate a function which returns the value of the
2801 // constant when queried. MEMORY_INTER causes an operand to generate
2802 // four functions which return the Base Register, the Index Register,
2803 // the Scale Value, and the Offset Value of the operand when queried.
2804 // COND_INTER causes an operand to generate six functions which return
2805 // the encoding code (ie - encoding bits for the instruction)
2806 // associated with each basic boolean condition for a conditional
2807 // instruction.
2808 //
2809 // Instructions specify two basic values for encoding. Again, a
2810 // function is available to check if the constant displacement is an
2811 // oop. They use the ins_encode keyword to specify their encoding
2812 // classes (which must be a sequence of enc_class names, and their
2813 // parameters, specified in the encoding block), and they use the
2814 // opcode keyword to specify, in order, their primary, secondary, and
2815 // tertiary opcode. Only the opcode sections which a particular
2816 // instruction needs for encoding need to be specified.
2817 encode %{
2818 // Build emit functions for each basic byte or larger field in the
2819 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2820 // from C++ code in the enc_class source block. Emit functions will
2821 // live in the main source block for now. In future, we can
2822 // generalize this by adding a syntax that specifies the sizes of
2823 // fields in an order, so that the adlc can build the emit functions
2824 // automagically
2825
2826 // catch all for unimplemented encodings
2827 enc_class enc_unimplemented %{
2828 __ unimplemented("C2 catch all");
2829 %}
2830
2831 // BEGIN Non-volatile memory access
2832
2833 // This encoding class is generated automatically from ad_encode.m4.
2834 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2835 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2836 Register dst_reg = as_Register($dst$$reg);
2837 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2838 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2839 %}
2840
2841 // This encoding class is generated automatically from ad_encode.m4.
2842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2843 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2844 Register dst_reg = as_Register($dst$$reg);
2845 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2846 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2847 %}
2848
2849 // This encoding class is generated automatically from ad_encode.m4.
2850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2851 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2852 Register dst_reg = as_Register($dst$$reg);
2853 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2854 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2855 %}
2856
2857 // This encoding class is generated automatically from ad_encode.m4.
2858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2859 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2860 Register dst_reg = as_Register($dst$$reg);
2861 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2862 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2863 %}
2864
2865 // This encoding class is generated automatically from ad_encode.m4.
2866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2867 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2868 Register dst_reg = as_Register($dst$$reg);
2869 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2870 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2871 %}
2872
2873 // This encoding class is generated automatically from ad_encode.m4.
2874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2875 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2876 Register dst_reg = as_Register($dst$$reg);
2877 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2878 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2879 %}
2880
2881 // This encoding class is generated automatically from ad_encode.m4.
2882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2883 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2884 Register dst_reg = as_Register($dst$$reg);
2885 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2886 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2887 %}
2888
2889 // This encoding class is generated automatically from ad_encode.m4.
2890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2891 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2892 Register dst_reg = as_Register($dst$$reg);
2893 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2894 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2895 %}
2896
2897 // This encoding class is generated automatically from ad_encode.m4.
2898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2899 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2900 Register dst_reg = as_Register($dst$$reg);
2901 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2902 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2903 %}
2904
2905 // This encoding class is generated automatically from ad_encode.m4.
2906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2907 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2908 Register dst_reg = as_Register($dst$$reg);
2909 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2910 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2911 %}
2912
2913 // This encoding class is generated automatically from ad_encode.m4.
2914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2915 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2916 Register dst_reg = as_Register($dst$$reg);
2917 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2918 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2919 %}
2920
2921 // This encoding class is generated automatically from ad_encode.m4.
2922 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2923 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2924 Register dst_reg = as_Register($dst$$reg);
2925 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2926 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2927 %}
2928
2929 // This encoding class is generated automatically from ad_encode.m4.
2930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2931 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2932 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2933 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2934 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2935 %}
2936
2937 // This encoding class is generated automatically from ad_encode.m4.
2938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2939 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2940 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2941 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2942 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2943 %}
2944
2945 // This encoding class is generated automatically from ad_encode.m4.
2946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2947 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2948 Register src_reg = as_Register($src$$reg);
2949 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2950 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2951 %}
2952
2953 // This encoding class is generated automatically from ad_encode.m4.
2954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2955 enc_class aarch64_enc_strb0(memory1 mem) %{
2956 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2957 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2958 %}
2959
2960 // This encoding class is generated automatically from ad_encode.m4.
2961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2962 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2963 Register src_reg = as_Register($src$$reg);
2964 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2965 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2966 %}
2967
2968 // This encoding class is generated automatically from ad_encode.m4.
2969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2970 enc_class aarch64_enc_strh0(memory2 mem) %{
2971 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2972 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2973 %}
2974
2975 // This encoding class is generated automatically from ad_encode.m4.
2976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2977 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2978 Register src_reg = as_Register($src$$reg);
2979 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
2980 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2981 %}
2982
2983 // This encoding class is generated automatically from ad_encode.m4.
2984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2985 enc_class aarch64_enc_strw0(memory4 mem) %{
2986 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
2987 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2988 %}
2989
2990 // This encoding class is generated automatically from ad_encode.m4.
2991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2992 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2993 Register src_reg = as_Register($src$$reg);
2994 // we sometimes get asked to store the stack pointer into the
2995 // current thread -- we cannot do that directly on AArch64
2996 if (src_reg == r31_sp) {
2997 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2998 __ mov(rscratch2, sp);
2999 src_reg = rscratch2;
3000 }
3001 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3002 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3003 %}
3004
3005 // This encoding class is generated automatically from ad_encode.m4.
3006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3007 enc_class aarch64_enc_str0(memory8 mem) %{
3008 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3009 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3010 %}
3011
3012 // This encoding class is generated automatically from ad_encode.m4.
3013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3014 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3015 FloatRegister src_reg = as_FloatRegister($src$$reg);
3016 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3017 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3018 %}
3019
3020 // This encoding class is generated automatically from ad_encode.m4.
3021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3022 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3023 FloatRegister src_reg = as_FloatRegister($src$$reg);
3024 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3025 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3026 %}
3027
3028 // This encoding class is generated automatically from ad_encode.m4.
3029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3030 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3031 __ membar(Assembler::StoreStore);
3032 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3033 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3034 %}
3035
3036 // END Non-volatile memory access
3037
3038 // Vector loads and stores
3039 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3040 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3041 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3042 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3043 %}
3044
3045 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3046 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3047 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3048 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3049 %}
3050
3051 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3052 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3053 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3054 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3055 %}
3056
3057 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3058 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3059 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3060 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3061 %}
3062
3063 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3064 FloatRegister src_reg = as_FloatRegister($src$$reg);
3065 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3066 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3067 %}
3068
3069 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3070 FloatRegister src_reg = as_FloatRegister($src$$reg);
3071 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3072 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3073 %}
3074
3075 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3076 FloatRegister src_reg = as_FloatRegister($src$$reg);
3077 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3078 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3079 %}
3080
3081 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3082 FloatRegister src_reg = as_FloatRegister($src$$reg);
3083 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3084 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3085 %}
3086
3087 // volatile loads and stores
3088
3089 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3090 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3091 rscratch1, stlrb);
3092 %}
3093
3094 enc_class aarch64_enc_stlrb0(memory mem) %{
3095 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3096 rscratch1, stlrb);
3097 %}
3098
3099 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3100 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3101 rscratch1, stlrh);
3102 %}
3103
3104 enc_class aarch64_enc_stlrh0(memory mem) %{
3105 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3106 rscratch1, stlrh);
3107 %}
3108
3109 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3110 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3111 rscratch1, stlrw);
3112 %}
3113
3114 enc_class aarch64_enc_stlrw0(memory mem) %{
3115 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3116 rscratch1, stlrw);
3117 %}
3118
3119 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3120 Register dst_reg = as_Register($dst$$reg);
3121 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3122 rscratch1, ldarb);
3123 __ sxtbw(dst_reg, dst_reg);
3124 %}
3125
3126 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3127 Register dst_reg = as_Register($dst$$reg);
3128 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3129 rscratch1, ldarb);
3130 __ sxtb(dst_reg, dst_reg);
3131 %}
3132
3133 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3134 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3135 rscratch1, ldarb);
3136 %}
3137
3138 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3139 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3140 rscratch1, ldarb);
3141 %}
3142
3143 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3144 Register dst_reg = as_Register($dst$$reg);
3145 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3146 rscratch1, ldarh);
3147 __ sxthw(dst_reg, dst_reg);
3148 %}
3149
3150 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3151 Register dst_reg = as_Register($dst$$reg);
3152 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3153 rscratch1, ldarh);
3154 __ sxth(dst_reg, dst_reg);
3155 %}
3156
3157 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3158 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3159 rscratch1, ldarh);
3160 %}
3161
3162 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3163 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3164 rscratch1, ldarh);
3165 %}
3166
3167 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3168 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3169 rscratch1, ldarw);
3170 %}
3171
3172 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3173 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3174 rscratch1, ldarw);
3175 %}
3176
3177 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3178 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3179 rscratch1, ldar);
3180 %}
3181
3182 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3183 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3184 rscratch1, ldarw);
3185 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3186 %}
3187
3188 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3189 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3190 rscratch1, ldar);
3191 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3192 %}
3193
3194 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3195 Register src_reg = as_Register($src$$reg);
3196 // we sometimes get asked to store the stack pointer into the
3197 // current thread -- we cannot do that directly on AArch64
3198 if (src_reg == r31_sp) {
3199 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3200 __ mov(rscratch2, sp);
3201 src_reg = rscratch2;
3202 }
3203 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3204 rscratch1, stlr);
3205 %}
3206
3207 enc_class aarch64_enc_stlr0(memory mem) %{
3208 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3209 rscratch1, stlr);
3210 %}
3211
3212 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3213 {
3214 FloatRegister src_reg = as_FloatRegister($src$$reg);
3215 __ fmovs(rscratch2, src_reg);
3216 }
3217 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3218 rscratch1, stlrw);
3219 %}
3220
3221 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3222 {
3223 FloatRegister src_reg = as_FloatRegister($src$$reg);
3224 __ fmovd(rscratch2, src_reg);
3225 }
3226 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3227 rscratch1, stlr);
3228 %}
3229
3230 // synchronized read/update encodings
3231
3232 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3233 Register dst_reg = as_Register($dst$$reg);
3234 Register base = as_Register($mem$$base);
3235 int index = $mem$$index;
3236 int scale = $mem$$scale;
3237 int disp = $mem$$disp;
3238 if (index == -1) {
3239 if (disp != 0) {
3240 __ lea(rscratch1, Address(base, disp));
3241 __ ldaxr(dst_reg, rscratch1);
3242 } else {
3243 // TODO
3244 // should we ever get anything other than this case?
3245 __ ldaxr(dst_reg, base);
3246 }
3247 } else {
3248 Register index_reg = as_Register(index);
3249 if (disp == 0) {
3250 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3251 __ ldaxr(dst_reg, rscratch1);
3252 } else {
3253 __ lea(rscratch1, Address(base, disp));
3254 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3255 __ ldaxr(dst_reg, rscratch1);
3256 }
3257 }
3258 %}
3259
3260 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3261 Register src_reg = as_Register($src$$reg);
3262 Register base = as_Register($mem$$base);
3263 int index = $mem$$index;
3264 int scale = $mem$$scale;
3265 int disp = $mem$$disp;
3266 if (index == -1) {
3267 if (disp != 0) {
3268 __ lea(rscratch2, Address(base, disp));
3269 __ stlxr(rscratch1, src_reg, rscratch2);
3270 } else {
3271 // TODO
3272 // should we ever get anything other than this case?
3273 __ stlxr(rscratch1, src_reg, base);
3274 }
3275 } else {
3276 Register index_reg = as_Register(index);
3277 if (disp == 0) {
3278 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3279 __ stlxr(rscratch1, src_reg, rscratch2);
3280 } else {
3281 __ lea(rscratch2, Address(base, disp));
3282 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3283 __ stlxr(rscratch1, src_reg, rscratch2);
3284 }
3285 }
3286 __ cmpw(rscratch1, zr);
3287 %}
3288
3289 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3290 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3291 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3292 Assembler::xword, /*acquire*/ false, /*release*/ true,
3293 /*weak*/ false, noreg);
3294 %}
3295
3296 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3297 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3298 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3299 Assembler::word, /*acquire*/ false, /*release*/ true,
3300 /*weak*/ false, noreg);
3301 %}
3302
3303 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3304 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3305 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3306 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3307 /*weak*/ false, noreg);
3308 %}
3309
3310 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3311 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3312 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3313 Assembler::byte, /*acquire*/ false, /*release*/ true,
3314 /*weak*/ false, noreg);
3315 %}
3316
3317
3318 // The only difference between aarch64_enc_cmpxchg and
3319 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3320 // CompareAndSwap sequence to serve as a barrier on acquiring a
3321 // lock.
3322 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3323 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3324 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3325 Assembler::xword, /*acquire*/ true, /*release*/ true,
3326 /*weak*/ false, noreg);
3327 %}
3328
3329 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3330 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3331 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3332 Assembler::word, /*acquire*/ true, /*release*/ true,
3333 /*weak*/ false, noreg);
3334 %}
3335
3336 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3337 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3338 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3339 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3340 /*weak*/ false, noreg);
3341 %}
3342
3343 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3344 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3345 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3346 Assembler::byte, /*acquire*/ true, /*release*/ true,
3347 /*weak*/ false, noreg);
3348 %}
3349
3350 // auxiliary used for CompareAndSwapX to set result register
3351 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3352 Register res_reg = as_Register($res$$reg);
3353 __ cset(res_reg, Assembler::EQ);
3354 %}
3355
3356 // prefetch encodings
3357
3358 enc_class aarch64_enc_prefetchw(memory mem) %{
3359 Register base = as_Register($mem$$base);
3360 int index = $mem$$index;
3361 int scale = $mem$$scale;
3362 int disp = $mem$$disp;
3363 if (index == -1) {
3364 // Fix up any out-of-range offsets.
3365 assert_different_registers(rscratch1, base);
3366 Address addr = Address(base, disp);
3367 addr = __ legitimize_address(addr, 8, rscratch1);
3368 __ prfm(addr, PSTL1KEEP);
3369 } else {
3370 Register index_reg = as_Register(index);
3371 if (disp == 0) {
3372 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3373 } else {
3374 __ lea(rscratch1, Address(base, disp));
3375 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3376 }
3377 }
3378 %}
3379
3380 // mov encodings
3381
3382 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3383 uint32_t con = (uint32_t)$src$$constant;
3384 Register dst_reg = as_Register($dst$$reg);
3385 if (con == 0) {
3386 __ movw(dst_reg, zr);
3387 } else {
3388 __ movw(dst_reg, con);
3389 }
3390 %}
3391
3392 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3393 Register dst_reg = as_Register($dst$$reg);
3394 uint64_t con = (uint64_t)$src$$constant;
3395 if (con == 0) {
3396 __ mov(dst_reg, zr);
3397 } else {
3398 __ mov(dst_reg, con);
3399 }
3400 %}
3401
3402 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3403 Register dst_reg = as_Register($dst$$reg);
3404 address con = (address)$src$$constant;
3405 if (con == nullptr || con == (address)1) {
3406 ShouldNotReachHere();
3407 } else {
3408 relocInfo::relocType rtype = $src->constant_reloc();
3409 if (rtype == relocInfo::oop_type) {
3410 __ movoop(dst_reg, (jobject)con);
3411 } else if (rtype == relocInfo::metadata_type) {
3412 __ mov_metadata(dst_reg, (Metadata*)con);
3413 } else {
3414 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3415 if (! __ is_valid_AArch64_address(con) ||
3416 con < (address)(uintptr_t)os::vm_page_size()) {
3417 __ mov(dst_reg, con);
3418 } else {
3419 uint64_t offset;
3420 __ adrp(dst_reg, con, offset);
3421 __ add(dst_reg, dst_reg, offset);
3422 }
3423 }
3424 }
3425 %}
3426
3427 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3428 Register dst_reg = as_Register($dst$$reg);
3429 __ mov(dst_reg, zr);
3430 %}
3431
3432 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3433 Register dst_reg = as_Register($dst$$reg);
3434 __ mov(dst_reg, (uint64_t)1);
3435 %}
3436
3437 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3438 Register dst_reg = as_Register($dst$$reg);
3439 address con = (address)$src$$constant;
3440 if (con == nullptr) {
3441 ShouldNotReachHere();
3442 } else {
3443 relocInfo::relocType rtype = $src->constant_reloc();
3444 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3445 __ set_narrow_oop(dst_reg, (jobject)con);
3446 }
3447 %}
3448
3449 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3450 Register dst_reg = as_Register($dst$$reg);
3451 __ mov(dst_reg, zr);
3452 %}
3453
3454 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3455 Register dst_reg = as_Register($dst$$reg);
3456 address con = (address)$src$$constant;
3457 if (con == nullptr) {
3458 ShouldNotReachHere();
3459 } else {
3460 relocInfo::relocType rtype = $src->constant_reloc();
3461 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3462 __ set_narrow_klass(dst_reg, (Klass *)con);
3463 }
3464 %}
3465
3466 // arithmetic encodings
3467
3468 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3469 Register dst_reg = as_Register($dst$$reg);
3470 Register src_reg = as_Register($src1$$reg);
3471 int32_t con = (int32_t)$src2$$constant;
3472 // add has primary == 0, subtract has primary == 1
3473 if ($primary) { con = -con; }
3474 if (con < 0) {
3475 __ subw(dst_reg, src_reg, -con);
3476 } else {
3477 __ addw(dst_reg, src_reg, con);
3478 }
3479 %}
3480
3481 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3482 Register dst_reg = as_Register($dst$$reg);
3483 Register src_reg = as_Register($src1$$reg);
3484 int32_t con = (int32_t)$src2$$constant;
3485 // add has primary == 0, subtract has primary == 1
3486 if ($primary) { con = -con; }
3487 if (con < 0) {
3488 __ sub(dst_reg, src_reg, -con);
3489 } else {
3490 __ add(dst_reg, src_reg, con);
3491 }
3492 %}
3493
3494 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3495 Register dst_reg = as_Register($dst$$reg);
3496 Register src1_reg = as_Register($src1$$reg);
3497 Register src2_reg = as_Register($src2$$reg);
3498 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3499 %}
3500
3501 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3502 Register dst_reg = as_Register($dst$$reg);
3503 Register src1_reg = as_Register($src1$$reg);
3504 Register src2_reg = as_Register($src2$$reg);
3505 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3506 %}
3507
3508 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3509 Register dst_reg = as_Register($dst$$reg);
3510 Register src1_reg = as_Register($src1$$reg);
3511 Register src2_reg = as_Register($src2$$reg);
3512 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3513 %}
3514
3515 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3516 Register dst_reg = as_Register($dst$$reg);
3517 Register src1_reg = as_Register($src1$$reg);
3518 Register src2_reg = as_Register($src2$$reg);
3519 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3520 %}
3521
3522 // compare instruction encodings
3523
3524 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3525 Register reg1 = as_Register($src1$$reg);
3526 Register reg2 = as_Register($src2$$reg);
3527 __ cmpw(reg1, reg2);
3528 %}
3529
3530 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3531 Register reg = as_Register($src1$$reg);
3532 int32_t val = $src2$$constant;
3533 if (val >= 0) {
3534 __ subsw(zr, reg, val);
3535 } else {
3536 __ addsw(zr, reg, -val);
3537 }
3538 %}
3539
3540 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3541 Register reg1 = as_Register($src1$$reg);
3542 uint32_t val = (uint32_t)$src2$$constant;
3543 __ movw(rscratch1, val);
3544 __ cmpw(reg1, rscratch1);
3545 %}
3546
3547 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3548 Register reg1 = as_Register($src1$$reg);
3549 Register reg2 = as_Register($src2$$reg);
3550 __ cmp(reg1, reg2);
3551 %}
3552
3553 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3554 Register reg = as_Register($src1$$reg);
3555 int64_t val = $src2$$constant;
3556 if (val >= 0) {
3557 __ subs(zr, reg, val);
3558 } else if (val != -val) {
3559 __ adds(zr, reg, -val);
3560 } else {
3561 // aargh, Long.MIN_VALUE is a special case
3562 __ orr(rscratch1, zr, (uint64_t)val);
3563 __ subs(zr, reg, rscratch1);
3564 }
3565 %}
3566
3567 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3568 Register reg1 = as_Register($src1$$reg);
3569 uint64_t val = (uint64_t)$src2$$constant;
3570 __ mov(rscratch1, val);
3571 __ cmp(reg1, rscratch1);
3572 %}
3573
3574 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3575 Register reg1 = as_Register($src1$$reg);
3576 Register reg2 = as_Register($src2$$reg);
3577 __ cmp(reg1, reg2);
3578 %}
3579
3580 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3581 Register reg1 = as_Register($src1$$reg);
3582 Register reg2 = as_Register($src2$$reg);
3583 __ cmpw(reg1, reg2);
3584 %}
3585
3586 enc_class aarch64_enc_testp(iRegP src) %{
3587 Register reg = as_Register($src$$reg);
3588 __ cmp(reg, zr);
3589 %}
3590
3591 enc_class aarch64_enc_testn(iRegN src) %{
3592 Register reg = as_Register($src$$reg);
3593 __ cmpw(reg, zr);
3594 %}
3595
3596 enc_class aarch64_enc_b(label lbl) %{
3597 Label *L = $lbl$$label;
3598 __ b(*L);
3599 %}
3600
3601 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3602 Label *L = $lbl$$label;
3603 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3604 %}
3605
3606 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3607 Label *L = $lbl$$label;
3608 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3609 %}
3610
3611 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3612 %{
3613 Register sub_reg = as_Register($sub$$reg);
3614 Register super_reg = as_Register($super$$reg);
3615 Register temp_reg = as_Register($temp$$reg);
3616 Register result_reg = as_Register($result$$reg);
3617
3618 Label miss;
3619 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3620 nullptr, &miss,
3621 /*set_cond_codes:*/ true);
3622 if ($primary) {
3623 __ mov(result_reg, zr);
3624 }
3625 __ bind(miss);
3626 %}
3627
3628 enc_class aarch64_enc_java_static_call(method meth) %{
3629 address addr = (address)$meth$$method;
3630 address call;
3631 if (!_method) {
3632 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3633 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3634 if (call == nullptr) {
3635 ciEnv::current()->record_failure("CodeCache is full");
3636 return;
3637 }
3638 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3639 // The NOP here is purely to ensure that eliding a call to
3640 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3641 __ nop();
3642 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3643 } else {
3644 int method_index = resolved_method_index(masm);
3645 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3646 : static_call_Relocation::spec(method_index);
3647 call = __ trampoline_call(Address(addr, rspec));
3648 if (call == nullptr) {
3649 ciEnv::current()->record_failure("CodeCache is full");
3650 return;
3651 }
3652 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3653 // Calls of the same statically bound method can share
3654 // a stub to the interpreter.
3655 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3656 } else {
3657 // Emit stub for static call
3658 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3659 if (stub == nullptr) {
3660 ciEnv::current()->record_failure("CodeCache is full");
3661 return;
3662 }
3663 }
3664 }
3665
3666 __ post_call_nop();
3667
3668 // Only non uncommon_trap calls need to reinitialize ptrue.
3669 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3670 __ reinitialize_ptrue();
3671 }
3672 %}
3673
3674 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3675 int method_index = resolved_method_index(masm);
3676 address call = __ ic_call((address)$meth$$method, method_index);
3677 if (call == nullptr) {
3678 ciEnv::current()->record_failure("CodeCache is full");
3679 return;
3680 }
3681 __ post_call_nop();
3682 if (Compile::current()->max_vector_size() > 0) {
3683 __ reinitialize_ptrue();
3684 }
3685 %}
3686
3687 enc_class aarch64_enc_call_epilog() %{
3688 if (VerifyStackAtCalls) {
3689 // Check that stack depth is unchanged: find majik cookie on stack
3690 __ call_Unimplemented();
3691 }
3692 %}
3693
3694 enc_class aarch64_enc_java_to_runtime(method meth) %{
3695 // some calls to generated routines (arraycopy code) are scheduled
3696 // by C2 as runtime calls. if so we can call them using a br (they
3697 // will be in a reachable segment) otherwise we have to use a blr
3698 // which loads the absolute address into a register.
3699 address entry = (address)$meth$$method;
3700 CodeBlob *cb = CodeCache::find_blob(entry);
3701 if (cb) {
3702 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3703 if (call == nullptr) {
3704 ciEnv::current()->record_failure("CodeCache is full");
3705 return;
3706 }
3707 __ post_call_nop();
3708 } else {
3709 Label retaddr;
3710 // Make the anchor frame walkable
3711 __ adr(rscratch2, retaddr);
3712 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3713 __ lea(rscratch1, RuntimeAddress(entry));
3714 __ blr(rscratch1);
3715 __ bind(retaddr);
3716 __ post_call_nop();
3717 }
3718 if (Compile::current()->max_vector_size() > 0) {
3719 __ reinitialize_ptrue();
3720 }
3721 %}
3722
3723 enc_class aarch64_enc_rethrow() %{
3724 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3725 %}
3726
3727 enc_class aarch64_enc_ret() %{
3728 #ifdef ASSERT
3729 if (Compile::current()->max_vector_size() > 0) {
3730 __ verify_ptrue();
3731 }
3732 #endif
3733 __ ret(lr);
3734 %}
3735
3736 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3737 Register target_reg = as_Register($jump_target$$reg);
3738 __ br(target_reg);
3739 %}
3740
3741 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3742 Register target_reg = as_Register($jump_target$$reg);
3743 // exception oop should be in r0
3744 // ret addr has been popped into lr
3745 // callee expects it in r3
3746 __ mov(r3, lr);
3747 __ br(target_reg);
3748 %}
3749
3750 %}
3751
3752 //----------FRAME--------------------------------------------------------------
3753 // Definition of frame structure and management information.
3754 //
3755 // S T A C K L A Y O U T Allocators stack-slot number
3756 // | (to get allocators register number
3757 // G Owned by | | v add OptoReg::stack0())
3758 // r CALLER | |
3759 // o | +--------+ pad to even-align allocators stack-slot
3760 // w V | pad0 | numbers; owned by CALLER
3761 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3762 // h ^ | in | 5
3763 // | | args | 4 Holes in incoming args owned by SELF
3764 // | | | | 3
3765 // | | +--------+
3766 // V | | old out| Empty on Intel, window on Sparc
3767 // | old |preserve| Must be even aligned.
3768 // | SP-+--------+----> Matcher::_old_SP, even aligned
3769 // | | in | 3 area for Intel ret address
3770 // Owned by |preserve| Empty on Sparc.
3771 // SELF +--------+
3772 // | | pad2 | 2 pad to align old SP
3773 // | +--------+ 1
3774 // | | locks | 0
3775 // | +--------+----> OptoReg::stack0(), even aligned
3776 // | | pad1 | 11 pad to align new SP
3777 // | +--------+
3778 // | | | 10
3779 // | | spills | 9 spills
3780 // V | | 8 (pad0 slot for callee)
3781 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3782 // ^ | out | 7
3783 // | | args | 6 Holes in outgoing args owned by CALLEE
3784 // Owned by +--------+
3785 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3786 // | new |preserve| Must be even-aligned.
3787 // | SP-+--------+----> Matcher::_new_SP, even aligned
3788 // | | |
3789 //
3790 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3791 // known from SELF's arguments and the Java calling convention.
3792 // Region 6-7 is determined per call site.
3793 // Note 2: If the calling convention leaves holes in the incoming argument
3794 // area, those holes are owned by SELF. Holes in the outgoing area
3795 // are owned by the CALLEE. Holes should not be necessary in the
3796 // incoming area, as the Java calling convention is completely under
3797 // the control of the AD file. Doubles can be sorted and packed to
3798 // avoid holes. Holes in the outgoing arguments may be necessary for
3799 // varargs C calling conventions.
3800 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3801 // even aligned with pad0 as needed.
3802 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3803 // (the latter is true on Intel but is it false on AArch64?)
3804 // region 6-11 is even aligned; it may be padded out more so that
3805 // the region from SP to FP meets the minimum stack alignment.
3806 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3807 // alignment. Region 11, pad1, may be dynamically extended so that
3808 // SP meets the minimum alignment.
3809
3810 frame %{
3811 // These three registers define part of the calling convention
3812 // between compiled code and the interpreter.
3813
3814 // Inline Cache Register or Method for I2C.
3815 inline_cache_reg(R12);
3816
3817 // Number of stack slots consumed by locking an object
3818 sync_stack_slots(2);
3819
3820 // Compiled code's Frame Pointer
3821 frame_pointer(R31);
3822
3823 // Interpreter stores its frame pointer in a register which is
3824 // stored to the stack by I2CAdaptors.
3825 // I2CAdaptors convert from interpreted java to compiled java.
3826 interpreter_frame_pointer(R29);
3827
3828 // Stack alignment requirement
3829 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3830
3831 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3832 // for calls to C. Supports the var-args backing area for register parms.
3833 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3834
3835 // The after-PROLOG location of the return address. Location of
3836 // return address specifies a type (REG or STACK) and a number
3837 // representing the register number (i.e. - use a register name) or
3838 // stack slot.
3839 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3840 // Otherwise, it is above the locks and verification slot and alignment word
3841 // TODO this may well be correct but need to check why that - 2 is there
3842 // ppc port uses 0 but we definitely need to allow for fixed_slots
3843 // which folds in the space used for monitors
3844 return_addr(STACK - 2 +
3845 align_up((Compile::current()->in_preserve_stack_slots() +
3846 Compile::current()->fixed_slots()),
3847 stack_alignment_in_slots()));
3848
3849 // Location of compiled Java return values. Same as C for now.
3850 return_value
3851 %{
3852 // TODO do we allow ideal_reg == Op_RegN???
3853 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3854 "only return normal values");
3855
3856 static const int lo[Op_RegL + 1] = { // enum name
3857 0, // Op_Node
3858 0, // Op_Set
3859 R0_num, // Op_RegN
3860 R0_num, // Op_RegI
3861 R0_num, // Op_RegP
3862 V0_num, // Op_RegF
3863 V0_num, // Op_RegD
3864 R0_num // Op_RegL
3865 };
3866
3867 static const int hi[Op_RegL + 1] = { // enum name
3868 0, // Op_Node
3869 0, // Op_Set
3870 OptoReg::Bad, // Op_RegN
3871 OptoReg::Bad, // Op_RegI
3872 R0_H_num, // Op_RegP
3873 OptoReg::Bad, // Op_RegF
3874 V0_H_num, // Op_RegD
3875 R0_H_num // Op_RegL
3876 };
3877
3878 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3879 %}
3880 %}
3881
3882 //----------ATTRIBUTES---------------------------------------------------------
3883 //----------Operand Attributes-------------------------------------------------
3884 op_attrib op_cost(1); // Required cost attribute
3885
3886 //----------Instruction Attributes---------------------------------------------
3887 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3888 ins_attrib ins_size(32); // Required size attribute (in bits)
3889 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3890 // a non-matching short branch variant
3891 // of some long branch?
3892 ins_attrib ins_alignment(4); // Required alignment attribute (must
3893 // be a power of 2) specifies the
3894 // alignment that some part of the
3895 // instruction (not necessarily the
3896 // start) requires. If > 1, a
3897 // compute_padding() function must be
3898 // provided for the instruction
3899
3900 //----------OPERANDS-----------------------------------------------------------
3901 // Operand definitions must precede instruction definitions for correct parsing
3902 // in the ADLC because operands constitute user defined types which are used in
3903 // instruction definitions.
3904
3905 //----------Simple Operands----------------------------------------------------
3906
3907 // Integer operands 32 bit
3908 // 32 bit immediate
3909 operand immI()
3910 %{
3911 match(ConI);
3912
3913 op_cost(0);
3914 format %{ %}
3915 interface(CONST_INTER);
3916 %}
3917
3918 // 32 bit zero
3919 operand immI0()
3920 %{
3921 predicate(n->get_int() == 0);
3922 match(ConI);
3923
3924 op_cost(0);
3925 format %{ %}
3926 interface(CONST_INTER);
3927 %}
3928
3929 // 32 bit unit increment
3930 operand immI_1()
3931 %{
3932 predicate(n->get_int() == 1);
3933 match(ConI);
3934
3935 op_cost(0);
3936 format %{ %}
3937 interface(CONST_INTER);
3938 %}
3939
3940 // 32 bit unit decrement
3941 operand immI_M1()
3942 %{
3943 predicate(n->get_int() == -1);
3944 match(ConI);
3945
3946 op_cost(0);
3947 format %{ %}
3948 interface(CONST_INTER);
3949 %}
3950
3951 // Shift values for add/sub extension shift
3952 operand immIExt()
3953 %{
3954 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3955 match(ConI);
3956
3957 op_cost(0);
3958 format %{ %}
3959 interface(CONST_INTER);
3960 %}
3961
3962 operand immI_gt_1()
3963 %{
3964 predicate(n->get_int() > 1);
3965 match(ConI);
3966
3967 op_cost(0);
3968 format %{ %}
3969 interface(CONST_INTER);
3970 %}
3971
3972 operand immI_le_4()
3973 %{
3974 predicate(n->get_int() <= 4);
3975 match(ConI);
3976
3977 op_cost(0);
3978 format %{ %}
3979 interface(CONST_INTER);
3980 %}
3981
3982 operand immI_16()
3983 %{
3984 predicate(n->get_int() == 16);
3985 match(ConI);
3986
3987 op_cost(0);
3988 format %{ %}
3989 interface(CONST_INTER);
3990 %}
3991
3992 operand immI_24()
3993 %{
3994 predicate(n->get_int() == 24);
3995 match(ConI);
3996
3997 op_cost(0);
3998 format %{ %}
3999 interface(CONST_INTER);
4000 %}
4001
4002 operand immI_32()
4003 %{
4004 predicate(n->get_int() == 32);
4005 match(ConI);
4006
4007 op_cost(0);
4008 format %{ %}
4009 interface(CONST_INTER);
4010 %}
4011
4012 operand immI_48()
4013 %{
4014 predicate(n->get_int() == 48);
4015 match(ConI);
4016
4017 op_cost(0);
4018 format %{ %}
4019 interface(CONST_INTER);
4020 %}
4021
4022 operand immI_56()
4023 %{
4024 predicate(n->get_int() == 56);
4025 match(ConI);
4026
4027 op_cost(0);
4028 format %{ %}
4029 interface(CONST_INTER);
4030 %}
4031
4032 operand immI_255()
4033 %{
4034 predicate(n->get_int() == 255);
4035 match(ConI);
4036
4037 op_cost(0);
4038 format %{ %}
4039 interface(CONST_INTER);
4040 %}
4041
4042 operand immI_65535()
4043 %{
4044 predicate(n->get_int() == 65535);
4045 match(ConI);
4046
4047 op_cost(0);
4048 format %{ %}
4049 interface(CONST_INTER);
4050 %}
4051
4052 operand immI_positive()
4053 %{
4054 predicate(n->get_int() > 0);
4055 match(ConI);
4056
4057 op_cost(0);
4058 format %{ %}
4059 interface(CONST_INTER);
4060 %}
4061
4062 // BoolTest condition for signed compare
4063 operand immI_cmp_cond()
4064 %{
4065 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4066 match(ConI);
4067
4068 op_cost(0);
4069 format %{ %}
4070 interface(CONST_INTER);
4071 %}
4072
4073 // BoolTest condition for unsigned compare
4074 operand immI_cmpU_cond()
4075 %{
4076 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4077 match(ConI);
4078
4079 op_cost(0);
4080 format %{ %}
4081 interface(CONST_INTER);
4082 %}
4083
4084 operand immL_255()
4085 %{
4086 predicate(n->get_long() == 255L);
4087 match(ConL);
4088
4089 op_cost(0);
4090 format %{ %}
4091 interface(CONST_INTER);
4092 %}
4093
4094 operand immL_65535()
4095 %{
4096 predicate(n->get_long() == 65535L);
4097 match(ConL);
4098
4099 op_cost(0);
4100 format %{ %}
4101 interface(CONST_INTER);
4102 %}
4103
4104 operand immL_4294967295()
4105 %{
4106 predicate(n->get_long() == 4294967295L);
4107 match(ConL);
4108
4109 op_cost(0);
4110 format %{ %}
4111 interface(CONST_INTER);
4112 %}
4113
4114 operand immL_bitmask()
4115 %{
4116 predicate((n->get_long() != 0)
4117 && ((n->get_long() & 0xc000000000000000l) == 0)
4118 && is_power_of_2(n->get_long() + 1));
4119 match(ConL);
4120
4121 op_cost(0);
4122 format %{ %}
4123 interface(CONST_INTER);
4124 %}
4125
4126 operand immI_bitmask()
4127 %{
4128 predicate((n->get_int() != 0)
4129 && ((n->get_int() & 0xc0000000) == 0)
4130 && is_power_of_2(n->get_int() + 1));
4131 match(ConI);
4132
4133 op_cost(0);
4134 format %{ %}
4135 interface(CONST_INTER);
4136 %}
4137
4138 operand immL_positive_bitmaskI()
4139 %{
4140 predicate((n->get_long() != 0)
4141 && ((julong)n->get_long() < 0x80000000ULL)
4142 && is_power_of_2(n->get_long() + 1));
4143 match(ConL);
4144
4145 op_cost(0);
4146 format %{ %}
4147 interface(CONST_INTER);
4148 %}
4149
4150 // Scale values for scaled offset addressing modes (up to long but not quad)
4151 operand immIScale()
4152 %{
4153 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4154 match(ConI);
4155
4156 op_cost(0);
4157 format %{ %}
4158 interface(CONST_INTER);
4159 %}
4160
4161 // 5 bit signed integer
4162 operand immI5()
4163 %{
4164 predicate(Assembler::is_simm(n->get_int(), 5));
4165 match(ConI);
4166
4167 op_cost(0);
4168 format %{ %}
4169 interface(CONST_INTER);
4170 %}
4171
4172 // 7 bit unsigned integer
4173 operand immIU7()
4174 %{
4175 predicate(Assembler::is_uimm(n->get_int(), 7));
4176 match(ConI);
4177
4178 op_cost(0);
4179 format %{ %}
4180 interface(CONST_INTER);
4181 %}
4182
4183 // Offset for scaled or unscaled immediate loads and stores
4184 operand immIOffset()
4185 %{
4186 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4187 match(ConI);
4188
4189 op_cost(0);
4190 format %{ %}
4191 interface(CONST_INTER);
4192 %}
4193
4194 operand immIOffset1()
4195 %{
4196 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4197 match(ConI);
4198
4199 op_cost(0);
4200 format %{ %}
4201 interface(CONST_INTER);
4202 %}
4203
4204 operand immIOffset2()
4205 %{
4206 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4207 match(ConI);
4208
4209 op_cost(0);
4210 format %{ %}
4211 interface(CONST_INTER);
4212 %}
4213
4214 operand immIOffset4()
4215 %{
4216 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4217 match(ConI);
4218
4219 op_cost(0);
4220 format %{ %}
4221 interface(CONST_INTER);
4222 %}
4223
4224 operand immIOffset8()
4225 %{
4226 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4227 match(ConI);
4228
4229 op_cost(0);
4230 format %{ %}
4231 interface(CONST_INTER);
4232 %}
4233
4234 operand immIOffset16()
4235 %{
4236 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4237 match(ConI);
4238
4239 op_cost(0);
4240 format %{ %}
4241 interface(CONST_INTER);
4242 %}
4243
4244 operand immLOffset()
4245 %{
4246 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4247 match(ConL);
4248
4249 op_cost(0);
4250 format %{ %}
4251 interface(CONST_INTER);
4252 %}
4253
4254 operand immLoffset1()
4255 %{
4256 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4257 match(ConL);
4258
4259 op_cost(0);
4260 format %{ %}
4261 interface(CONST_INTER);
4262 %}
4263
4264 operand immLoffset2()
4265 %{
4266 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4267 match(ConL);
4268
4269 op_cost(0);
4270 format %{ %}
4271 interface(CONST_INTER);
4272 %}
4273
4274 operand immLoffset4()
4275 %{
4276 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4277 match(ConL);
4278
4279 op_cost(0);
4280 format %{ %}
4281 interface(CONST_INTER);
4282 %}
4283
4284 operand immLoffset8()
4285 %{
4286 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4287 match(ConL);
4288
4289 op_cost(0);
4290 format %{ %}
4291 interface(CONST_INTER);
4292 %}
4293
4294 operand immLoffset16()
4295 %{
4296 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4297 match(ConL);
4298
4299 op_cost(0);
4300 format %{ %}
4301 interface(CONST_INTER);
4302 %}
4303
4304 // 5 bit signed long integer
4305 operand immL5()
4306 %{
4307 predicate(Assembler::is_simm(n->get_long(), 5));
4308 match(ConL);
4309
4310 op_cost(0);
4311 format %{ %}
4312 interface(CONST_INTER);
4313 %}
4314
4315 // 7 bit unsigned long integer
4316 operand immLU7()
4317 %{
4318 predicate(Assembler::is_uimm(n->get_long(), 7));
4319 match(ConL);
4320
4321 op_cost(0);
4322 format %{ %}
4323 interface(CONST_INTER);
4324 %}
4325
4326 // 8 bit signed value.
4327 operand immI8()
4328 %{
4329 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4330 match(ConI);
4331
4332 op_cost(0);
4333 format %{ %}
4334 interface(CONST_INTER);
4335 %}
4336
4337 // 8 bit signed value (simm8), or #simm8 LSL 8.
4338 operand immI8_shift8()
4339 %{
4340 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4341 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4342 match(ConI);
4343
4344 op_cost(0);
4345 format %{ %}
4346 interface(CONST_INTER);
4347 %}
4348
4349 // 8 bit signed value (simm8), or #simm8 LSL 8.
4350 operand immL8_shift8()
4351 %{
4352 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4353 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4354 match(ConL);
4355
4356 op_cost(0);
4357 format %{ %}
4358 interface(CONST_INTER);
4359 %}
4360
4361 // 8 bit integer valid for vector add sub immediate
4362 operand immBAddSubV()
4363 %{
4364 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4365 match(ConI);
4366
4367 op_cost(0);
4368 format %{ %}
4369 interface(CONST_INTER);
4370 %}
4371
4372 // 32 bit integer valid for add sub immediate
4373 operand immIAddSub()
4374 %{
4375 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4376 match(ConI);
4377 op_cost(0);
4378 format %{ %}
4379 interface(CONST_INTER);
4380 %}
4381
4382 // 32 bit integer valid for vector add sub immediate
4383 operand immIAddSubV()
4384 %{
4385 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4386 match(ConI);
4387
4388 op_cost(0);
4389 format %{ %}
4390 interface(CONST_INTER);
4391 %}
4392
4393 // 32 bit unsigned integer valid for logical immediate
4394
4395 operand immBLog()
4396 %{
4397 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4398 match(ConI);
4399
4400 op_cost(0);
4401 format %{ %}
4402 interface(CONST_INTER);
4403 %}
4404
4405 operand immSLog()
4406 %{
4407 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4408 match(ConI);
4409
4410 op_cost(0);
4411 format %{ %}
4412 interface(CONST_INTER);
4413 %}
4414
4415 operand immILog()
4416 %{
4417 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4418 match(ConI);
4419
4420 op_cost(0);
4421 format %{ %}
4422 interface(CONST_INTER);
4423 %}
4424
4425 // Integer operands 64 bit
4426 // 64 bit immediate
4427 operand immL()
4428 %{
4429 match(ConL);
4430
4431 op_cost(0);
4432 format %{ %}
4433 interface(CONST_INTER);
4434 %}
4435
4436 // 64 bit zero
4437 operand immL0()
4438 %{
4439 predicate(n->get_long() == 0);
4440 match(ConL);
4441
4442 op_cost(0);
4443 format %{ %}
4444 interface(CONST_INTER);
4445 %}
4446
4447 // 64 bit unit decrement
4448 operand immL_M1()
4449 %{
4450 predicate(n->get_long() == -1);
4451 match(ConL);
4452
4453 op_cost(0);
4454 format %{ %}
4455 interface(CONST_INTER);
4456 %}
4457
4458 // 64 bit integer valid for add sub immediate
4459 operand immLAddSub()
4460 %{
4461 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4462 match(ConL);
4463 op_cost(0);
4464 format %{ %}
4465 interface(CONST_INTER);
4466 %}
4467
4468 // 64 bit integer valid for addv subv immediate
4469 operand immLAddSubV()
4470 %{
4471 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4472 match(ConL);
4473
4474 op_cost(0);
4475 format %{ %}
4476 interface(CONST_INTER);
4477 %}
4478
4479 // 64 bit integer valid for logical immediate
4480 operand immLLog()
4481 %{
4482 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4483 match(ConL);
4484 op_cost(0);
4485 format %{ %}
4486 interface(CONST_INTER);
4487 %}
4488
4489 // Long Immediate: low 32-bit mask
4490 operand immL_32bits()
4491 %{
4492 predicate(n->get_long() == 0xFFFFFFFFL);
4493 match(ConL);
4494 op_cost(0);
4495 format %{ %}
4496 interface(CONST_INTER);
4497 %}
4498
4499 // Pointer operands
4500 // Pointer Immediate
4501 operand immP()
4502 %{
4503 match(ConP);
4504
4505 op_cost(0);
4506 format %{ %}
4507 interface(CONST_INTER);
4508 %}
4509
4510 // nullptr Pointer Immediate
4511 operand immP0()
4512 %{
4513 predicate(n->get_ptr() == 0);
4514 match(ConP);
4515
4516 op_cost(0);
4517 format %{ %}
4518 interface(CONST_INTER);
4519 %}
4520
4521 // Pointer Immediate One
4522 // this is used in object initialization (initial object header)
4523 operand immP_1()
4524 %{
4525 predicate(n->get_ptr() == 1);
4526 match(ConP);
4527
4528 op_cost(0);
4529 format %{ %}
4530 interface(CONST_INTER);
4531 %}
4532
4533 // Card Table Byte Map Base
4534 operand immByteMapBase()
4535 %{
4536 // Get base of card map
4537 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4538 is_card_table_address((address)(n->get_ptr())));
4539 match(ConP);
4540
4541 op_cost(0);
4542 format %{ %}
4543 interface(CONST_INTER);
4544 %}
4545
4546 // AOT Runtime Constants Address
4547 operand immAOTRuntimeConstantsAddress()
4548 %{
4549 // Check if the address is in the range of AOT Runtime Constants
4550 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4551 match(ConP);
4552
4553 op_cost(0);
4554 format %{ %}
4555 interface(CONST_INTER);
4556 %}
4557
4558 // Float and Double operands
4559 // Double Immediate
4560 operand immD()
4561 %{
4562 match(ConD);
4563 op_cost(0);
4564 format %{ %}
4565 interface(CONST_INTER);
4566 %}
4567
4568 // Double Immediate: +0.0d
4569 operand immD0()
4570 %{
4571 predicate(jlong_cast(n->getd()) == 0);
4572 match(ConD);
4573
4574 op_cost(0);
4575 format %{ %}
4576 interface(CONST_INTER);
4577 %}
4578
4579 // constant 'double +0.0'.
4580 operand immDPacked()
4581 %{
4582 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4583 match(ConD);
4584 op_cost(0);
4585 format %{ %}
4586 interface(CONST_INTER);
4587 %}
4588
4589 // Float Immediate
4590 operand immF()
4591 %{
4592 match(ConF);
4593 op_cost(0);
4594 format %{ %}
4595 interface(CONST_INTER);
4596 %}
4597
4598 // Float Immediate: +0.0f.
4599 operand immF0()
4600 %{
4601 predicate(jint_cast(n->getf()) == 0);
4602 match(ConF);
4603
4604 op_cost(0);
4605 format %{ %}
4606 interface(CONST_INTER);
4607 %}
4608
4609 //
4610 operand immFPacked()
4611 %{
4612 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4613 match(ConF);
4614 op_cost(0);
4615 format %{ %}
4616 interface(CONST_INTER);
4617 %}
4618
4619 // Narrow pointer operands
4620 // Narrow Pointer Immediate
4621 operand immN()
4622 %{
4623 match(ConN);
4624
4625 op_cost(0);
4626 format %{ %}
4627 interface(CONST_INTER);
4628 %}
4629
4630 // Narrow nullptr Pointer Immediate
4631 operand immN0()
4632 %{
4633 predicate(n->get_narrowcon() == 0);
4634 match(ConN);
4635
4636 op_cost(0);
4637 format %{ %}
4638 interface(CONST_INTER);
4639 %}
4640
4641 operand immNKlass()
4642 %{
4643 match(ConNKlass);
4644
4645 op_cost(0);
4646 format %{ %}
4647 interface(CONST_INTER);
4648 %}
4649
4650 // Integer 32 bit Register Operands
4651 // Integer 32 bitRegister (excludes SP)
4652 operand iRegI()
4653 %{
4654 constraint(ALLOC_IN_RC(any_reg32));
4655 match(RegI);
4656 match(iRegINoSp);
4657 op_cost(0);
4658 format %{ %}
4659 interface(REG_INTER);
4660 %}
4661
4662 // Integer 32 bit Register not Special
4663 operand iRegINoSp()
4664 %{
4665 constraint(ALLOC_IN_RC(no_special_reg32));
4666 match(RegI);
4667 op_cost(0);
4668 format %{ %}
4669 interface(REG_INTER);
4670 %}
4671
4672 // Integer 64 bit Register Operands
4673 // Integer 64 bit Register (includes SP)
4674 operand iRegL()
4675 %{
4676 constraint(ALLOC_IN_RC(any_reg));
4677 match(RegL);
4678 match(iRegLNoSp);
4679 op_cost(0);
4680 format %{ %}
4681 interface(REG_INTER);
4682 %}
4683
4684 // Integer 64 bit Register not Special
4685 operand iRegLNoSp()
4686 %{
4687 constraint(ALLOC_IN_RC(no_special_reg));
4688 match(RegL);
4689 match(iRegL_R0);
4690 format %{ %}
4691 interface(REG_INTER);
4692 %}
4693
4694 // Pointer Register Operands
4695 // Pointer Register
4696 operand iRegP()
4697 %{
4698 constraint(ALLOC_IN_RC(ptr_reg));
4699 match(RegP);
4700 match(iRegPNoSp);
4701 match(iRegP_R0);
4702 //match(iRegP_R2);
4703 //match(iRegP_R4);
4704 match(iRegP_R5);
4705 match(thread_RegP);
4706 op_cost(0);
4707 format %{ %}
4708 interface(REG_INTER);
4709 %}
4710
4711 // Pointer 64 bit Register not Special
4712 operand iRegPNoSp()
4713 %{
4714 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4715 match(RegP);
4716 // match(iRegP);
4717 // match(iRegP_R0);
4718 // match(iRegP_R2);
4719 // match(iRegP_R4);
4720 // match(iRegP_R5);
4721 // match(thread_RegP);
4722 op_cost(0);
4723 format %{ %}
4724 interface(REG_INTER);
4725 %}
4726
4727 // This operand is not allowed to use rfp even if
4728 // rfp is not used to hold the frame pointer.
4729 operand iRegPNoSpNoRfp()
4730 %{
4731 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4732 match(RegP);
4733 match(iRegPNoSp);
4734 op_cost(0);
4735 format %{ %}
4736 interface(REG_INTER);
4737 %}
4738
4739 // Pointer 64 bit Register R0 only
4740 operand iRegP_R0()
4741 %{
4742 constraint(ALLOC_IN_RC(r0_reg));
4743 match(RegP);
4744 // match(iRegP);
4745 match(iRegPNoSp);
4746 op_cost(0);
4747 format %{ %}
4748 interface(REG_INTER);
4749 %}
4750
4751 // Pointer 64 bit Register R1 only
4752 operand iRegP_R1()
4753 %{
4754 constraint(ALLOC_IN_RC(r1_reg));
4755 match(RegP);
4756 // match(iRegP);
4757 match(iRegPNoSp);
4758 op_cost(0);
4759 format %{ %}
4760 interface(REG_INTER);
4761 %}
4762
4763 // Pointer 64 bit Register R2 only
4764 operand iRegP_R2()
4765 %{
4766 constraint(ALLOC_IN_RC(r2_reg));
4767 match(RegP);
4768 // match(iRegP);
4769 match(iRegPNoSp);
4770 op_cost(0);
4771 format %{ %}
4772 interface(REG_INTER);
4773 %}
4774
4775 // Pointer 64 bit Register R3 only
4776 operand iRegP_R3()
4777 %{
4778 constraint(ALLOC_IN_RC(r3_reg));
4779 match(RegP);
4780 // match(iRegP);
4781 match(iRegPNoSp);
4782 op_cost(0);
4783 format %{ %}
4784 interface(REG_INTER);
4785 %}
4786
4787 // Pointer 64 bit Register R4 only
4788 operand iRegP_R4()
4789 %{
4790 constraint(ALLOC_IN_RC(r4_reg));
4791 match(RegP);
4792 // match(iRegP);
4793 match(iRegPNoSp);
4794 op_cost(0);
4795 format %{ %}
4796 interface(REG_INTER);
4797 %}
4798
4799 // Pointer 64 bit Register R5 only
4800 operand iRegP_R5()
4801 %{
4802 constraint(ALLOC_IN_RC(r5_reg));
4803 match(RegP);
4804 // match(iRegP);
4805 match(iRegPNoSp);
4806 op_cost(0);
4807 format %{ %}
4808 interface(REG_INTER);
4809 %}
4810
4811 // Pointer 64 bit Register R10 only
4812 operand iRegP_R10()
4813 %{
4814 constraint(ALLOC_IN_RC(r10_reg));
4815 match(RegP);
4816 // match(iRegP);
4817 match(iRegPNoSp);
4818 op_cost(0);
4819 format %{ %}
4820 interface(REG_INTER);
4821 %}
4822
4823 // Long 64 bit Register R0 only
4824 operand iRegL_R0()
4825 %{
4826 constraint(ALLOC_IN_RC(r0_reg));
4827 match(RegL);
4828 match(iRegLNoSp);
4829 op_cost(0);
4830 format %{ %}
4831 interface(REG_INTER);
4832 %}
4833
4834 // Long 64 bit Register R11 only
4835 operand iRegL_R11()
4836 %{
4837 constraint(ALLOC_IN_RC(r11_reg));
4838 match(RegL);
4839 match(iRegLNoSp);
4840 op_cost(0);
4841 format %{ %}
4842 interface(REG_INTER);
4843 %}
4844
4845 // Register R0 only
4846 operand iRegI_R0()
4847 %{
4848 constraint(ALLOC_IN_RC(int_r0_reg));
4849 match(RegI);
4850 match(iRegINoSp);
4851 op_cost(0);
4852 format %{ %}
4853 interface(REG_INTER);
4854 %}
4855
4856 // Register R2 only
4857 operand iRegI_R2()
4858 %{
4859 constraint(ALLOC_IN_RC(int_r2_reg));
4860 match(RegI);
4861 match(iRegINoSp);
4862 op_cost(0);
4863 format %{ %}
4864 interface(REG_INTER);
4865 %}
4866
4867 // Register R3 only
4868 operand iRegI_R3()
4869 %{
4870 constraint(ALLOC_IN_RC(int_r3_reg));
4871 match(RegI);
4872 match(iRegINoSp);
4873 op_cost(0);
4874 format %{ %}
4875 interface(REG_INTER);
4876 %}
4877
4878
4879 // Register R4 only
4880 operand iRegI_R4()
4881 %{
4882 constraint(ALLOC_IN_RC(int_r4_reg));
4883 match(RegI);
4884 match(iRegINoSp);
4885 op_cost(0);
4886 format %{ %}
4887 interface(REG_INTER);
4888 %}
4889
4890
4891 // Pointer Register Operands
4892 // Narrow Pointer Register
4893 operand iRegN()
4894 %{
4895 constraint(ALLOC_IN_RC(any_reg32));
4896 match(RegN);
4897 match(iRegNNoSp);
4898 op_cost(0);
4899 format %{ %}
4900 interface(REG_INTER);
4901 %}
4902
4903 // Integer 64 bit Register not Special
4904 operand iRegNNoSp()
4905 %{
4906 constraint(ALLOC_IN_RC(no_special_reg32));
4907 match(RegN);
4908 op_cost(0);
4909 format %{ %}
4910 interface(REG_INTER);
4911 %}
4912
4913 // Float Register
4914 // Float register operands
4915 operand vRegF()
4916 %{
4917 constraint(ALLOC_IN_RC(float_reg));
4918 match(RegF);
4919
4920 op_cost(0);
4921 format %{ %}
4922 interface(REG_INTER);
4923 %}
4924
4925 // Double Register
4926 // Double register operands
4927 operand vRegD()
4928 %{
4929 constraint(ALLOC_IN_RC(double_reg));
4930 match(RegD);
4931
4932 op_cost(0);
4933 format %{ %}
4934 interface(REG_INTER);
4935 %}
4936
4937 // Generic vector class. This will be used for
4938 // all vector operands, including NEON and SVE.
4939 operand vReg()
4940 %{
4941 constraint(ALLOC_IN_RC(dynamic));
4942 match(VecA);
4943 match(VecD);
4944 match(VecX);
4945
4946 op_cost(0);
4947 format %{ %}
4948 interface(REG_INTER);
4949 %}
4950
4951 operand vecA()
4952 %{
4953 constraint(ALLOC_IN_RC(vectora_reg));
4954 match(VecA);
4955
4956 op_cost(0);
4957 format %{ %}
4958 interface(REG_INTER);
4959 %}
4960
4961 operand vecD()
4962 %{
4963 constraint(ALLOC_IN_RC(vectord_reg));
4964 match(VecD);
4965
4966 op_cost(0);
4967 format %{ %}
4968 interface(REG_INTER);
4969 %}
4970
4971 operand vecX()
4972 %{
4973 constraint(ALLOC_IN_RC(vectorx_reg));
4974 match(VecX);
4975
4976 op_cost(0);
4977 format %{ %}
4978 interface(REG_INTER);
4979 %}
4980
4981 operand vRegD_V0()
4982 %{
4983 constraint(ALLOC_IN_RC(v0_reg));
4984 match(RegD);
4985 op_cost(0);
4986 format %{ %}
4987 interface(REG_INTER);
4988 %}
4989
4990 operand vRegD_V1()
4991 %{
4992 constraint(ALLOC_IN_RC(v1_reg));
4993 match(RegD);
4994 op_cost(0);
4995 format %{ %}
4996 interface(REG_INTER);
4997 %}
4998
4999 operand vRegD_V2()
5000 %{
5001 constraint(ALLOC_IN_RC(v2_reg));
5002 match(RegD);
5003 op_cost(0);
5004 format %{ %}
5005 interface(REG_INTER);
5006 %}
5007
5008 operand vRegD_V3()
5009 %{
5010 constraint(ALLOC_IN_RC(v3_reg));
5011 match(RegD);
5012 op_cost(0);
5013 format %{ %}
5014 interface(REG_INTER);
5015 %}
5016
5017 operand vRegD_V4()
5018 %{
5019 constraint(ALLOC_IN_RC(v4_reg));
5020 match(RegD);
5021 op_cost(0);
5022 format %{ %}
5023 interface(REG_INTER);
5024 %}
5025
5026 operand vRegD_V5()
5027 %{
5028 constraint(ALLOC_IN_RC(v5_reg));
5029 match(RegD);
5030 op_cost(0);
5031 format %{ %}
5032 interface(REG_INTER);
5033 %}
5034
5035 operand vRegD_V6()
5036 %{
5037 constraint(ALLOC_IN_RC(v6_reg));
5038 match(RegD);
5039 op_cost(0);
5040 format %{ %}
5041 interface(REG_INTER);
5042 %}
5043
5044 operand vRegD_V7()
5045 %{
5046 constraint(ALLOC_IN_RC(v7_reg));
5047 match(RegD);
5048 op_cost(0);
5049 format %{ %}
5050 interface(REG_INTER);
5051 %}
5052
5053 operand vRegD_V12()
5054 %{
5055 constraint(ALLOC_IN_RC(v12_reg));
5056 match(RegD);
5057 op_cost(0);
5058 format %{ %}
5059 interface(REG_INTER);
5060 %}
5061
5062 operand vRegD_V13()
5063 %{
5064 constraint(ALLOC_IN_RC(v13_reg));
5065 match(RegD);
5066 op_cost(0);
5067 format %{ %}
5068 interface(REG_INTER);
5069 %}
5070
5071 operand pReg()
5072 %{
5073 constraint(ALLOC_IN_RC(pr_reg));
5074 match(RegVectMask);
5075 match(pRegGov);
5076 op_cost(0);
5077 format %{ %}
5078 interface(REG_INTER);
5079 %}
5080
5081 operand pRegGov()
5082 %{
5083 constraint(ALLOC_IN_RC(gov_pr));
5084 match(RegVectMask);
5085 match(pReg);
5086 op_cost(0);
5087 format %{ %}
5088 interface(REG_INTER);
5089 %}
5090
5091 operand pRegGov_P0()
5092 %{
5093 constraint(ALLOC_IN_RC(p0_reg));
5094 match(RegVectMask);
5095 op_cost(0);
5096 format %{ %}
5097 interface(REG_INTER);
5098 %}
5099
5100 operand pRegGov_P1()
5101 %{
5102 constraint(ALLOC_IN_RC(p1_reg));
5103 match(RegVectMask);
5104 op_cost(0);
5105 format %{ %}
5106 interface(REG_INTER);
5107 %}
5108
5109 // Flags register, used as output of signed compare instructions
5110
5111 // note that on AArch64 we also use this register as the output for
5112 // for floating point compare instructions (CmpF CmpD). this ensures
5113 // that ordered inequality tests use GT, GE, LT or LE none of which
5114 // pass through cases where the result is unordered i.e. one or both
5115 // inputs to the compare is a NaN. this means that the ideal code can
5116 // replace e.g. a GT with an LE and not end up capturing the NaN case
5117 // (where the comparison should always fail). EQ and NE tests are
5118 // always generated in ideal code so that unordered folds into the NE
5119 // case, matching the behaviour of AArch64 NE.
5120 //
5121 // This differs from x86 where the outputs of FP compares use a
5122 // special FP flags registers and where compares based on this
5123 // register are distinguished into ordered inequalities (cmpOpUCF) and
5124 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5125 // to explicitly handle the unordered case in branches. x86 also has
5126 // to include extra CMoveX rules to accept a cmpOpUCF input.
5127
5128 operand rFlagsReg()
5129 %{
5130 constraint(ALLOC_IN_RC(int_flags));
5131 match(RegFlags);
5132
5133 op_cost(0);
5134 format %{ "RFLAGS" %}
5135 interface(REG_INTER);
5136 %}
5137
5138 // Flags register, used as output of unsigned compare instructions
5139 operand rFlagsRegU()
5140 %{
5141 constraint(ALLOC_IN_RC(int_flags));
5142 match(RegFlags);
5143
5144 op_cost(0);
5145 format %{ "RFLAGSU" %}
5146 interface(REG_INTER);
5147 %}
5148
5149 // Special Registers
5150
5151 // Method Register
5152 operand inline_cache_RegP(iRegP reg)
5153 %{
5154 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5155 match(reg);
5156 match(iRegPNoSp);
5157 op_cost(0);
5158 format %{ %}
5159 interface(REG_INTER);
5160 %}
5161
5162 // Thread Register
5163 operand thread_RegP(iRegP reg)
5164 %{
5165 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5166 match(reg);
5167 op_cost(0);
5168 format %{ %}
5169 interface(REG_INTER);
5170 %}
5171
5172 //----------Memory Operands----------------------------------------------------
5173
5174 operand indirect(iRegP reg)
5175 %{
5176 constraint(ALLOC_IN_RC(ptr_reg));
5177 match(reg);
5178 op_cost(0);
5179 format %{ "[$reg]" %}
5180 interface(MEMORY_INTER) %{
5181 base($reg);
5182 index(0xffffffff);
5183 scale(0x0);
5184 disp(0x0);
5185 %}
5186 %}
5187
5188 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5189 %{
5190 constraint(ALLOC_IN_RC(ptr_reg));
5191 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5192 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5193 op_cost(0);
5194 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5195 interface(MEMORY_INTER) %{
5196 base($reg);
5197 index($ireg);
5198 scale($scale);
5199 disp(0x0);
5200 %}
5201 %}
5202
5203 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5204 %{
5205 constraint(ALLOC_IN_RC(ptr_reg));
5206 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5207 match(AddP reg (LShiftL lreg scale));
5208 op_cost(0);
5209 format %{ "$reg, $lreg lsl($scale)" %}
5210 interface(MEMORY_INTER) %{
5211 base($reg);
5212 index($lreg);
5213 scale($scale);
5214 disp(0x0);
5215 %}
5216 %}
5217
5218 operand indIndexI2L(iRegP reg, iRegI ireg)
5219 %{
5220 constraint(ALLOC_IN_RC(ptr_reg));
5221 match(AddP reg (ConvI2L ireg));
5222 op_cost(0);
5223 format %{ "$reg, $ireg, 0, I2L" %}
5224 interface(MEMORY_INTER) %{
5225 base($reg);
5226 index($ireg);
5227 scale(0x0);
5228 disp(0x0);
5229 %}
5230 %}
5231
5232 operand indIndex(iRegP reg, iRegL lreg)
5233 %{
5234 constraint(ALLOC_IN_RC(ptr_reg));
5235 match(AddP reg lreg);
5236 op_cost(0);
5237 format %{ "$reg, $lreg" %}
5238 interface(MEMORY_INTER) %{
5239 base($reg);
5240 index($lreg);
5241 scale(0x0);
5242 disp(0x0);
5243 %}
5244 %}
5245
5246 operand indOffI1(iRegP reg, immIOffset1 off)
5247 %{
5248 constraint(ALLOC_IN_RC(ptr_reg));
5249 match(AddP reg off);
5250 op_cost(0);
5251 format %{ "[$reg, $off]" %}
5252 interface(MEMORY_INTER) %{
5253 base($reg);
5254 index(0xffffffff);
5255 scale(0x0);
5256 disp($off);
5257 %}
5258 %}
5259
5260 operand indOffI2(iRegP reg, immIOffset2 off)
5261 %{
5262 constraint(ALLOC_IN_RC(ptr_reg));
5263 match(AddP reg off);
5264 op_cost(0);
5265 format %{ "[$reg, $off]" %}
5266 interface(MEMORY_INTER) %{
5267 base($reg);
5268 index(0xffffffff);
5269 scale(0x0);
5270 disp($off);
5271 %}
5272 %}
5273
5274 operand indOffI4(iRegP reg, immIOffset4 off)
5275 %{
5276 constraint(ALLOC_IN_RC(ptr_reg));
5277 match(AddP reg off);
5278 op_cost(0);
5279 format %{ "[$reg, $off]" %}
5280 interface(MEMORY_INTER) %{
5281 base($reg);
5282 index(0xffffffff);
5283 scale(0x0);
5284 disp($off);
5285 %}
5286 %}
5287
5288 operand indOffI8(iRegP reg, immIOffset8 off)
5289 %{
5290 constraint(ALLOC_IN_RC(ptr_reg));
5291 match(AddP reg off);
5292 op_cost(0);
5293 format %{ "[$reg, $off]" %}
5294 interface(MEMORY_INTER) %{
5295 base($reg);
5296 index(0xffffffff);
5297 scale(0x0);
5298 disp($off);
5299 %}
5300 %}
5301
5302 operand indOffI16(iRegP reg, immIOffset16 off)
5303 %{
5304 constraint(ALLOC_IN_RC(ptr_reg));
5305 match(AddP reg off);
5306 op_cost(0);
5307 format %{ "[$reg, $off]" %}
5308 interface(MEMORY_INTER) %{
5309 base($reg);
5310 index(0xffffffff);
5311 scale(0x0);
5312 disp($off);
5313 %}
5314 %}
5315
5316 operand indOffL1(iRegP reg, immLoffset1 off)
5317 %{
5318 constraint(ALLOC_IN_RC(ptr_reg));
5319 match(AddP reg off);
5320 op_cost(0);
5321 format %{ "[$reg, $off]" %}
5322 interface(MEMORY_INTER) %{
5323 base($reg);
5324 index(0xffffffff);
5325 scale(0x0);
5326 disp($off);
5327 %}
5328 %}
5329
5330 operand indOffL2(iRegP reg, immLoffset2 off)
5331 %{
5332 constraint(ALLOC_IN_RC(ptr_reg));
5333 match(AddP reg off);
5334 op_cost(0);
5335 format %{ "[$reg, $off]" %}
5336 interface(MEMORY_INTER) %{
5337 base($reg);
5338 index(0xffffffff);
5339 scale(0x0);
5340 disp($off);
5341 %}
5342 %}
5343
5344 operand indOffL4(iRegP reg, immLoffset4 off)
5345 %{
5346 constraint(ALLOC_IN_RC(ptr_reg));
5347 match(AddP reg off);
5348 op_cost(0);
5349 format %{ "[$reg, $off]" %}
5350 interface(MEMORY_INTER) %{
5351 base($reg);
5352 index(0xffffffff);
5353 scale(0x0);
5354 disp($off);
5355 %}
5356 %}
5357
5358 operand indOffL8(iRegP reg, immLoffset8 off)
5359 %{
5360 constraint(ALLOC_IN_RC(ptr_reg));
5361 match(AddP reg off);
5362 op_cost(0);
5363 format %{ "[$reg, $off]" %}
5364 interface(MEMORY_INTER) %{
5365 base($reg);
5366 index(0xffffffff);
5367 scale(0x0);
5368 disp($off);
5369 %}
5370 %}
5371
5372 operand indOffL16(iRegP reg, immLoffset16 off)
5373 %{
5374 constraint(ALLOC_IN_RC(ptr_reg));
5375 match(AddP reg off);
5376 op_cost(0);
5377 format %{ "[$reg, $off]" %}
5378 interface(MEMORY_INTER) %{
5379 base($reg);
5380 index(0xffffffff);
5381 scale(0x0);
5382 disp($off);
5383 %}
5384 %}
5385
5386 operand indirectX2P(iRegL reg)
5387 %{
5388 constraint(ALLOC_IN_RC(ptr_reg));
5389 match(CastX2P reg);
5390 op_cost(0);
5391 format %{ "[$reg]\t# long -> ptr" %}
5392 interface(MEMORY_INTER) %{
5393 base($reg);
5394 index(0xffffffff);
5395 scale(0x0);
5396 disp(0x0);
5397 %}
5398 %}
5399
5400 operand indOffX2P(iRegL reg, immLOffset off)
5401 %{
5402 constraint(ALLOC_IN_RC(ptr_reg));
5403 match(AddP (CastX2P reg) off);
5404 op_cost(0);
5405 format %{ "[$reg, $off]\t# long -> ptr" %}
5406 interface(MEMORY_INTER) %{
5407 base($reg);
5408 index(0xffffffff);
5409 scale(0x0);
5410 disp($off);
5411 %}
5412 %}
5413
5414 operand indirectN(iRegN reg)
5415 %{
5416 predicate(CompressedOops::shift() == 0);
5417 constraint(ALLOC_IN_RC(ptr_reg));
5418 match(DecodeN reg);
5419 op_cost(0);
5420 format %{ "[$reg]\t# narrow" %}
5421 interface(MEMORY_INTER) %{
5422 base($reg);
5423 index(0xffffffff);
5424 scale(0x0);
5425 disp(0x0);
5426 %}
5427 %}
5428
5429 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5430 %{
5431 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5432 constraint(ALLOC_IN_RC(ptr_reg));
5433 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5434 op_cost(0);
5435 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5436 interface(MEMORY_INTER) %{
5437 base($reg);
5438 index($ireg);
5439 scale($scale);
5440 disp(0x0);
5441 %}
5442 %}
5443
5444 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5445 %{
5446 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5447 constraint(ALLOC_IN_RC(ptr_reg));
5448 match(AddP (DecodeN reg) (LShiftL lreg scale));
5449 op_cost(0);
5450 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5451 interface(MEMORY_INTER) %{
5452 base($reg);
5453 index($lreg);
5454 scale($scale);
5455 disp(0x0);
5456 %}
5457 %}
5458
5459 operand indIndexI2LN(iRegN reg, iRegI ireg)
5460 %{
5461 predicate(CompressedOops::shift() == 0);
5462 constraint(ALLOC_IN_RC(ptr_reg));
5463 match(AddP (DecodeN reg) (ConvI2L ireg));
5464 op_cost(0);
5465 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5466 interface(MEMORY_INTER) %{
5467 base($reg);
5468 index($ireg);
5469 scale(0x0);
5470 disp(0x0);
5471 %}
5472 %}
5473
5474 operand indIndexN(iRegN reg, iRegL lreg)
5475 %{
5476 predicate(CompressedOops::shift() == 0);
5477 constraint(ALLOC_IN_RC(ptr_reg));
5478 match(AddP (DecodeN reg) lreg);
5479 op_cost(0);
5480 format %{ "$reg, $lreg\t# narrow" %}
5481 interface(MEMORY_INTER) %{
5482 base($reg);
5483 index($lreg);
5484 scale(0x0);
5485 disp(0x0);
5486 %}
5487 %}
5488
5489 operand indOffIN(iRegN reg, immIOffset off)
5490 %{
5491 predicate(CompressedOops::shift() == 0);
5492 constraint(ALLOC_IN_RC(ptr_reg));
5493 match(AddP (DecodeN reg) off);
5494 op_cost(0);
5495 format %{ "[$reg, $off]\t# narrow" %}
5496 interface(MEMORY_INTER) %{
5497 base($reg);
5498 index(0xffffffff);
5499 scale(0x0);
5500 disp($off);
5501 %}
5502 %}
5503
5504 operand indOffLN(iRegN reg, immLOffset off)
5505 %{
5506 predicate(CompressedOops::shift() == 0);
5507 constraint(ALLOC_IN_RC(ptr_reg));
5508 match(AddP (DecodeN reg) off);
5509 op_cost(0);
5510 format %{ "[$reg, $off]\t# narrow" %}
5511 interface(MEMORY_INTER) %{
5512 base($reg);
5513 index(0xffffffff);
5514 scale(0x0);
5515 disp($off);
5516 %}
5517 %}
5518
5519
5520 //----------Special Memory Operands--------------------------------------------
5521 // Stack Slot Operand - This operand is used for loading and storing temporary
5522 // values on the stack where a match requires a value to
5523 // flow through memory.
5524 operand stackSlotP(sRegP reg)
5525 %{
5526 constraint(ALLOC_IN_RC(stack_slots));
5527 op_cost(100);
5528 // No match rule because this operand is only generated in matching
5529 // match(RegP);
5530 format %{ "[$reg]" %}
5531 interface(MEMORY_INTER) %{
5532 base(0x1e); // RSP
5533 index(0x0); // No Index
5534 scale(0x0); // No Scale
5535 disp($reg); // Stack Offset
5536 %}
5537 %}
5538
5539 operand stackSlotI(sRegI reg)
5540 %{
5541 constraint(ALLOC_IN_RC(stack_slots));
5542 // No match rule because this operand is only generated in matching
5543 // match(RegI);
5544 format %{ "[$reg]" %}
5545 interface(MEMORY_INTER) %{
5546 base(0x1e); // RSP
5547 index(0x0); // No Index
5548 scale(0x0); // No Scale
5549 disp($reg); // Stack Offset
5550 %}
5551 %}
5552
5553 operand stackSlotF(sRegF reg)
5554 %{
5555 constraint(ALLOC_IN_RC(stack_slots));
5556 // No match rule because this operand is only generated in matching
5557 // match(RegF);
5558 format %{ "[$reg]" %}
5559 interface(MEMORY_INTER) %{
5560 base(0x1e); // RSP
5561 index(0x0); // No Index
5562 scale(0x0); // No Scale
5563 disp($reg); // Stack Offset
5564 %}
5565 %}
5566
5567 operand stackSlotD(sRegD reg)
5568 %{
5569 constraint(ALLOC_IN_RC(stack_slots));
5570 // No match rule because this operand is only generated in matching
5571 // match(RegD);
5572 format %{ "[$reg]" %}
5573 interface(MEMORY_INTER) %{
5574 base(0x1e); // RSP
5575 index(0x0); // No Index
5576 scale(0x0); // No Scale
5577 disp($reg); // Stack Offset
5578 %}
5579 %}
5580
5581 operand stackSlotL(sRegL reg)
5582 %{
5583 constraint(ALLOC_IN_RC(stack_slots));
5584 // No match rule because this operand is only generated in matching
5585 // match(RegL);
5586 format %{ "[$reg]" %}
5587 interface(MEMORY_INTER) %{
5588 base(0x1e); // RSP
5589 index(0x0); // No Index
5590 scale(0x0); // No Scale
5591 disp($reg); // Stack Offset
5592 %}
5593 %}
5594
5595 // Operands for expressing Control Flow
5596 // NOTE: Label is a predefined operand which should not be redefined in
5597 // the AD file. It is generically handled within the ADLC.
5598
5599 //----------Conditional Branch Operands----------------------------------------
5600 // Comparison Op - This is the operation of the comparison, and is limited to
5601 // the following set of codes:
5602 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5603 //
5604 // Other attributes of the comparison, such as unsignedness, are specified
5605 // by the comparison instruction that sets a condition code flags register.
5606 // That result is represented by a flags operand whose subtype is appropriate
5607 // to the unsignedness (etc.) of the comparison.
5608 //
5609 // Later, the instruction which matches both the Comparison Op (a Bool) and
5610 // the flags (produced by the Cmp) specifies the coding of the comparison op
5611 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5612
5613 // used for signed integral comparisons and fp comparisons
5614
5615 operand cmpOp()
5616 %{
5617 match(Bool);
5618
5619 format %{ "" %}
5620 interface(COND_INTER) %{
5621 equal(0x0, "eq");
5622 not_equal(0x1, "ne");
5623 less(0xb, "lt");
5624 greater_equal(0xa, "ge");
5625 less_equal(0xd, "le");
5626 greater(0xc, "gt");
5627 overflow(0x6, "vs");
5628 no_overflow(0x7, "vc");
5629 %}
5630 %}
5631
5632 // used for unsigned integral comparisons
5633
5634 operand cmpOpU()
5635 %{
5636 match(Bool);
5637
5638 format %{ "" %}
5639 interface(COND_INTER) %{
5640 equal(0x0, "eq");
5641 not_equal(0x1, "ne");
5642 less(0x3, "lo");
5643 greater_equal(0x2, "hs");
5644 less_equal(0x9, "ls");
5645 greater(0x8, "hi");
5646 overflow(0x6, "vs");
5647 no_overflow(0x7, "vc");
5648 %}
5649 %}
5650
5651 // used for certain integral comparisons which can be
5652 // converted to cbxx or tbxx instructions
5653
5654 operand cmpOpEqNe()
5655 %{
5656 match(Bool);
5657 op_cost(0);
5658 predicate(n->as_Bool()->_test._test == BoolTest::ne
5659 || n->as_Bool()->_test._test == BoolTest::eq);
5660
5661 format %{ "" %}
5662 interface(COND_INTER) %{
5663 equal(0x0, "eq");
5664 not_equal(0x1, "ne");
5665 less(0xb, "lt");
5666 greater_equal(0xa, "ge");
5667 less_equal(0xd, "le");
5668 greater(0xc, "gt");
5669 overflow(0x6, "vs");
5670 no_overflow(0x7, "vc");
5671 %}
5672 %}
5673
5674 // used for certain integral comparisons which can be
5675 // converted to cbxx or tbxx instructions
5676
5677 operand cmpOpLtGe()
5678 %{
5679 match(Bool);
5680 op_cost(0);
5681
5682 predicate(n->as_Bool()->_test._test == BoolTest::lt
5683 || n->as_Bool()->_test._test == BoolTest::ge);
5684
5685 format %{ "" %}
5686 interface(COND_INTER) %{
5687 equal(0x0, "eq");
5688 not_equal(0x1, "ne");
5689 less(0xb, "lt");
5690 greater_equal(0xa, "ge");
5691 less_equal(0xd, "le");
5692 greater(0xc, "gt");
5693 overflow(0x6, "vs");
5694 no_overflow(0x7, "vc");
5695 %}
5696 %}
5697
5698 // used for certain unsigned integral comparisons which can be
5699 // converted to cbxx or tbxx instructions
5700
5701 operand cmpOpUEqNeLeGt()
5702 %{
5703 match(Bool);
5704 op_cost(0);
5705
5706 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5707 n->as_Bool()->_test._test == BoolTest::ne ||
5708 n->as_Bool()->_test._test == BoolTest::le ||
5709 n->as_Bool()->_test._test == BoolTest::gt);
5710
5711 format %{ "" %}
5712 interface(COND_INTER) %{
5713 equal(0x0, "eq");
5714 not_equal(0x1, "ne");
5715 less(0x3, "lo");
5716 greater_equal(0x2, "hs");
5717 less_equal(0x9, "ls");
5718 greater(0x8, "hi");
5719 overflow(0x6, "vs");
5720 no_overflow(0x7, "vc");
5721 %}
5722 %}
5723
5724 // Special operand allowing long args to int ops to be truncated for free
5725
5726 operand iRegL2I(iRegL reg) %{
5727
5728 op_cost(0);
5729
5730 match(ConvL2I reg);
5731
5732 format %{ "l2i($reg)" %}
5733
5734 interface(REG_INTER)
5735 %}
5736
5737 operand iRegL2P(iRegL reg) %{
5738
5739 op_cost(0);
5740
5741 match(CastX2P reg);
5742
5743 format %{ "l2p($reg)" %}
5744
5745 interface(REG_INTER)
5746 %}
5747
5748 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5749 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5750 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5751 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5752
5753 //----------OPERAND CLASSES----------------------------------------------------
5754 // Operand Classes are groups of operands that are used as to simplify
5755 // instruction definitions by not requiring the AD writer to specify
5756 // separate instructions for every form of operand when the
5757 // instruction accepts multiple operand types with the same basic
5758 // encoding and format. The classic case of this is memory operands.
5759
5760 // memory is used to define read/write location for load/store
5761 // instruction defs. we can turn a memory op into an Address
5762
5763 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5764 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5765
5766 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5767 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5768
5769 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5770 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5771
5772 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5773 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5774
5775 // All of the memory operands. For the pipeline description.
5776 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5777 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5778 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5779
5780
5781 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5782 // operations. it allows the src to be either an iRegI or a (ConvL2I
5783 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5784 // can be elided because the 32-bit instruction will just employ the
5785 // lower 32 bits anyway.
5786 //
5787 // n.b. this does not elide all L2I conversions. if the truncated
5788 // value is consumed by more than one operation then the ConvL2I
5789 // cannot be bundled into the consuming nodes so an l2i gets planted
5790 // (actually a movw $dst $src) and the downstream instructions consume
5791 // the result of the l2i as an iRegI input. That's a shame since the
5792 // movw is actually redundant but its not too costly.
5793
5794 opclass iRegIorL2I(iRegI, iRegL2I);
5795 opclass iRegPorL2P(iRegP, iRegL2P);
5796
5797 //----------PIPELINE-----------------------------------------------------------
5798 // Rules which define the behavior of the target architectures pipeline.
5799
5800 // For specific pipelines, eg A53, define the stages of that pipeline
5801 //pipe_desc(ISS, EX1, EX2, WR);
5802 #define ISS S0
5803 #define EX1 S1
5804 #define EX2 S2
5805 #define WR S3
5806
5807 // Integer ALU reg operation
5808 pipeline %{
5809
5810 attributes %{
5811 // ARM instructions are of fixed length
5812 fixed_size_instructions; // Fixed size instructions TODO does
5813 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5814 // ARM instructions come in 32-bit word units
5815 instruction_unit_size = 4; // An instruction is 4 bytes long
5816 instruction_fetch_unit_size = 64; // The processor fetches one line
5817 instruction_fetch_units = 1; // of 64 bytes
5818
5819 // List of nop instructions
5820 nops( MachNop );
5821 %}
5822
5823 // We don't use an actual pipeline model so don't care about resources
5824 // or description. we do use pipeline classes to introduce fixed
5825 // latencies
5826
5827 //----------RESOURCES----------------------------------------------------------
5828 // Resources are the functional units available to the machine
5829
5830 resources( INS0, INS1, INS01 = INS0 | INS1,
5831 ALU0, ALU1, ALU = ALU0 | ALU1,
5832 MAC,
5833 DIV,
5834 BRANCH,
5835 LDST,
5836 NEON_FP);
5837
5838 //----------PIPELINE DESCRIPTION-----------------------------------------------
5839 // Pipeline Description specifies the stages in the machine's pipeline
5840
5841 // Define the pipeline as a generic 6 stage pipeline
5842 pipe_desc(S0, S1, S2, S3, S4, S5);
5843
5844 //----------PIPELINE CLASSES---------------------------------------------------
5845 // Pipeline Classes describe the stages in which input and output are
5846 // referenced by the hardware pipeline.
5847
5848 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5849 %{
5850 single_instruction;
5851 src1 : S1(read);
5852 src2 : S2(read);
5853 dst : S5(write);
5854 INS01 : ISS;
5855 NEON_FP : S5;
5856 %}
5857
5858 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5859 %{
5860 single_instruction;
5861 src1 : S1(read);
5862 src2 : S2(read);
5863 dst : S5(write);
5864 INS01 : ISS;
5865 NEON_FP : S5;
5866 %}
5867
5868 pipe_class fp_uop_s(vRegF dst, vRegF src)
5869 %{
5870 single_instruction;
5871 src : S1(read);
5872 dst : S5(write);
5873 INS01 : ISS;
5874 NEON_FP : S5;
5875 %}
5876
5877 pipe_class fp_uop_d(vRegD dst, vRegD src)
5878 %{
5879 single_instruction;
5880 src : S1(read);
5881 dst : S5(write);
5882 INS01 : ISS;
5883 NEON_FP : S5;
5884 %}
5885
5886 pipe_class fp_d2f(vRegF dst, vRegD src)
5887 %{
5888 single_instruction;
5889 src : S1(read);
5890 dst : S5(write);
5891 INS01 : ISS;
5892 NEON_FP : S5;
5893 %}
5894
5895 pipe_class fp_f2d(vRegD dst, vRegF src)
5896 %{
5897 single_instruction;
5898 src : S1(read);
5899 dst : S5(write);
5900 INS01 : ISS;
5901 NEON_FP : S5;
5902 %}
5903
5904 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5905 %{
5906 single_instruction;
5907 src : S1(read);
5908 dst : S5(write);
5909 INS01 : ISS;
5910 NEON_FP : S5;
5911 %}
5912
5913 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5914 %{
5915 single_instruction;
5916 src : S1(read);
5917 dst : S5(write);
5918 INS01 : ISS;
5919 NEON_FP : S5;
5920 %}
5921
5922 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5923 %{
5924 single_instruction;
5925 src : S1(read);
5926 dst : S5(write);
5927 INS01 : ISS;
5928 NEON_FP : S5;
5929 %}
5930
5931 pipe_class fp_l2f(vRegF dst, iRegL src)
5932 %{
5933 single_instruction;
5934 src : S1(read);
5935 dst : S5(write);
5936 INS01 : ISS;
5937 NEON_FP : S5;
5938 %}
5939
5940 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5941 %{
5942 single_instruction;
5943 src : S1(read);
5944 dst : S5(write);
5945 INS01 : ISS;
5946 NEON_FP : S5;
5947 %}
5948
5949 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5950 %{
5951 single_instruction;
5952 src : S1(read);
5953 dst : S5(write);
5954 INS01 : ISS;
5955 NEON_FP : S5;
5956 %}
5957
5958 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5959 %{
5960 single_instruction;
5961 src : S1(read);
5962 dst : S5(write);
5963 INS01 : ISS;
5964 NEON_FP : S5;
5965 %}
5966
5967 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5968 %{
5969 single_instruction;
5970 src : S1(read);
5971 dst : S5(write);
5972 INS01 : ISS;
5973 NEON_FP : S5;
5974 %}
5975
5976 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5977 %{
5978 single_instruction;
5979 src1 : S1(read);
5980 src2 : S2(read);
5981 dst : S5(write);
5982 INS0 : ISS;
5983 NEON_FP : S5;
5984 %}
5985
5986 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
5987 %{
5988 single_instruction;
5989 src1 : S1(read);
5990 src2 : S2(read);
5991 dst : S5(write);
5992 INS0 : ISS;
5993 NEON_FP : S5;
5994 %}
5995
5996 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
5997 %{
5998 single_instruction;
5999 cr : S1(read);
6000 src1 : S1(read);
6001 src2 : S1(read);
6002 dst : S3(write);
6003 INS01 : ISS;
6004 NEON_FP : S3;
6005 %}
6006
6007 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6008 %{
6009 single_instruction;
6010 cr : S1(read);
6011 src1 : S1(read);
6012 src2 : S1(read);
6013 dst : S3(write);
6014 INS01 : ISS;
6015 NEON_FP : S3;
6016 %}
6017
6018 pipe_class fp_imm_s(vRegF dst)
6019 %{
6020 single_instruction;
6021 dst : S3(write);
6022 INS01 : ISS;
6023 NEON_FP : S3;
6024 %}
6025
6026 pipe_class fp_imm_d(vRegD dst)
6027 %{
6028 single_instruction;
6029 dst : S3(write);
6030 INS01 : ISS;
6031 NEON_FP : S3;
6032 %}
6033
6034 pipe_class fp_load_constant_s(vRegF dst)
6035 %{
6036 single_instruction;
6037 dst : S4(write);
6038 INS01 : ISS;
6039 NEON_FP : S4;
6040 %}
6041
6042 pipe_class fp_load_constant_d(vRegD dst)
6043 %{
6044 single_instruction;
6045 dst : S4(write);
6046 INS01 : ISS;
6047 NEON_FP : S4;
6048 %}
6049
6050 //------- Integer ALU operations --------------------------
6051
6052 // Integer ALU reg-reg operation
6053 // Operands needed in EX1, result generated in EX2
6054 // Eg. ADD x0, x1, x2
6055 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6056 %{
6057 single_instruction;
6058 dst : EX2(write);
6059 src1 : EX1(read);
6060 src2 : EX1(read);
6061 INS01 : ISS; // Dual issue as instruction 0 or 1
6062 ALU : EX2;
6063 %}
6064
6065 // Integer ALU reg-reg operation with constant shift
6066 // Shifted register must be available in LATE_ISS instead of EX1
6067 // Eg. ADD x0, x1, x2, LSL #2
6068 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6069 %{
6070 single_instruction;
6071 dst : EX2(write);
6072 src1 : EX1(read);
6073 src2 : ISS(read);
6074 INS01 : ISS;
6075 ALU : EX2;
6076 %}
6077
6078 // Integer ALU reg operation with constant shift
6079 // Eg. LSL x0, x1, #shift
6080 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6081 %{
6082 single_instruction;
6083 dst : EX2(write);
6084 src1 : ISS(read);
6085 INS01 : ISS;
6086 ALU : EX2;
6087 %}
6088
6089 // Integer ALU reg-reg operation with variable shift
6090 // Both operands must be available in LATE_ISS instead of EX1
6091 // Result is available in EX1 instead of EX2
6092 // Eg. LSLV x0, x1, x2
6093 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6094 %{
6095 single_instruction;
6096 dst : EX1(write);
6097 src1 : ISS(read);
6098 src2 : ISS(read);
6099 INS01 : ISS;
6100 ALU : EX1;
6101 %}
6102
6103 // Integer ALU reg-reg operation with extract
6104 // As for _vshift above, but result generated in EX2
6105 // Eg. EXTR x0, x1, x2, #N
6106 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6107 %{
6108 single_instruction;
6109 dst : EX2(write);
6110 src1 : ISS(read);
6111 src2 : ISS(read);
6112 INS1 : ISS; // Can only dual issue as Instruction 1
6113 ALU : EX1;
6114 %}
6115
6116 // Integer ALU reg operation
6117 // Eg. NEG x0, x1
6118 pipe_class ialu_reg(iRegI dst, iRegI src)
6119 %{
6120 single_instruction;
6121 dst : EX2(write);
6122 src : EX1(read);
6123 INS01 : ISS;
6124 ALU : EX2;
6125 %}
6126
6127 // Integer ALU reg mmediate operation
6128 // Eg. ADD x0, x1, #N
6129 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6130 %{
6131 single_instruction;
6132 dst : EX2(write);
6133 src1 : EX1(read);
6134 INS01 : ISS;
6135 ALU : EX2;
6136 %}
6137
6138 // Integer ALU immediate operation (no source operands)
6139 // Eg. MOV x0, #N
6140 pipe_class ialu_imm(iRegI dst)
6141 %{
6142 single_instruction;
6143 dst : EX1(write);
6144 INS01 : ISS;
6145 ALU : EX1;
6146 %}
6147
6148 //------- Compare operation -------------------------------
6149
6150 // Compare reg-reg
6151 // Eg. CMP x0, x1
6152 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6153 %{
6154 single_instruction;
6155 // fixed_latency(16);
6156 cr : EX2(write);
6157 op1 : EX1(read);
6158 op2 : EX1(read);
6159 INS01 : ISS;
6160 ALU : EX2;
6161 %}
6162
6163 // Compare reg-reg
6164 // Eg. CMP x0, #N
6165 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6166 %{
6167 single_instruction;
6168 // fixed_latency(16);
6169 cr : EX2(write);
6170 op1 : EX1(read);
6171 INS01 : ISS;
6172 ALU : EX2;
6173 %}
6174
6175 //------- Conditional instructions ------------------------
6176
6177 // Conditional no operands
6178 // Eg. CSINC x0, zr, zr, <cond>
6179 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6180 %{
6181 single_instruction;
6182 cr : EX1(read);
6183 dst : EX2(write);
6184 INS01 : ISS;
6185 ALU : EX2;
6186 %}
6187
6188 // Conditional 2 operand
6189 // EG. CSEL X0, X1, X2, <cond>
6190 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6191 %{
6192 single_instruction;
6193 cr : EX1(read);
6194 src1 : EX1(read);
6195 src2 : EX1(read);
6196 dst : EX2(write);
6197 INS01 : ISS;
6198 ALU : EX2;
6199 %}
6200
6201 // Conditional 2 operand
6202 // EG. CSEL X0, X1, X2, <cond>
6203 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6204 %{
6205 single_instruction;
6206 cr : EX1(read);
6207 src : EX1(read);
6208 dst : EX2(write);
6209 INS01 : ISS;
6210 ALU : EX2;
6211 %}
6212
6213 //------- Multiply pipeline operations --------------------
6214
6215 // Multiply reg-reg
6216 // Eg. MUL w0, w1, w2
6217 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6218 %{
6219 single_instruction;
6220 dst : WR(write);
6221 src1 : ISS(read);
6222 src2 : ISS(read);
6223 INS01 : ISS;
6224 MAC : WR;
6225 %}
6226
6227 // Multiply accumulate
6228 // Eg. MADD w0, w1, w2, w3
6229 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6230 %{
6231 single_instruction;
6232 dst : WR(write);
6233 src1 : ISS(read);
6234 src2 : ISS(read);
6235 src3 : ISS(read);
6236 INS01 : ISS;
6237 MAC : WR;
6238 %}
6239
6240 // Eg. MUL w0, w1, w2
6241 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6242 %{
6243 single_instruction;
6244 fixed_latency(3); // Maximum latency for 64 bit mul
6245 dst : WR(write);
6246 src1 : ISS(read);
6247 src2 : ISS(read);
6248 INS01 : ISS;
6249 MAC : WR;
6250 %}
6251
6252 // Multiply accumulate
6253 // Eg. MADD w0, w1, w2, w3
6254 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6255 %{
6256 single_instruction;
6257 fixed_latency(3); // Maximum latency for 64 bit mul
6258 dst : WR(write);
6259 src1 : ISS(read);
6260 src2 : ISS(read);
6261 src3 : ISS(read);
6262 INS01 : ISS;
6263 MAC : WR;
6264 %}
6265
6266 //------- Divide pipeline operations --------------------
6267
6268 // Eg. SDIV w0, w1, w2
6269 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6270 %{
6271 single_instruction;
6272 fixed_latency(8); // Maximum latency for 32 bit divide
6273 dst : WR(write);
6274 src1 : ISS(read);
6275 src2 : ISS(read);
6276 INS0 : ISS; // Can only dual issue as instruction 0
6277 DIV : WR;
6278 %}
6279
6280 // Eg. SDIV x0, x1, x2
6281 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6282 %{
6283 single_instruction;
6284 fixed_latency(16); // Maximum latency for 64 bit divide
6285 dst : WR(write);
6286 src1 : ISS(read);
6287 src2 : ISS(read);
6288 INS0 : ISS; // Can only dual issue as instruction 0
6289 DIV : WR;
6290 %}
6291
6292 //------- Load pipeline operations ------------------------
6293
6294 // Load - prefetch
6295 // Eg. PFRM <mem>
6296 pipe_class iload_prefetch(memory mem)
6297 %{
6298 single_instruction;
6299 mem : ISS(read);
6300 INS01 : ISS;
6301 LDST : WR;
6302 %}
6303
6304 // Load - reg, mem
6305 // Eg. LDR x0, <mem>
6306 pipe_class iload_reg_mem(iRegI dst, memory mem)
6307 %{
6308 single_instruction;
6309 dst : WR(write);
6310 mem : ISS(read);
6311 INS01 : ISS;
6312 LDST : WR;
6313 %}
6314
6315 // Load - reg, reg
6316 // Eg. LDR x0, [sp, x1]
6317 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6318 %{
6319 single_instruction;
6320 dst : WR(write);
6321 src : ISS(read);
6322 INS01 : ISS;
6323 LDST : WR;
6324 %}
6325
6326 //------- Store pipeline operations -----------------------
6327
6328 // Store - zr, mem
6329 // Eg. STR zr, <mem>
6330 pipe_class istore_mem(memory mem)
6331 %{
6332 single_instruction;
6333 mem : ISS(read);
6334 INS01 : ISS;
6335 LDST : WR;
6336 %}
6337
6338 // Store - reg, mem
6339 // Eg. STR x0, <mem>
6340 pipe_class istore_reg_mem(iRegI src, memory mem)
6341 %{
6342 single_instruction;
6343 mem : ISS(read);
6344 src : EX2(read);
6345 INS01 : ISS;
6346 LDST : WR;
6347 %}
6348
6349 // Store - reg, reg
6350 // Eg. STR x0, [sp, x1]
6351 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6352 %{
6353 single_instruction;
6354 dst : ISS(read);
6355 src : EX2(read);
6356 INS01 : ISS;
6357 LDST : WR;
6358 %}
6359
6360 //------- Store pipeline operations -----------------------
6361
6362 // Branch
6363 pipe_class pipe_branch()
6364 %{
6365 single_instruction;
6366 INS01 : ISS;
6367 BRANCH : EX1;
6368 %}
6369
6370 // Conditional branch
6371 pipe_class pipe_branch_cond(rFlagsReg cr)
6372 %{
6373 single_instruction;
6374 cr : EX1(read);
6375 INS01 : ISS;
6376 BRANCH : EX1;
6377 %}
6378
6379 // Compare & Branch
6380 // EG. CBZ/CBNZ
6381 pipe_class pipe_cmp_branch(iRegI op1)
6382 %{
6383 single_instruction;
6384 op1 : EX1(read);
6385 INS01 : ISS;
6386 BRANCH : EX1;
6387 %}
6388
6389 //------- Synchronisation operations ----------------------
6390
6391 // Any operation requiring serialization.
6392 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6393 pipe_class pipe_serial()
6394 %{
6395 single_instruction;
6396 force_serialization;
6397 fixed_latency(16);
6398 INS01 : ISS(2); // Cannot dual issue with any other instruction
6399 LDST : WR;
6400 %}
6401
6402 // Generic big/slow expanded idiom - also serialized
6403 pipe_class pipe_slow()
6404 %{
6405 instruction_count(10);
6406 multiple_bundles;
6407 force_serialization;
6408 fixed_latency(16);
6409 INS01 : ISS(2); // Cannot dual issue with any other instruction
6410 LDST : WR;
6411 %}
6412
6413 // Empty pipeline class
6414 pipe_class pipe_class_empty()
6415 %{
6416 single_instruction;
6417 fixed_latency(0);
6418 %}
6419
6420 // Default pipeline class.
6421 pipe_class pipe_class_default()
6422 %{
6423 single_instruction;
6424 fixed_latency(2);
6425 %}
6426
6427 // Pipeline class for compares.
6428 pipe_class pipe_class_compare()
6429 %{
6430 single_instruction;
6431 fixed_latency(16);
6432 %}
6433
6434 // Pipeline class for memory operations.
6435 pipe_class pipe_class_memory()
6436 %{
6437 single_instruction;
6438 fixed_latency(16);
6439 %}
6440
6441 // Pipeline class for call.
6442 pipe_class pipe_class_call()
6443 %{
6444 single_instruction;
6445 fixed_latency(100);
6446 %}
6447
6448 // Define the class for the Nop node.
6449 define %{
6450 MachNop = pipe_class_empty;
6451 %}
6452
6453 %}
6454 //----------INSTRUCTIONS-------------------------------------------------------
6455 //
6456 // match -- States which machine-independent subtree may be replaced
6457 // by this instruction.
6458 // ins_cost -- The estimated cost of this instruction is used by instruction
6459 // selection to identify a minimum cost tree of machine
6460 // instructions that matches a tree of machine-independent
6461 // instructions.
6462 // format -- A string providing the disassembly for this instruction.
6463 // The value of an instruction's operand may be inserted
6464 // by referring to it with a '$' prefix.
6465 // opcode -- Three instruction opcodes may be provided. These are referred
6466 // to within an encode class as $primary, $secondary, and $tertiary
6467 // rrspectively. The primary opcode is commonly used to
6468 // indicate the type of machine instruction, while secondary
6469 // and tertiary are often used for prefix options or addressing
6470 // modes.
6471 // ins_encode -- A list of encode classes with parameters. The encode class
6472 // name must have been defined in an 'enc_class' specification
6473 // in the encode section of the architecture description.
6474
6475 // ============================================================================
6476 // Memory (Load/Store) Instructions
6477
6478 // Load Instructions
6479
6480 // Load Byte (8 bit signed)
6481 instruct loadB(iRegINoSp dst, memory1 mem)
6482 %{
6483 match(Set dst (LoadB mem));
6484 predicate(!needs_acquiring_load(n));
6485
6486 ins_cost(4 * INSN_COST);
6487 format %{ "ldrsbw $dst, $mem\t# byte" %}
6488
6489 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6490
6491 ins_pipe(iload_reg_mem);
6492 %}
6493
6494 // Load Byte (8 bit signed) into long
6495 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6496 %{
6497 match(Set dst (ConvI2L (LoadB mem)));
6498 predicate(!needs_acquiring_load(n->in(1)));
6499
6500 ins_cost(4 * INSN_COST);
6501 format %{ "ldrsb $dst, $mem\t# byte" %}
6502
6503 ins_encode(aarch64_enc_ldrsb(dst, mem));
6504
6505 ins_pipe(iload_reg_mem);
6506 %}
6507
6508 // Load Byte (8 bit unsigned)
6509 instruct loadUB(iRegINoSp dst, memory1 mem)
6510 %{
6511 match(Set dst (LoadUB mem));
6512 predicate(!needs_acquiring_load(n));
6513
6514 ins_cost(4 * INSN_COST);
6515 format %{ "ldrbw $dst, $mem\t# byte" %}
6516
6517 ins_encode(aarch64_enc_ldrb(dst, mem));
6518
6519 ins_pipe(iload_reg_mem);
6520 %}
6521
6522 // Load Byte (8 bit unsigned) into long
6523 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6524 %{
6525 match(Set dst (ConvI2L (LoadUB mem)));
6526 predicate(!needs_acquiring_load(n->in(1)));
6527
6528 ins_cost(4 * INSN_COST);
6529 format %{ "ldrb $dst, $mem\t# byte" %}
6530
6531 ins_encode(aarch64_enc_ldrb(dst, mem));
6532
6533 ins_pipe(iload_reg_mem);
6534 %}
6535
6536 // Load Short (16 bit signed)
6537 instruct loadS(iRegINoSp dst, memory2 mem)
6538 %{
6539 match(Set dst (LoadS mem));
6540 predicate(!needs_acquiring_load(n));
6541
6542 ins_cost(4 * INSN_COST);
6543 format %{ "ldrshw $dst, $mem\t# short" %}
6544
6545 ins_encode(aarch64_enc_ldrshw(dst, mem));
6546
6547 ins_pipe(iload_reg_mem);
6548 %}
6549
6550 // Load Short (16 bit signed) into long
6551 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6552 %{
6553 match(Set dst (ConvI2L (LoadS mem)));
6554 predicate(!needs_acquiring_load(n->in(1)));
6555
6556 ins_cost(4 * INSN_COST);
6557 format %{ "ldrsh $dst, $mem\t# short" %}
6558
6559 ins_encode(aarch64_enc_ldrsh(dst, mem));
6560
6561 ins_pipe(iload_reg_mem);
6562 %}
6563
6564 // Load Char (16 bit unsigned)
6565 instruct loadUS(iRegINoSp dst, memory2 mem)
6566 %{
6567 match(Set dst (LoadUS mem));
6568 predicate(!needs_acquiring_load(n));
6569
6570 ins_cost(4 * INSN_COST);
6571 format %{ "ldrh $dst, $mem\t# short" %}
6572
6573 ins_encode(aarch64_enc_ldrh(dst, mem));
6574
6575 ins_pipe(iload_reg_mem);
6576 %}
6577
6578 // Load Short/Char (16 bit unsigned) into long
6579 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6580 %{
6581 match(Set dst (ConvI2L (LoadUS mem)));
6582 predicate(!needs_acquiring_load(n->in(1)));
6583
6584 ins_cost(4 * INSN_COST);
6585 format %{ "ldrh $dst, $mem\t# short" %}
6586
6587 ins_encode(aarch64_enc_ldrh(dst, mem));
6588
6589 ins_pipe(iload_reg_mem);
6590 %}
6591
6592 // Load Integer (32 bit signed)
6593 instruct loadI(iRegINoSp dst, memory4 mem)
6594 %{
6595 match(Set dst (LoadI mem));
6596 predicate(!needs_acquiring_load(n));
6597
6598 ins_cost(4 * INSN_COST);
6599 format %{ "ldrw $dst, $mem\t# int" %}
6600
6601 ins_encode(aarch64_enc_ldrw(dst, mem));
6602
6603 ins_pipe(iload_reg_mem);
6604 %}
6605
6606 // Load Integer (32 bit signed) into long
6607 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6608 %{
6609 match(Set dst (ConvI2L (LoadI mem)));
6610 predicate(!needs_acquiring_load(n->in(1)));
6611
6612 ins_cost(4 * INSN_COST);
6613 format %{ "ldrsw $dst, $mem\t# int" %}
6614
6615 ins_encode(aarch64_enc_ldrsw(dst, mem));
6616
6617 ins_pipe(iload_reg_mem);
6618 %}
6619
6620 // Load Integer (32 bit unsigned) into long
6621 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6622 %{
6623 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6624 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6625
6626 ins_cost(4 * INSN_COST);
6627 format %{ "ldrw $dst, $mem\t# int" %}
6628
6629 ins_encode(aarch64_enc_ldrw(dst, mem));
6630
6631 ins_pipe(iload_reg_mem);
6632 %}
6633
6634 // Load Long (64 bit signed)
6635 instruct loadL(iRegLNoSp dst, memory8 mem)
6636 %{
6637 match(Set dst (LoadL mem));
6638 predicate(!needs_acquiring_load(n));
6639
6640 ins_cost(4 * INSN_COST);
6641 format %{ "ldr $dst, $mem\t# int" %}
6642
6643 ins_encode(aarch64_enc_ldr(dst, mem));
6644
6645 ins_pipe(iload_reg_mem);
6646 %}
6647
6648 // Load Range
6649 instruct loadRange(iRegINoSp dst, memory4 mem)
6650 %{
6651 match(Set dst (LoadRange mem));
6652
6653 ins_cost(4 * INSN_COST);
6654 format %{ "ldrw $dst, $mem\t# range" %}
6655
6656 ins_encode(aarch64_enc_ldrw(dst, mem));
6657
6658 ins_pipe(iload_reg_mem);
6659 %}
6660
6661 // Load Pointer
6662 instruct loadP(iRegPNoSp dst, memory8 mem)
6663 %{
6664 match(Set dst (LoadP mem));
6665 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6666
6667 ins_cost(4 * INSN_COST);
6668 format %{ "ldr $dst, $mem\t# ptr" %}
6669
6670 ins_encode(aarch64_enc_ldr(dst, mem));
6671
6672 ins_pipe(iload_reg_mem);
6673 %}
6674
6675 // Load Compressed Pointer
6676 instruct loadN(iRegNNoSp dst, memory4 mem)
6677 %{
6678 match(Set dst (LoadN mem));
6679 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6680
6681 ins_cost(4 * INSN_COST);
6682 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6683
6684 ins_encode(aarch64_enc_ldrw(dst, mem));
6685
6686 ins_pipe(iload_reg_mem);
6687 %}
6688
6689 // Load Klass Pointer
6690 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6691 %{
6692 match(Set dst (LoadKlass mem));
6693 predicate(!needs_acquiring_load(n));
6694
6695 ins_cost(4 * INSN_COST);
6696 format %{ "ldr $dst, $mem\t# class" %}
6697
6698 ins_encode(aarch64_enc_ldr(dst, mem));
6699
6700 ins_pipe(iload_reg_mem);
6701 %}
6702
6703 // Load Narrow Klass Pointer
6704 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6705 %{
6706 match(Set dst (LoadNKlass mem));
6707 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6708
6709 ins_cost(4 * INSN_COST);
6710 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6711
6712 ins_encode(aarch64_enc_ldrw(dst, mem));
6713
6714 ins_pipe(iload_reg_mem);
6715 %}
6716
6717 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6718 %{
6719 match(Set dst (LoadNKlass mem));
6720 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6721
6722 ins_cost(4 * INSN_COST);
6723 format %{
6724 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6725 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6726 %}
6727 ins_encode %{
6728 // inlined aarch64_enc_ldrw
6729 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6730 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6731 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6732 %}
6733 ins_pipe(iload_reg_mem);
6734 %}
6735
6736 // Load Float
6737 instruct loadF(vRegF dst, memory4 mem)
6738 %{
6739 match(Set dst (LoadF mem));
6740 predicate(!needs_acquiring_load(n));
6741
6742 ins_cost(4 * INSN_COST);
6743 format %{ "ldrs $dst, $mem\t# float" %}
6744
6745 ins_encode( aarch64_enc_ldrs(dst, mem) );
6746
6747 ins_pipe(pipe_class_memory);
6748 %}
6749
6750 // Load Double
6751 instruct loadD(vRegD dst, memory8 mem)
6752 %{
6753 match(Set dst (LoadD mem));
6754 predicate(!needs_acquiring_load(n));
6755
6756 ins_cost(4 * INSN_COST);
6757 format %{ "ldrd $dst, $mem\t# double" %}
6758
6759 ins_encode( aarch64_enc_ldrd(dst, mem) );
6760
6761 ins_pipe(pipe_class_memory);
6762 %}
6763
6764
6765 // Load Int Constant
6766 instruct loadConI(iRegINoSp dst, immI src)
6767 %{
6768 match(Set dst src);
6769
6770 ins_cost(INSN_COST);
6771 format %{ "mov $dst, $src\t# int" %}
6772
6773 ins_encode( aarch64_enc_movw_imm(dst, src) );
6774
6775 ins_pipe(ialu_imm);
6776 %}
6777
6778 // Load Long Constant
6779 instruct loadConL(iRegLNoSp dst, immL src)
6780 %{
6781 match(Set dst src);
6782
6783 ins_cost(INSN_COST);
6784 format %{ "mov $dst, $src\t# long" %}
6785
6786 ins_encode( aarch64_enc_mov_imm(dst, src) );
6787
6788 ins_pipe(ialu_imm);
6789 %}
6790
6791 // Load Pointer Constant
6792
6793 instruct loadConP(iRegPNoSp dst, immP con)
6794 %{
6795 match(Set dst con);
6796
6797 ins_cost(INSN_COST * 4);
6798 format %{
6799 "mov $dst, $con\t# ptr\n\t"
6800 %}
6801
6802 ins_encode(aarch64_enc_mov_p(dst, con));
6803
6804 ins_pipe(ialu_imm);
6805 %}
6806
6807 // Load Null Pointer Constant
6808
6809 instruct loadConP0(iRegPNoSp dst, immP0 con)
6810 %{
6811 match(Set dst con);
6812
6813 ins_cost(INSN_COST);
6814 format %{ "mov $dst, $con\t# nullptr ptr" %}
6815
6816 ins_encode(aarch64_enc_mov_p0(dst, con));
6817
6818 ins_pipe(ialu_imm);
6819 %}
6820
6821 // Load Pointer Constant One
6822
6823 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6824 %{
6825 match(Set dst con);
6826
6827 ins_cost(INSN_COST);
6828 format %{ "mov $dst, $con\t# nullptr ptr" %}
6829
6830 ins_encode(aarch64_enc_mov_p1(dst, con));
6831
6832 ins_pipe(ialu_imm);
6833 %}
6834
6835 // Load Byte Map Base Constant
6836
6837 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6838 %{
6839 match(Set dst con);
6840
6841 ins_cost(INSN_COST);
6842 format %{ "adr $dst, $con\t# Byte Map Base" %}
6843
6844 ins_encode %{
6845 __ load_byte_map_base($dst$$Register);
6846 %}
6847
6848 ins_pipe(ialu_imm);
6849 %}
6850
6851 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6852 %{
6853 match(Set dst con);
6854
6855 ins_cost(INSN_COST);
6856 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6857
6858 ins_encode %{
6859 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6860 %}
6861
6862 ins_pipe(ialu_imm);
6863 %}
6864
6865 // Load Narrow Pointer Constant
6866
6867 instruct loadConN(iRegNNoSp dst, immN con)
6868 %{
6869 match(Set dst con);
6870
6871 ins_cost(INSN_COST * 4);
6872 format %{ "mov $dst, $con\t# compressed ptr" %}
6873
6874 ins_encode(aarch64_enc_mov_n(dst, con));
6875
6876 ins_pipe(ialu_imm);
6877 %}
6878
6879 // Load Narrow Null Pointer Constant
6880
6881 instruct loadConN0(iRegNNoSp dst, immN0 con)
6882 %{
6883 match(Set dst con);
6884
6885 ins_cost(INSN_COST);
6886 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6887
6888 ins_encode(aarch64_enc_mov_n0(dst, con));
6889
6890 ins_pipe(ialu_imm);
6891 %}
6892
6893 // Load Narrow Klass Constant
6894
6895 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6896 %{
6897 match(Set dst con);
6898
6899 ins_cost(INSN_COST);
6900 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6901
6902 ins_encode(aarch64_enc_mov_nk(dst, con));
6903
6904 ins_pipe(ialu_imm);
6905 %}
6906
6907 // Load Packed Float Constant
6908
6909 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6910 match(Set dst con);
6911 ins_cost(INSN_COST * 4);
6912 format %{ "fmovs $dst, $con"%}
6913 ins_encode %{
6914 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6915 %}
6916
6917 ins_pipe(fp_imm_s);
6918 %}
6919
6920 // Load Float Constant
6921
6922 instruct loadConF(vRegF dst, immF con) %{
6923 match(Set dst con);
6924
6925 ins_cost(INSN_COST * 4);
6926
6927 format %{
6928 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6929 %}
6930
6931 ins_encode %{
6932 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6933 %}
6934
6935 ins_pipe(fp_load_constant_s);
6936 %}
6937
6938 // Load Packed Double Constant
6939
6940 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6941 match(Set dst con);
6942 ins_cost(INSN_COST);
6943 format %{ "fmovd $dst, $con"%}
6944 ins_encode %{
6945 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6946 %}
6947
6948 ins_pipe(fp_imm_d);
6949 %}
6950
6951 // Load Double Constant
6952
6953 instruct loadConD(vRegD dst, immD con) %{
6954 match(Set dst con);
6955
6956 ins_cost(INSN_COST * 5);
6957 format %{
6958 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6959 %}
6960
6961 ins_encode %{
6962 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6963 %}
6964
6965 ins_pipe(fp_load_constant_d);
6966 %}
6967
6968 // Store Instructions
6969
6970 // Store Byte
6971 instruct storeB(iRegIorL2I src, memory1 mem)
6972 %{
6973 match(Set mem (StoreB mem src));
6974 predicate(!needs_releasing_store(n));
6975
6976 ins_cost(INSN_COST);
6977 format %{ "strb $src, $mem\t# byte" %}
6978
6979 ins_encode(aarch64_enc_strb(src, mem));
6980
6981 ins_pipe(istore_reg_mem);
6982 %}
6983
6984
6985 instruct storeimmB0(immI0 zero, memory1 mem)
6986 %{
6987 match(Set mem (StoreB mem zero));
6988 predicate(!needs_releasing_store(n));
6989
6990 ins_cost(INSN_COST);
6991 format %{ "strb rscractch2, $mem\t# byte" %}
6992
6993 ins_encode(aarch64_enc_strb0(mem));
6994
6995 ins_pipe(istore_mem);
6996 %}
6997
6998 // Store Char/Short
6999 instruct storeC(iRegIorL2I src, memory2 mem)
7000 %{
7001 match(Set mem (StoreC mem src));
7002 predicate(!needs_releasing_store(n));
7003
7004 ins_cost(INSN_COST);
7005 format %{ "strh $src, $mem\t# short" %}
7006
7007 ins_encode(aarch64_enc_strh(src, mem));
7008
7009 ins_pipe(istore_reg_mem);
7010 %}
7011
7012 instruct storeimmC0(immI0 zero, memory2 mem)
7013 %{
7014 match(Set mem (StoreC mem zero));
7015 predicate(!needs_releasing_store(n));
7016
7017 ins_cost(INSN_COST);
7018 format %{ "strh zr, $mem\t# short" %}
7019
7020 ins_encode(aarch64_enc_strh0(mem));
7021
7022 ins_pipe(istore_mem);
7023 %}
7024
7025 // Store Integer
7026
7027 instruct storeI(iRegIorL2I src, memory4 mem)
7028 %{
7029 match(Set mem(StoreI mem src));
7030 predicate(!needs_releasing_store(n));
7031
7032 ins_cost(INSN_COST);
7033 format %{ "strw $src, $mem\t# int" %}
7034
7035 ins_encode(aarch64_enc_strw(src, mem));
7036
7037 ins_pipe(istore_reg_mem);
7038 %}
7039
7040 instruct storeimmI0(immI0 zero, memory4 mem)
7041 %{
7042 match(Set mem(StoreI mem zero));
7043 predicate(!needs_releasing_store(n));
7044
7045 ins_cost(INSN_COST);
7046 format %{ "strw zr, $mem\t# int" %}
7047
7048 ins_encode(aarch64_enc_strw0(mem));
7049
7050 ins_pipe(istore_mem);
7051 %}
7052
7053 // Store Long (64 bit signed)
7054 instruct storeL(iRegL src, memory8 mem)
7055 %{
7056 match(Set mem (StoreL mem src));
7057 predicate(!needs_releasing_store(n));
7058
7059 ins_cost(INSN_COST);
7060 format %{ "str $src, $mem\t# int" %}
7061
7062 ins_encode(aarch64_enc_str(src, mem));
7063
7064 ins_pipe(istore_reg_mem);
7065 %}
7066
7067 // Store Long (64 bit signed)
7068 instruct storeimmL0(immL0 zero, memory8 mem)
7069 %{
7070 match(Set mem (StoreL mem zero));
7071 predicate(!needs_releasing_store(n));
7072
7073 ins_cost(INSN_COST);
7074 format %{ "str zr, $mem\t# int" %}
7075
7076 ins_encode(aarch64_enc_str0(mem));
7077
7078 ins_pipe(istore_mem);
7079 %}
7080
7081 // Store Pointer
7082 instruct storeP(iRegP src, memory8 mem)
7083 %{
7084 match(Set mem (StoreP mem src));
7085 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7086
7087 ins_cost(INSN_COST);
7088 format %{ "str $src, $mem\t# ptr" %}
7089
7090 ins_encode(aarch64_enc_str(src, mem));
7091
7092 ins_pipe(istore_reg_mem);
7093 %}
7094
7095 // Store Pointer
7096 instruct storeimmP0(immP0 zero, memory8 mem)
7097 %{
7098 match(Set mem (StoreP mem zero));
7099 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7100
7101 ins_cost(INSN_COST);
7102 format %{ "str zr, $mem\t# ptr" %}
7103
7104 ins_encode(aarch64_enc_str0(mem));
7105
7106 ins_pipe(istore_mem);
7107 %}
7108
7109 // Store Compressed Pointer
7110 instruct storeN(iRegN src, memory4 mem)
7111 %{
7112 match(Set mem (StoreN mem src));
7113 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7114
7115 ins_cost(INSN_COST);
7116 format %{ "strw $src, $mem\t# compressed ptr" %}
7117
7118 ins_encode(aarch64_enc_strw(src, mem));
7119
7120 ins_pipe(istore_reg_mem);
7121 %}
7122
7123 instruct storeImmN0(immN0 zero, memory4 mem)
7124 %{
7125 match(Set mem (StoreN mem zero));
7126 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7127
7128 ins_cost(INSN_COST);
7129 format %{ "strw zr, $mem\t# compressed ptr" %}
7130
7131 ins_encode(aarch64_enc_strw0(mem));
7132
7133 ins_pipe(istore_mem);
7134 %}
7135
7136 // Store Float
7137 instruct storeF(vRegF src, memory4 mem)
7138 %{
7139 match(Set mem (StoreF mem src));
7140 predicate(!needs_releasing_store(n));
7141
7142 ins_cost(INSN_COST);
7143 format %{ "strs $src, $mem\t# float" %}
7144
7145 ins_encode( aarch64_enc_strs(src, mem) );
7146
7147 ins_pipe(pipe_class_memory);
7148 %}
7149
7150 // TODO
7151 // implement storeImmF0 and storeFImmPacked
7152
7153 // Store Double
7154 instruct storeD(vRegD src, memory8 mem)
7155 %{
7156 match(Set mem (StoreD mem src));
7157 predicate(!needs_releasing_store(n));
7158
7159 ins_cost(INSN_COST);
7160 format %{ "strd $src, $mem\t# double" %}
7161
7162 ins_encode( aarch64_enc_strd(src, mem) );
7163
7164 ins_pipe(pipe_class_memory);
7165 %}
7166
7167 // Store Compressed Klass Pointer
7168 instruct storeNKlass(iRegN src, memory4 mem)
7169 %{
7170 predicate(!needs_releasing_store(n));
7171 match(Set mem (StoreNKlass mem src));
7172
7173 ins_cost(INSN_COST);
7174 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7175
7176 ins_encode(aarch64_enc_strw(src, mem));
7177
7178 ins_pipe(istore_reg_mem);
7179 %}
7180
7181 // TODO
7182 // implement storeImmD0 and storeDImmPacked
7183
7184 // prefetch instructions
7185 // Must be safe to execute with invalid address (cannot fault).
7186
7187 instruct prefetchalloc( memory8 mem ) %{
7188 match(PrefetchAllocation mem);
7189
7190 ins_cost(INSN_COST);
7191 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7192
7193 ins_encode( aarch64_enc_prefetchw(mem) );
7194
7195 ins_pipe(iload_prefetch);
7196 %}
7197
7198 // ---------------- volatile loads and stores ----------------
7199
7200 // Load Byte (8 bit signed)
7201 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7202 %{
7203 match(Set dst (LoadB mem));
7204
7205 ins_cost(VOLATILE_REF_COST);
7206 format %{ "ldarsb $dst, $mem\t# byte" %}
7207
7208 ins_encode(aarch64_enc_ldarsb(dst, mem));
7209
7210 ins_pipe(pipe_serial);
7211 %}
7212
7213 // Load Byte (8 bit signed) into long
7214 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7215 %{
7216 match(Set dst (ConvI2L (LoadB mem)));
7217
7218 ins_cost(VOLATILE_REF_COST);
7219 format %{ "ldarsb $dst, $mem\t# byte" %}
7220
7221 ins_encode(aarch64_enc_ldarsb(dst, mem));
7222
7223 ins_pipe(pipe_serial);
7224 %}
7225
7226 // Load Byte (8 bit unsigned)
7227 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7228 %{
7229 match(Set dst (LoadUB mem));
7230
7231 ins_cost(VOLATILE_REF_COST);
7232 format %{ "ldarb $dst, $mem\t# byte" %}
7233
7234 ins_encode(aarch64_enc_ldarb(dst, mem));
7235
7236 ins_pipe(pipe_serial);
7237 %}
7238
7239 // Load Byte (8 bit unsigned) into long
7240 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7241 %{
7242 match(Set dst (ConvI2L (LoadUB mem)));
7243
7244 ins_cost(VOLATILE_REF_COST);
7245 format %{ "ldarb $dst, $mem\t# byte" %}
7246
7247 ins_encode(aarch64_enc_ldarb(dst, mem));
7248
7249 ins_pipe(pipe_serial);
7250 %}
7251
7252 // Load Short (16 bit signed)
7253 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7254 %{
7255 match(Set dst (LoadS mem));
7256
7257 ins_cost(VOLATILE_REF_COST);
7258 format %{ "ldarshw $dst, $mem\t# short" %}
7259
7260 ins_encode(aarch64_enc_ldarshw(dst, mem));
7261
7262 ins_pipe(pipe_serial);
7263 %}
7264
7265 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7266 %{
7267 match(Set dst (LoadUS mem));
7268
7269 ins_cost(VOLATILE_REF_COST);
7270 format %{ "ldarhw $dst, $mem\t# short" %}
7271
7272 ins_encode(aarch64_enc_ldarhw(dst, mem));
7273
7274 ins_pipe(pipe_serial);
7275 %}
7276
7277 // Load Short/Char (16 bit unsigned) into long
7278 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7279 %{
7280 match(Set dst (ConvI2L (LoadUS mem)));
7281
7282 ins_cost(VOLATILE_REF_COST);
7283 format %{ "ldarh $dst, $mem\t# short" %}
7284
7285 ins_encode(aarch64_enc_ldarh(dst, mem));
7286
7287 ins_pipe(pipe_serial);
7288 %}
7289
7290 // Load Short/Char (16 bit signed) into long
7291 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7292 %{
7293 match(Set dst (ConvI2L (LoadS mem)));
7294
7295 ins_cost(VOLATILE_REF_COST);
7296 format %{ "ldarh $dst, $mem\t# short" %}
7297
7298 ins_encode(aarch64_enc_ldarsh(dst, mem));
7299
7300 ins_pipe(pipe_serial);
7301 %}
7302
7303 // Load Integer (32 bit signed)
7304 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7305 %{
7306 match(Set dst (LoadI mem));
7307
7308 ins_cost(VOLATILE_REF_COST);
7309 format %{ "ldarw $dst, $mem\t# int" %}
7310
7311 ins_encode(aarch64_enc_ldarw(dst, mem));
7312
7313 ins_pipe(pipe_serial);
7314 %}
7315
7316 // Load Integer (32 bit unsigned) into long
7317 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7318 %{
7319 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7320
7321 ins_cost(VOLATILE_REF_COST);
7322 format %{ "ldarw $dst, $mem\t# int" %}
7323
7324 ins_encode(aarch64_enc_ldarw(dst, mem));
7325
7326 ins_pipe(pipe_serial);
7327 %}
7328
7329 // Load Long (64 bit signed)
7330 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7331 %{
7332 match(Set dst (LoadL mem));
7333
7334 ins_cost(VOLATILE_REF_COST);
7335 format %{ "ldar $dst, $mem\t# int" %}
7336
7337 ins_encode(aarch64_enc_ldar(dst, mem));
7338
7339 ins_pipe(pipe_serial);
7340 %}
7341
7342 // Load Pointer
7343 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7344 %{
7345 match(Set dst (LoadP mem));
7346 predicate(n->as_Load()->barrier_data() == 0);
7347
7348 ins_cost(VOLATILE_REF_COST);
7349 format %{ "ldar $dst, $mem\t# ptr" %}
7350
7351 ins_encode(aarch64_enc_ldar(dst, mem));
7352
7353 ins_pipe(pipe_serial);
7354 %}
7355
7356 // Load Compressed Pointer
7357 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7358 %{
7359 match(Set dst (LoadN mem));
7360 predicate(n->as_Load()->barrier_data() == 0);
7361
7362 ins_cost(VOLATILE_REF_COST);
7363 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7364
7365 ins_encode(aarch64_enc_ldarw(dst, mem));
7366
7367 ins_pipe(pipe_serial);
7368 %}
7369
7370 // Load Float
7371 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7372 %{
7373 match(Set dst (LoadF mem));
7374
7375 ins_cost(VOLATILE_REF_COST);
7376 format %{ "ldars $dst, $mem\t# float" %}
7377
7378 ins_encode( aarch64_enc_fldars(dst, mem) );
7379
7380 ins_pipe(pipe_serial);
7381 %}
7382
7383 // Load Double
7384 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7385 %{
7386 match(Set dst (LoadD mem));
7387
7388 ins_cost(VOLATILE_REF_COST);
7389 format %{ "ldard $dst, $mem\t# double" %}
7390
7391 ins_encode( aarch64_enc_fldard(dst, mem) );
7392
7393 ins_pipe(pipe_serial);
7394 %}
7395
7396 // Store Byte
7397 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7398 %{
7399 match(Set mem (StoreB mem src));
7400
7401 ins_cost(VOLATILE_REF_COST);
7402 format %{ "stlrb $src, $mem\t# byte" %}
7403
7404 ins_encode(aarch64_enc_stlrb(src, mem));
7405
7406 ins_pipe(pipe_class_memory);
7407 %}
7408
7409 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7410 %{
7411 match(Set mem (StoreB mem zero));
7412
7413 ins_cost(VOLATILE_REF_COST);
7414 format %{ "stlrb zr, $mem\t# byte" %}
7415
7416 ins_encode(aarch64_enc_stlrb0(mem));
7417
7418 ins_pipe(pipe_class_memory);
7419 %}
7420
7421 // Store Char/Short
7422 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7423 %{
7424 match(Set mem (StoreC mem src));
7425
7426 ins_cost(VOLATILE_REF_COST);
7427 format %{ "stlrh $src, $mem\t# short" %}
7428
7429 ins_encode(aarch64_enc_stlrh(src, mem));
7430
7431 ins_pipe(pipe_class_memory);
7432 %}
7433
7434 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7435 %{
7436 match(Set mem (StoreC mem zero));
7437
7438 ins_cost(VOLATILE_REF_COST);
7439 format %{ "stlrh zr, $mem\t# short" %}
7440
7441 ins_encode(aarch64_enc_stlrh0(mem));
7442
7443 ins_pipe(pipe_class_memory);
7444 %}
7445
7446 // Store Integer
7447
7448 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7449 %{
7450 match(Set mem(StoreI mem src));
7451
7452 ins_cost(VOLATILE_REF_COST);
7453 format %{ "stlrw $src, $mem\t# int" %}
7454
7455 ins_encode(aarch64_enc_stlrw(src, mem));
7456
7457 ins_pipe(pipe_class_memory);
7458 %}
7459
7460 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7461 %{
7462 match(Set mem(StoreI mem zero));
7463
7464 ins_cost(VOLATILE_REF_COST);
7465 format %{ "stlrw zr, $mem\t# int" %}
7466
7467 ins_encode(aarch64_enc_stlrw0(mem));
7468
7469 ins_pipe(pipe_class_memory);
7470 %}
7471
7472 // Store Long (64 bit signed)
7473 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7474 %{
7475 match(Set mem (StoreL mem src));
7476
7477 ins_cost(VOLATILE_REF_COST);
7478 format %{ "stlr $src, $mem\t# int" %}
7479
7480 ins_encode(aarch64_enc_stlr(src, mem));
7481
7482 ins_pipe(pipe_class_memory);
7483 %}
7484
7485 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7486 %{
7487 match(Set mem (StoreL mem zero));
7488
7489 ins_cost(VOLATILE_REF_COST);
7490 format %{ "stlr zr, $mem\t# int" %}
7491
7492 ins_encode(aarch64_enc_stlr0(mem));
7493
7494 ins_pipe(pipe_class_memory);
7495 %}
7496
7497 // Store Pointer
7498 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7499 %{
7500 match(Set mem (StoreP mem src));
7501 predicate(n->as_Store()->barrier_data() == 0);
7502
7503 ins_cost(VOLATILE_REF_COST);
7504 format %{ "stlr $src, $mem\t# ptr" %}
7505
7506 ins_encode(aarch64_enc_stlr(src, mem));
7507
7508 ins_pipe(pipe_class_memory);
7509 %}
7510
7511 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7512 %{
7513 match(Set mem (StoreP mem zero));
7514 predicate(n->as_Store()->barrier_data() == 0);
7515
7516 ins_cost(VOLATILE_REF_COST);
7517 format %{ "stlr zr, $mem\t# ptr" %}
7518
7519 ins_encode(aarch64_enc_stlr0(mem));
7520
7521 ins_pipe(pipe_class_memory);
7522 %}
7523
7524 // Store Compressed Pointer
7525 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7526 %{
7527 match(Set mem (StoreN mem src));
7528 predicate(n->as_Store()->barrier_data() == 0);
7529
7530 ins_cost(VOLATILE_REF_COST);
7531 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7532
7533 ins_encode(aarch64_enc_stlrw(src, mem));
7534
7535 ins_pipe(pipe_class_memory);
7536 %}
7537
7538 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7539 %{
7540 match(Set mem (StoreN mem zero));
7541 predicate(n->as_Store()->barrier_data() == 0);
7542
7543 ins_cost(VOLATILE_REF_COST);
7544 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7545
7546 ins_encode(aarch64_enc_stlrw0(mem));
7547
7548 ins_pipe(pipe_class_memory);
7549 %}
7550
7551 // Store Float
7552 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7553 %{
7554 match(Set mem (StoreF mem src));
7555
7556 ins_cost(VOLATILE_REF_COST);
7557 format %{ "stlrs $src, $mem\t# float" %}
7558
7559 ins_encode( aarch64_enc_fstlrs(src, mem) );
7560
7561 ins_pipe(pipe_class_memory);
7562 %}
7563
7564 // TODO
7565 // implement storeImmF0 and storeFImmPacked
7566
7567 // Store Double
7568 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7569 %{
7570 match(Set mem (StoreD mem src));
7571
7572 ins_cost(VOLATILE_REF_COST);
7573 format %{ "stlrd $src, $mem\t# double" %}
7574
7575 ins_encode( aarch64_enc_fstlrd(src, mem) );
7576
7577 ins_pipe(pipe_class_memory);
7578 %}
7579
7580 // ---------------- end of volatile loads and stores ----------------
7581
7582 instruct cacheWB(indirect addr)
7583 %{
7584 predicate(VM_Version::supports_data_cache_line_flush());
7585 match(CacheWB addr);
7586
7587 ins_cost(100);
7588 format %{"cache wb $addr" %}
7589 ins_encode %{
7590 assert($addr->index_position() < 0, "should be");
7591 assert($addr$$disp == 0, "should be");
7592 __ cache_wb(Address($addr$$base$$Register, 0));
7593 %}
7594 ins_pipe(pipe_slow); // XXX
7595 %}
7596
7597 instruct cacheWBPreSync()
7598 %{
7599 predicate(VM_Version::supports_data_cache_line_flush());
7600 match(CacheWBPreSync);
7601
7602 ins_cost(100);
7603 format %{"cache wb presync" %}
7604 ins_encode %{
7605 __ cache_wbsync(true);
7606 %}
7607 ins_pipe(pipe_slow); // XXX
7608 %}
7609
7610 instruct cacheWBPostSync()
7611 %{
7612 predicate(VM_Version::supports_data_cache_line_flush());
7613 match(CacheWBPostSync);
7614
7615 ins_cost(100);
7616 format %{"cache wb postsync" %}
7617 ins_encode %{
7618 __ cache_wbsync(false);
7619 %}
7620 ins_pipe(pipe_slow); // XXX
7621 %}
7622
7623 // ============================================================================
7624 // BSWAP Instructions
7625
7626 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7627 match(Set dst (ReverseBytesI src));
7628
7629 ins_cost(INSN_COST);
7630 format %{ "revw $dst, $src" %}
7631
7632 ins_encode %{
7633 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7634 %}
7635
7636 ins_pipe(ialu_reg);
7637 %}
7638
7639 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7640 match(Set dst (ReverseBytesL src));
7641
7642 ins_cost(INSN_COST);
7643 format %{ "rev $dst, $src" %}
7644
7645 ins_encode %{
7646 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7647 %}
7648
7649 ins_pipe(ialu_reg);
7650 %}
7651
7652 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7653 match(Set dst (ReverseBytesUS src));
7654
7655 ins_cost(INSN_COST);
7656 format %{ "rev16w $dst, $src" %}
7657
7658 ins_encode %{
7659 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7660 %}
7661
7662 ins_pipe(ialu_reg);
7663 %}
7664
7665 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7666 match(Set dst (ReverseBytesS src));
7667
7668 ins_cost(INSN_COST);
7669 format %{ "rev16w $dst, $src\n\t"
7670 "sbfmw $dst, $dst, #0, #15" %}
7671
7672 ins_encode %{
7673 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7674 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7675 %}
7676
7677 ins_pipe(ialu_reg);
7678 %}
7679
7680 // ============================================================================
7681 // Zero Count Instructions
7682
7683 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7684 match(Set dst (CountLeadingZerosI src));
7685
7686 ins_cost(INSN_COST);
7687 format %{ "clzw $dst, $src" %}
7688 ins_encode %{
7689 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7690 %}
7691
7692 ins_pipe(ialu_reg);
7693 %}
7694
7695 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7696 match(Set dst (CountLeadingZerosL src));
7697
7698 ins_cost(INSN_COST);
7699 format %{ "clz $dst, $src" %}
7700 ins_encode %{
7701 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7702 %}
7703
7704 ins_pipe(ialu_reg);
7705 %}
7706
7707 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7708 match(Set dst (CountTrailingZerosI src));
7709
7710 ins_cost(INSN_COST * 2);
7711 format %{ "rbitw $dst, $src\n\t"
7712 "clzw $dst, $dst" %}
7713 ins_encode %{
7714 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7715 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7716 %}
7717
7718 ins_pipe(ialu_reg);
7719 %}
7720
7721 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7722 match(Set dst (CountTrailingZerosL src));
7723
7724 ins_cost(INSN_COST * 2);
7725 format %{ "rbit $dst, $src\n\t"
7726 "clz $dst, $dst" %}
7727 ins_encode %{
7728 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7729 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7730 %}
7731
7732 ins_pipe(ialu_reg);
7733 %}
7734
7735 //---------- Population Count Instructions -------------------------------------
7736 //
7737
7738 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7739 match(Set dst (PopCountI src));
7740 effect(TEMP tmp);
7741 ins_cost(INSN_COST * 13);
7742
7743 format %{ "movw $src, $src\n\t"
7744 "mov $tmp, $src\t# vector (1D)\n\t"
7745 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7746 "addv $tmp, $tmp\t# vector (8B)\n\t"
7747 "mov $dst, $tmp\t# vector (1D)" %}
7748 ins_encode %{
7749 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7750 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7751 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7752 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7753 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7754 %}
7755
7756 ins_pipe(pipe_class_default);
7757 %}
7758
7759 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7760 match(Set dst (PopCountI (LoadI mem)));
7761 effect(TEMP tmp);
7762 ins_cost(INSN_COST * 13);
7763
7764 format %{ "ldrs $tmp, $mem\n\t"
7765 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7766 "addv $tmp, $tmp\t# vector (8B)\n\t"
7767 "mov $dst, $tmp\t# vector (1D)" %}
7768 ins_encode %{
7769 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7770 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7771 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7772 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7773 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7774 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7775 %}
7776
7777 ins_pipe(pipe_class_default);
7778 %}
7779
7780 // Note: Long.bitCount(long) returns an int.
7781 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7782 match(Set dst (PopCountL src));
7783 effect(TEMP tmp);
7784 ins_cost(INSN_COST * 13);
7785
7786 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7787 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7788 "addv $tmp, $tmp\t# vector (8B)\n\t"
7789 "mov $dst, $tmp\t# vector (1D)" %}
7790 ins_encode %{
7791 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7792 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7793 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7794 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7795 %}
7796
7797 ins_pipe(pipe_class_default);
7798 %}
7799
7800 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7801 match(Set dst (PopCountL (LoadL mem)));
7802 effect(TEMP tmp);
7803 ins_cost(INSN_COST * 13);
7804
7805 format %{ "ldrd $tmp, $mem\n\t"
7806 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7807 "addv $tmp, $tmp\t# vector (8B)\n\t"
7808 "mov $dst, $tmp\t# vector (1D)" %}
7809 ins_encode %{
7810 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7811 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7812 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7813 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7814 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7815 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7816 %}
7817
7818 ins_pipe(pipe_class_default);
7819 %}
7820
7821 // ============================================================================
7822 // VerifyVectorAlignment Instruction
7823
7824 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7825 match(Set addr (VerifyVectorAlignment addr mask));
7826 effect(KILL cr);
7827 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7828 ins_encode %{
7829 Label Lskip;
7830 // check if masked bits of addr are zero
7831 __ tst($addr$$Register, $mask$$constant);
7832 __ br(Assembler::EQ, Lskip);
7833 __ stop("verify_vector_alignment found a misaligned vector memory access");
7834 __ bind(Lskip);
7835 %}
7836 ins_pipe(pipe_slow);
7837 %}
7838
7839 // ============================================================================
7840 // MemBar Instruction
7841
7842 instruct load_fence() %{
7843 match(LoadFence);
7844 ins_cost(VOLATILE_REF_COST);
7845
7846 format %{ "load_fence" %}
7847
7848 ins_encode %{
7849 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7850 %}
7851 ins_pipe(pipe_serial);
7852 %}
7853
7854 instruct unnecessary_membar_acquire() %{
7855 predicate(unnecessary_acquire(n));
7856 match(MemBarAcquire);
7857 ins_cost(0);
7858
7859 format %{ "membar_acquire (elided)" %}
7860
7861 ins_encode %{
7862 __ block_comment("membar_acquire (elided)");
7863 %}
7864
7865 ins_pipe(pipe_class_empty);
7866 %}
7867
7868 instruct membar_acquire() %{
7869 match(MemBarAcquire);
7870 ins_cost(VOLATILE_REF_COST);
7871
7872 format %{ "membar_acquire\n\t"
7873 "dmb ishld" %}
7874
7875 ins_encode %{
7876 __ block_comment("membar_acquire");
7877 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7878 %}
7879
7880 ins_pipe(pipe_serial);
7881 %}
7882
7883
7884 instruct membar_acquire_lock() %{
7885 match(MemBarAcquireLock);
7886 ins_cost(VOLATILE_REF_COST);
7887
7888 format %{ "membar_acquire_lock (elided)" %}
7889
7890 ins_encode %{
7891 __ block_comment("membar_acquire_lock (elided)");
7892 %}
7893
7894 ins_pipe(pipe_serial);
7895 %}
7896
7897 instruct store_fence() %{
7898 match(StoreFence);
7899 ins_cost(VOLATILE_REF_COST);
7900
7901 format %{ "store_fence" %}
7902
7903 ins_encode %{
7904 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7905 %}
7906 ins_pipe(pipe_serial);
7907 %}
7908
7909 instruct unnecessary_membar_release() %{
7910 predicate(unnecessary_release(n));
7911 match(MemBarRelease);
7912 ins_cost(0);
7913
7914 format %{ "membar_release (elided)" %}
7915
7916 ins_encode %{
7917 __ block_comment("membar_release (elided)");
7918 %}
7919 ins_pipe(pipe_serial);
7920 %}
7921
7922 instruct membar_release() %{
7923 match(MemBarRelease);
7924 ins_cost(VOLATILE_REF_COST);
7925
7926 format %{ "membar_release\n\t"
7927 "dmb ishst\n\tdmb ishld" %}
7928
7929 ins_encode %{
7930 __ block_comment("membar_release");
7931 // These will be merged if AlwaysMergeDMB is enabled.
7932 __ membar(Assembler::StoreStore);
7933 __ membar(Assembler::LoadStore);
7934 %}
7935 ins_pipe(pipe_serial);
7936 %}
7937
7938 instruct membar_storestore() %{
7939 match(MemBarStoreStore);
7940 match(StoreStoreFence);
7941 ins_cost(VOLATILE_REF_COST);
7942
7943 format %{ "MEMBAR-store-store" %}
7944
7945 ins_encode %{
7946 __ membar(Assembler::StoreStore);
7947 %}
7948 ins_pipe(pipe_serial);
7949 %}
7950
7951 instruct membar_release_lock() %{
7952 match(MemBarReleaseLock);
7953 ins_cost(VOLATILE_REF_COST);
7954
7955 format %{ "membar_release_lock (elided)" %}
7956
7957 ins_encode %{
7958 __ block_comment("membar_release_lock (elided)");
7959 %}
7960
7961 ins_pipe(pipe_serial);
7962 %}
7963
7964 instruct unnecessary_membar_volatile() %{
7965 predicate(unnecessary_volatile(n));
7966 match(MemBarVolatile);
7967 ins_cost(0);
7968
7969 format %{ "membar_volatile (elided)" %}
7970
7971 ins_encode %{
7972 __ block_comment("membar_volatile (elided)");
7973 %}
7974
7975 ins_pipe(pipe_serial);
7976 %}
7977
7978 instruct membar_volatile() %{
7979 match(MemBarVolatile);
7980 ins_cost(VOLATILE_REF_COST*100);
7981
7982 format %{ "membar_volatile\n\t"
7983 "dmb ish"%}
7984
7985 ins_encode %{
7986 __ block_comment("membar_volatile");
7987 __ membar(Assembler::StoreLoad);
7988 %}
7989
7990 ins_pipe(pipe_serial);
7991 %}
7992
7993 // ============================================================================
7994 // Cast/Convert Instructions
7995
7996 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7997 match(Set dst (CastX2P src));
7998
7999 ins_cost(INSN_COST);
8000 format %{ "mov $dst, $src\t# long -> ptr" %}
8001
8002 ins_encode %{
8003 if ($dst$$reg != $src$$reg) {
8004 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8005 }
8006 %}
8007
8008 ins_pipe(ialu_reg);
8009 %}
8010
8011 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8012 match(Set dst (CastP2X src));
8013
8014 ins_cost(INSN_COST);
8015 format %{ "mov $dst, $src\t# ptr -> long" %}
8016
8017 ins_encode %{
8018 if ($dst$$reg != $src$$reg) {
8019 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8020 }
8021 %}
8022
8023 ins_pipe(ialu_reg);
8024 %}
8025
8026 // Convert oop into int for vectors alignment masking
8027 instruct convP2I(iRegINoSp dst, iRegP src) %{
8028 match(Set dst (ConvL2I (CastP2X src)));
8029
8030 ins_cost(INSN_COST);
8031 format %{ "movw $dst, $src\t# ptr -> int" %}
8032 ins_encode %{
8033 __ movw($dst$$Register, $src$$Register);
8034 %}
8035
8036 ins_pipe(ialu_reg);
8037 %}
8038
8039 // Convert compressed oop into int for vectors alignment masking
8040 // in case of 32bit oops (heap < 4Gb).
8041 instruct convN2I(iRegINoSp dst, iRegN src)
8042 %{
8043 predicate(CompressedOops::shift() == 0);
8044 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8045
8046 ins_cost(INSN_COST);
8047 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8048 ins_encode %{
8049 __ movw($dst$$Register, $src$$Register);
8050 %}
8051
8052 ins_pipe(ialu_reg);
8053 %}
8054
8055
8056 // Convert oop pointer into compressed form
8057 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8058 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8059 match(Set dst (EncodeP src));
8060 effect(KILL cr);
8061 ins_cost(INSN_COST * 3);
8062 format %{ "encode_heap_oop $dst, $src" %}
8063 ins_encode %{
8064 Register s = $src$$Register;
8065 Register d = $dst$$Register;
8066 __ encode_heap_oop(d, s);
8067 %}
8068 ins_pipe(ialu_reg);
8069 %}
8070
8071 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8072 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8073 match(Set dst (EncodeP src));
8074 ins_cost(INSN_COST * 3);
8075 format %{ "encode_heap_oop_not_null $dst, $src" %}
8076 ins_encode %{
8077 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8078 %}
8079 ins_pipe(ialu_reg);
8080 %}
8081
8082 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8083 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8084 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8085 match(Set dst (DecodeN src));
8086 ins_cost(INSN_COST * 3);
8087 format %{ "decode_heap_oop $dst, $src" %}
8088 ins_encode %{
8089 Register s = $src$$Register;
8090 Register d = $dst$$Register;
8091 __ decode_heap_oop(d, s);
8092 %}
8093 ins_pipe(ialu_reg);
8094 %}
8095
8096 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8097 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8098 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8099 match(Set dst (DecodeN src));
8100 ins_cost(INSN_COST * 3);
8101 format %{ "decode_heap_oop_not_null $dst, $src" %}
8102 ins_encode %{
8103 Register s = $src$$Register;
8104 Register d = $dst$$Register;
8105 __ decode_heap_oop_not_null(d, s);
8106 %}
8107 ins_pipe(ialu_reg);
8108 %}
8109
8110 // n.b. AArch64 implementations of encode_klass_not_null and
8111 // decode_klass_not_null do not modify the flags register so, unlike
8112 // Intel, we don't kill CR as a side effect here
8113
8114 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8115 match(Set dst (EncodePKlass src));
8116
8117 ins_cost(INSN_COST * 3);
8118 format %{ "encode_klass_not_null $dst,$src" %}
8119
8120 ins_encode %{
8121 Register src_reg = as_Register($src$$reg);
8122 Register dst_reg = as_Register($dst$$reg);
8123 __ encode_klass_not_null(dst_reg, src_reg);
8124 %}
8125
8126 ins_pipe(ialu_reg);
8127 %}
8128
8129 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8130 match(Set dst (DecodeNKlass src));
8131
8132 ins_cost(INSN_COST * 3);
8133 format %{ "decode_klass_not_null $dst,$src" %}
8134
8135 ins_encode %{
8136 Register src_reg = as_Register($src$$reg);
8137 Register dst_reg = as_Register($dst$$reg);
8138 if (dst_reg != src_reg) {
8139 __ decode_klass_not_null(dst_reg, src_reg);
8140 } else {
8141 __ decode_klass_not_null(dst_reg);
8142 }
8143 %}
8144
8145 ins_pipe(ialu_reg);
8146 %}
8147
8148 instruct checkCastPP(iRegPNoSp dst)
8149 %{
8150 match(Set dst (CheckCastPP dst));
8151
8152 size(0);
8153 format %{ "# checkcastPP of $dst" %}
8154 ins_encode(/* empty encoding */);
8155 ins_pipe(pipe_class_empty);
8156 %}
8157
8158 instruct castPP(iRegPNoSp dst)
8159 %{
8160 match(Set dst (CastPP dst));
8161
8162 size(0);
8163 format %{ "# castPP of $dst" %}
8164 ins_encode(/* empty encoding */);
8165 ins_pipe(pipe_class_empty);
8166 %}
8167
8168 instruct castII(iRegI dst)
8169 %{
8170 match(Set dst (CastII dst));
8171
8172 size(0);
8173 format %{ "# castII of $dst" %}
8174 ins_encode(/* empty encoding */);
8175 ins_cost(0);
8176 ins_pipe(pipe_class_empty);
8177 %}
8178
8179 instruct castLL(iRegL dst)
8180 %{
8181 match(Set dst (CastLL dst));
8182
8183 size(0);
8184 format %{ "# castLL of $dst" %}
8185 ins_encode(/* empty encoding */);
8186 ins_cost(0);
8187 ins_pipe(pipe_class_empty);
8188 %}
8189
8190 instruct castFF(vRegF dst)
8191 %{
8192 match(Set dst (CastFF dst));
8193
8194 size(0);
8195 format %{ "# castFF of $dst" %}
8196 ins_encode(/* empty encoding */);
8197 ins_cost(0);
8198 ins_pipe(pipe_class_empty);
8199 %}
8200
8201 instruct castDD(vRegD dst)
8202 %{
8203 match(Set dst (CastDD dst));
8204
8205 size(0);
8206 format %{ "# castDD of $dst" %}
8207 ins_encode(/* empty encoding */);
8208 ins_cost(0);
8209 ins_pipe(pipe_class_empty);
8210 %}
8211
8212 instruct castVV(vReg dst)
8213 %{
8214 match(Set dst (CastVV dst));
8215
8216 size(0);
8217 format %{ "# castVV of $dst" %}
8218 ins_encode(/* empty encoding */);
8219 ins_cost(0);
8220 ins_pipe(pipe_class_empty);
8221 %}
8222
8223 instruct castVVMask(pRegGov dst)
8224 %{
8225 match(Set dst (CastVV dst));
8226
8227 size(0);
8228 format %{ "# castVV of $dst" %}
8229 ins_encode(/* empty encoding */);
8230 ins_cost(0);
8231 ins_pipe(pipe_class_empty);
8232 %}
8233
8234 // ============================================================================
8235 // Atomic operation instructions
8236 //
8237
8238 // standard CompareAndSwapX when we are using barriers
8239 // these have higher priority than the rules selected by a predicate
8240
8241 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8242 // can't match them
8243
8244 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8245
8246 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8247 ins_cost(2 * VOLATILE_REF_COST);
8248
8249 effect(KILL cr);
8250
8251 format %{
8252 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8253 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8254 %}
8255
8256 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8257 aarch64_enc_cset_eq(res));
8258
8259 ins_pipe(pipe_slow);
8260 %}
8261
8262 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8263
8264 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8265 ins_cost(2 * VOLATILE_REF_COST);
8266
8267 effect(KILL cr);
8268
8269 format %{
8270 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8271 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8272 %}
8273
8274 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8275 aarch64_enc_cset_eq(res));
8276
8277 ins_pipe(pipe_slow);
8278 %}
8279
8280 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8281
8282 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8283 ins_cost(2 * VOLATILE_REF_COST);
8284
8285 effect(KILL cr);
8286
8287 format %{
8288 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8289 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8290 %}
8291
8292 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8293 aarch64_enc_cset_eq(res));
8294
8295 ins_pipe(pipe_slow);
8296 %}
8297
8298 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8299
8300 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8301 ins_cost(2 * VOLATILE_REF_COST);
8302
8303 effect(KILL cr);
8304
8305 format %{
8306 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8307 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8308 %}
8309
8310 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8311 aarch64_enc_cset_eq(res));
8312
8313 ins_pipe(pipe_slow);
8314 %}
8315
8316 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8317
8318 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8319 predicate(n->as_LoadStore()->barrier_data() == 0);
8320 ins_cost(2 * VOLATILE_REF_COST);
8321
8322 effect(KILL cr);
8323
8324 format %{
8325 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8326 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8327 %}
8328
8329 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8330 aarch64_enc_cset_eq(res));
8331
8332 ins_pipe(pipe_slow);
8333 %}
8334
8335 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8336
8337 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8338 predicate(n->as_LoadStore()->barrier_data() == 0);
8339 ins_cost(2 * VOLATILE_REF_COST);
8340
8341 effect(KILL cr);
8342
8343 format %{
8344 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8345 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8346 %}
8347
8348 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8349 aarch64_enc_cset_eq(res));
8350
8351 ins_pipe(pipe_slow);
8352 %}
8353
8354 // alternative CompareAndSwapX when we are eliding barriers
8355
8356 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8357
8358 predicate(needs_acquiring_load_exclusive(n));
8359 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8360 ins_cost(VOLATILE_REF_COST);
8361
8362 effect(KILL cr);
8363
8364 format %{
8365 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8366 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8367 %}
8368
8369 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8370 aarch64_enc_cset_eq(res));
8371
8372 ins_pipe(pipe_slow);
8373 %}
8374
8375 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8376
8377 predicate(needs_acquiring_load_exclusive(n));
8378 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8379 ins_cost(VOLATILE_REF_COST);
8380
8381 effect(KILL cr);
8382
8383 format %{
8384 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8385 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8386 %}
8387
8388 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8389 aarch64_enc_cset_eq(res));
8390
8391 ins_pipe(pipe_slow);
8392 %}
8393
8394 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8395
8396 predicate(needs_acquiring_load_exclusive(n));
8397 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8398 ins_cost(VOLATILE_REF_COST);
8399
8400 effect(KILL cr);
8401
8402 format %{
8403 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8404 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8405 %}
8406
8407 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8408 aarch64_enc_cset_eq(res));
8409
8410 ins_pipe(pipe_slow);
8411 %}
8412
8413 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8414
8415 predicate(needs_acquiring_load_exclusive(n));
8416 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8417 ins_cost(VOLATILE_REF_COST);
8418
8419 effect(KILL cr);
8420
8421 format %{
8422 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8423 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8424 %}
8425
8426 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8427 aarch64_enc_cset_eq(res));
8428
8429 ins_pipe(pipe_slow);
8430 %}
8431
8432 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8433
8434 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8435 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8436 ins_cost(VOLATILE_REF_COST);
8437
8438 effect(KILL cr);
8439
8440 format %{
8441 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8442 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8443 %}
8444
8445 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8446 aarch64_enc_cset_eq(res));
8447
8448 ins_pipe(pipe_slow);
8449 %}
8450
8451 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8452
8453 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8454 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8455 ins_cost(VOLATILE_REF_COST);
8456
8457 effect(KILL cr);
8458
8459 format %{
8460 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8461 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8462 %}
8463
8464 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8465 aarch64_enc_cset_eq(res));
8466
8467 ins_pipe(pipe_slow);
8468 %}
8469
8470
8471 // ---------------------------------------------------------------------
8472
8473 // BEGIN This section of the file is automatically generated. Do not edit --------------
8474
8475 // Sundry CAS operations. Note that release is always true,
8476 // regardless of the memory ordering of the CAS. This is because we
8477 // need the volatile case to be sequentially consistent but there is
8478 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8479 // can't check the type of memory ordering here, so we always emit a
8480 // STLXR.
8481
8482 // This section is generated from cas.m4
8483
8484
8485 // This pattern is generated automatically from cas.m4.
8486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8487 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8488 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8489 ins_cost(2 * VOLATILE_REF_COST);
8490 effect(TEMP_DEF res, KILL cr);
8491 format %{
8492 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8493 %}
8494 ins_encode %{
8495 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8496 Assembler::byte, /*acquire*/ false, /*release*/ true,
8497 /*weak*/ false, $res$$Register);
8498 __ sxtbw($res$$Register, $res$$Register);
8499 %}
8500 ins_pipe(pipe_slow);
8501 %}
8502
8503 // This pattern is generated automatically from cas.m4.
8504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8505 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8506 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8507 ins_cost(2 * VOLATILE_REF_COST);
8508 effect(TEMP_DEF res, KILL cr);
8509 format %{
8510 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8511 %}
8512 ins_encode %{
8513 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8514 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8515 /*weak*/ false, $res$$Register);
8516 __ sxthw($res$$Register, $res$$Register);
8517 %}
8518 ins_pipe(pipe_slow);
8519 %}
8520
8521 // This pattern is generated automatically from cas.m4.
8522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8523 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8524 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8525 ins_cost(2 * VOLATILE_REF_COST);
8526 effect(TEMP_DEF res, KILL cr);
8527 format %{
8528 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8529 %}
8530 ins_encode %{
8531 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8532 Assembler::word, /*acquire*/ false, /*release*/ true,
8533 /*weak*/ false, $res$$Register);
8534 %}
8535 ins_pipe(pipe_slow);
8536 %}
8537
8538 // This pattern is generated automatically from cas.m4.
8539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8540 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8541 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8542 ins_cost(2 * VOLATILE_REF_COST);
8543 effect(TEMP_DEF res, KILL cr);
8544 format %{
8545 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8546 %}
8547 ins_encode %{
8548 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8549 Assembler::xword, /*acquire*/ false, /*release*/ true,
8550 /*weak*/ false, $res$$Register);
8551 %}
8552 ins_pipe(pipe_slow);
8553 %}
8554
8555 // This pattern is generated automatically from cas.m4.
8556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8557 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8558 predicate(n->as_LoadStore()->barrier_data() == 0);
8559 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8560 ins_cost(2 * VOLATILE_REF_COST);
8561 effect(TEMP_DEF res, KILL cr);
8562 format %{
8563 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8564 %}
8565 ins_encode %{
8566 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8567 Assembler::word, /*acquire*/ false, /*release*/ true,
8568 /*weak*/ false, $res$$Register);
8569 %}
8570 ins_pipe(pipe_slow);
8571 %}
8572
8573 // This pattern is generated automatically from cas.m4.
8574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8575 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8576 predicate(n->as_LoadStore()->barrier_data() == 0);
8577 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8578 ins_cost(2 * VOLATILE_REF_COST);
8579 effect(TEMP_DEF res, KILL cr);
8580 format %{
8581 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8582 %}
8583 ins_encode %{
8584 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8585 Assembler::xword, /*acquire*/ false, /*release*/ true,
8586 /*weak*/ false, $res$$Register);
8587 %}
8588 ins_pipe(pipe_slow);
8589 %}
8590
8591 // This pattern is generated automatically from cas.m4.
8592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8593 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8594 predicate(needs_acquiring_load_exclusive(n));
8595 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8596 ins_cost(VOLATILE_REF_COST);
8597 effect(TEMP_DEF res, KILL cr);
8598 format %{
8599 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8600 %}
8601 ins_encode %{
8602 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8603 Assembler::byte, /*acquire*/ true, /*release*/ true,
8604 /*weak*/ false, $res$$Register);
8605 __ sxtbw($res$$Register, $res$$Register);
8606 %}
8607 ins_pipe(pipe_slow);
8608 %}
8609
8610 // This pattern is generated automatically from cas.m4.
8611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8612 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8613 predicate(needs_acquiring_load_exclusive(n));
8614 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8615 ins_cost(VOLATILE_REF_COST);
8616 effect(TEMP_DEF res, KILL cr);
8617 format %{
8618 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8619 %}
8620 ins_encode %{
8621 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8622 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8623 /*weak*/ false, $res$$Register);
8624 __ sxthw($res$$Register, $res$$Register);
8625 %}
8626 ins_pipe(pipe_slow);
8627 %}
8628
8629 // This pattern is generated automatically from cas.m4.
8630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8631 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8632 predicate(needs_acquiring_load_exclusive(n));
8633 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8634 ins_cost(VOLATILE_REF_COST);
8635 effect(TEMP_DEF res, KILL cr);
8636 format %{
8637 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8638 %}
8639 ins_encode %{
8640 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8641 Assembler::word, /*acquire*/ true, /*release*/ true,
8642 /*weak*/ false, $res$$Register);
8643 %}
8644 ins_pipe(pipe_slow);
8645 %}
8646
8647 // This pattern is generated automatically from cas.m4.
8648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8649 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8650 predicate(needs_acquiring_load_exclusive(n));
8651 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8652 ins_cost(VOLATILE_REF_COST);
8653 effect(TEMP_DEF res, KILL cr);
8654 format %{
8655 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8656 %}
8657 ins_encode %{
8658 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8659 Assembler::xword, /*acquire*/ true, /*release*/ true,
8660 /*weak*/ false, $res$$Register);
8661 %}
8662 ins_pipe(pipe_slow);
8663 %}
8664
8665 // This pattern is generated automatically from cas.m4.
8666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8667 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8668 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8669 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8670 ins_cost(VOLATILE_REF_COST);
8671 effect(TEMP_DEF res, KILL cr);
8672 format %{
8673 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8674 %}
8675 ins_encode %{
8676 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8677 Assembler::word, /*acquire*/ true, /*release*/ true,
8678 /*weak*/ false, $res$$Register);
8679 %}
8680 ins_pipe(pipe_slow);
8681 %}
8682
8683 // This pattern is generated automatically from cas.m4.
8684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8685 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8686 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8687 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8688 ins_cost(VOLATILE_REF_COST);
8689 effect(TEMP_DEF res, KILL cr);
8690 format %{
8691 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8692 %}
8693 ins_encode %{
8694 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8695 Assembler::xword, /*acquire*/ true, /*release*/ true,
8696 /*weak*/ false, $res$$Register);
8697 %}
8698 ins_pipe(pipe_slow);
8699 %}
8700
8701 // This pattern is generated automatically from cas.m4.
8702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8703 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8704 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8705 ins_cost(2 * VOLATILE_REF_COST);
8706 effect(KILL cr);
8707 format %{
8708 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8709 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8710 %}
8711 ins_encode %{
8712 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8713 Assembler::byte, /*acquire*/ false, /*release*/ true,
8714 /*weak*/ true, noreg);
8715 __ csetw($res$$Register, Assembler::EQ);
8716 %}
8717 ins_pipe(pipe_slow);
8718 %}
8719
8720 // This pattern is generated automatically from cas.m4.
8721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8722 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8723 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8724 ins_cost(2 * VOLATILE_REF_COST);
8725 effect(KILL cr);
8726 format %{
8727 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8728 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8729 %}
8730 ins_encode %{
8731 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8732 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8733 /*weak*/ true, noreg);
8734 __ csetw($res$$Register, Assembler::EQ);
8735 %}
8736 ins_pipe(pipe_slow);
8737 %}
8738
8739 // This pattern is generated automatically from cas.m4.
8740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8741 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8742 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8743 ins_cost(2 * VOLATILE_REF_COST);
8744 effect(KILL cr);
8745 format %{
8746 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8747 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8748 %}
8749 ins_encode %{
8750 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8751 Assembler::word, /*acquire*/ false, /*release*/ true,
8752 /*weak*/ true, noreg);
8753 __ csetw($res$$Register, Assembler::EQ);
8754 %}
8755 ins_pipe(pipe_slow);
8756 %}
8757
8758 // This pattern is generated automatically from cas.m4.
8759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8760 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8761 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8762 ins_cost(2 * VOLATILE_REF_COST);
8763 effect(KILL cr);
8764 format %{
8765 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8766 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8767 %}
8768 ins_encode %{
8769 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8770 Assembler::xword, /*acquire*/ false, /*release*/ true,
8771 /*weak*/ true, noreg);
8772 __ csetw($res$$Register, Assembler::EQ);
8773 %}
8774 ins_pipe(pipe_slow);
8775 %}
8776
8777 // This pattern is generated automatically from cas.m4.
8778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8779 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8780 predicate(n->as_LoadStore()->barrier_data() == 0);
8781 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8782 ins_cost(2 * VOLATILE_REF_COST);
8783 effect(KILL cr);
8784 format %{
8785 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8786 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8787 %}
8788 ins_encode %{
8789 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8790 Assembler::word, /*acquire*/ false, /*release*/ true,
8791 /*weak*/ true, noreg);
8792 __ csetw($res$$Register, Assembler::EQ);
8793 %}
8794 ins_pipe(pipe_slow);
8795 %}
8796
8797 // This pattern is generated automatically from cas.m4.
8798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8799 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8800 predicate(n->as_LoadStore()->barrier_data() == 0);
8801 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8802 ins_cost(2 * VOLATILE_REF_COST);
8803 effect(KILL cr);
8804 format %{
8805 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8806 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8807 %}
8808 ins_encode %{
8809 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8810 Assembler::xword, /*acquire*/ false, /*release*/ true,
8811 /*weak*/ true, noreg);
8812 __ csetw($res$$Register, Assembler::EQ);
8813 %}
8814 ins_pipe(pipe_slow);
8815 %}
8816
8817 // This pattern is generated automatically from cas.m4.
8818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8819 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8820 predicate(needs_acquiring_load_exclusive(n));
8821 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8822 ins_cost(VOLATILE_REF_COST);
8823 effect(KILL cr);
8824 format %{
8825 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8826 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8827 %}
8828 ins_encode %{
8829 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8830 Assembler::byte, /*acquire*/ true, /*release*/ true,
8831 /*weak*/ true, noreg);
8832 __ csetw($res$$Register, Assembler::EQ);
8833 %}
8834 ins_pipe(pipe_slow);
8835 %}
8836
8837 // This pattern is generated automatically from cas.m4.
8838 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8839 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8840 predicate(needs_acquiring_load_exclusive(n));
8841 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8842 ins_cost(VOLATILE_REF_COST);
8843 effect(KILL cr);
8844 format %{
8845 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8846 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8847 %}
8848 ins_encode %{
8849 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8850 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8851 /*weak*/ true, noreg);
8852 __ csetw($res$$Register, Assembler::EQ);
8853 %}
8854 ins_pipe(pipe_slow);
8855 %}
8856
8857 // This pattern is generated automatically from cas.m4.
8858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8859 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8860 predicate(needs_acquiring_load_exclusive(n));
8861 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8862 ins_cost(VOLATILE_REF_COST);
8863 effect(KILL cr);
8864 format %{
8865 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8866 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8867 %}
8868 ins_encode %{
8869 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8870 Assembler::word, /*acquire*/ true, /*release*/ true,
8871 /*weak*/ true, noreg);
8872 __ csetw($res$$Register, Assembler::EQ);
8873 %}
8874 ins_pipe(pipe_slow);
8875 %}
8876
8877 // This pattern is generated automatically from cas.m4.
8878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8879 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8880 predicate(needs_acquiring_load_exclusive(n));
8881 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8882 ins_cost(VOLATILE_REF_COST);
8883 effect(KILL cr);
8884 format %{
8885 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8886 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8887 %}
8888 ins_encode %{
8889 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8890 Assembler::xword, /*acquire*/ true, /*release*/ true,
8891 /*weak*/ true, noreg);
8892 __ csetw($res$$Register, Assembler::EQ);
8893 %}
8894 ins_pipe(pipe_slow);
8895 %}
8896
8897 // This pattern is generated automatically from cas.m4.
8898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8899 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8900 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8901 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8902 ins_cost(VOLATILE_REF_COST);
8903 effect(KILL cr);
8904 format %{
8905 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8906 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8907 %}
8908 ins_encode %{
8909 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8910 Assembler::word, /*acquire*/ true, /*release*/ true,
8911 /*weak*/ true, noreg);
8912 __ csetw($res$$Register, Assembler::EQ);
8913 %}
8914 ins_pipe(pipe_slow);
8915 %}
8916
8917 // This pattern is generated automatically from cas.m4.
8918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8919 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8920 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8921 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8922 ins_cost(VOLATILE_REF_COST);
8923 effect(KILL cr);
8924 format %{
8925 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8926 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8927 %}
8928 ins_encode %{
8929 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8930 Assembler::xword, /*acquire*/ true, /*release*/ true,
8931 /*weak*/ true, noreg);
8932 __ csetw($res$$Register, Assembler::EQ);
8933 %}
8934 ins_pipe(pipe_slow);
8935 %}
8936
8937 // END This section of the file is automatically generated. Do not edit --------------
8938 // ---------------------------------------------------------------------
8939
8940 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8941 match(Set prev (GetAndSetI mem newv));
8942 ins_cost(2 * VOLATILE_REF_COST);
8943 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8944 ins_encode %{
8945 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8946 %}
8947 ins_pipe(pipe_serial);
8948 %}
8949
8950 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8951 match(Set prev (GetAndSetL mem newv));
8952 ins_cost(2 * VOLATILE_REF_COST);
8953 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8954 ins_encode %{
8955 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8956 %}
8957 ins_pipe(pipe_serial);
8958 %}
8959
8960 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8961 predicate(n->as_LoadStore()->barrier_data() == 0);
8962 match(Set prev (GetAndSetN mem newv));
8963 ins_cost(2 * VOLATILE_REF_COST);
8964 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8965 ins_encode %{
8966 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8967 %}
8968 ins_pipe(pipe_serial);
8969 %}
8970
8971 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
8972 predicate(n->as_LoadStore()->barrier_data() == 0);
8973 match(Set prev (GetAndSetP mem newv));
8974 ins_cost(2 * VOLATILE_REF_COST);
8975 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8976 ins_encode %{
8977 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8978 %}
8979 ins_pipe(pipe_serial);
8980 %}
8981
8982 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
8983 predicate(needs_acquiring_load_exclusive(n));
8984 match(Set prev (GetAndSetI mem newv));
8985 ins_cost(VOLATILE_REF_COST);
8986 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8987 ins_encode %{
8988 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8989 %}
8990 ins_pipe(pipe_serial);
8991 %}
8992
8993 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
8994 predicate(needs_acquiring_load_exclusive(n));
8995 match(Set prev (GetAndSetL mem newv));
8996 ins_cost(VOLATILE_REF_COST);
8997 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8998 ins_encode %{
8999 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9000 %}
9001 ins_pipe(pipe_serial);
9002 %}
9003
9004 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9005 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9006 match(Set prev (GetAndSetN mem newv));
9007 ins_cost(VOLATILE_REF_COST);
9008 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9009 ins_encode %{
9010 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9011 %}
9012 ins_pipe(pipe_serial);
9013 %}
9014
9015 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9016 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9017 match(Set prev (GetAndSetP mem newv));
9018 ins_cost(VOLATILE_REF_COST);
9019 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9020 ins_encode %{
9021 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9022 %}
9023 ins_pipe(pipe_serial);
9024 %}
9025
9026
9027 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9028 match(Set newval (GetAndAddL mem incr));
9029 ins_cost(2 * VOLATILE_REF_COST + 1);
9030 format %{ "get_and_addL $newval, [$mem], $incr" %}
9031 ins_encode %{
9032 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9033 %}
9034 ins_pipe(pipe_serial);
9035 %}
9036
9037 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9038 predicate(n->as_LoadStore()->result_not_used());
9039 match(Set dummy (GetAndAddL mem incr));
9040 ins_cost(2 * VOLATILE_REF_COST);
9041 format %{ "get_and_addL [$mem], $incr" %}
9042 ins_encode %{
9043 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9044 %}
9045 ins_pipe(pipe_serial);
9046 %}
9047
9048 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9049 match(Set newval (GetAndAddL mem incr));
9050 ins_cost(2 * VOLATILE_REF_COST + 1);
9051 format %{ "get_and_addL $newval, [$mem], $incr" %}
9052 ins_encode %{
9053 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9054 %}
9055 ins_pipe(pipe_serial);
9056 %}
9057
9058 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9059 predicate(n->as_LoadStore()->result_not_used());
9060 match(Set dummy (GetAndAddL mem incr));
9061 ins_cost(2 * VOLATILE_REF_COST);
9062 format %{ "get_and_addL [$mem], $incr" %}
9063 ins_encode %{
9064 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9065 %}
9066 ins_pipe(pipe_serial);
9067 %}
9068
9069 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9070 match(Set newval (GetAndAddI mem incr));
9071 ins_cost(2 * VOLATILE_REF_COST + 1);
9072 format %{ "get_and_addI $newval, [$mem], $incr" %}
9073 ins_encode %{
9074 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9075 %}
9076 ins_pipe(pipe_serial);
9077 %}
9078
9079 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9080 predicate(n->as_LoadStore()->result_not_used());
9081 match(Set dummy (GetAndAddI mem incr));
9082 ins_cost(2 * VOLATILE_REF_COST);
9083 format %{ "get_and_addI [$mem], $incr" %}
9084 ins_encode %{
9085 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9086 %}
9087 ins_pipe(pipe_serial);
9088 %}
9089
9090 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9091 match(Set newval (GetAndAddI mem incr));
9092 ins_cost(2 * VOLATILE_REF_COST + 1);
9093 format %{ "get_and_addI $newval, [$mem], $incr" %}
9094 ins_encode %{
9095 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9096 %}
9097 ins_pipe(pipe_serial);
9098 %}
9099
9100 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9101 predicate(n->as_LoadStore()->result_not_used());
9102 match(Set dummy (GetAndAddI mem incr));
9103 ins_cost(2 * VOLATILE_REF_COST);
9104 format %{ "get_and_addI [$mem], $incr" %}
9105 ins_encode %{
9106 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9107 %}
9108 ins_pipe(pipe_serial);
9109 %}
9110
9111 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9112 predicate(needs_acquiring_load_exclusive(n));
9113 match(Set newval (GetAndAddL mem incr));
9114 ins_cost(VOLATILE_REF_COST + 1);
9115 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9116 ins_encode %{
9117 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9118 %}
9119 ins_pipe(pipe_serial);
9120 %}
9121
9122 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9123 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9124 match(Set dummy (GetAndAddL mem incr));
9125 ins_cost(VOLATILE_REF_COST);
9126 format %{ "get_and_addL_acq [$mem], $incr" %}
9127 ins_encode %{
9128 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9129 %}
9130 ins_pipe(pipe_serial);
9131 %}
9132
9133 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9134 predicate(needs_acquiring_load_exclusive(n));
9135 match(Set newval (GetAndAddL mem incr));
9136 ins_cost(VOLATILE_REF_COST + 1);
9137 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9138 ins_encode %{
9139 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9140 %}
9141 ins_pipe(pipe_serial);
9142 %}
9143
9144 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9145 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9146 match(Set dummy (GetAndAddL mem incr));
9147 ins_cost(VOLATILE_REF_COST);
9148 format %{ "get_and_addL_acq [$mem], $incr" %}
9149 ins_encode %{
9150 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9151 %}
9152 ins_pipe(pipe_serial);
9153 %}
9154
9155 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9156 predicate(needs_acquiring_load_exclusive(n));
9157 match(Set newval (GetAndAddI mem incr));
9158 ins_cost(VOLATILE_REF_COST + 1);
9159 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9160 ins_encode %{
9161 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9162 %}
9163 ins_pipe(pipe_serial);
9164 %}
9165
9166 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9167 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9168 match(Set dummy (GetAndAddI mem incr));
9169 ins_cost(VOLATILE_REF_COST);
9170 format %{ "get_and_addI_acq [$mem], $incr" %}
9171 ins_encode %{
9172 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9173 %}
9174 ins_pipe(pipe_serial);
9175 %}
9176
9177 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9178 predicate(needs_acquiring_load_exclusive(n));
9179 match(Set newval (GetAndAddI mem incr));
9180 ins_cost(VOLATILE_REF_COST + 1);
9181 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9182 ins_encode %{
9183 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9184 %}
9185 ins_pipe(pipe_serial);
9186 %}
9187
9188 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9189 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9190 match(Set dummy (GetAndAddI mem incr));
9191 ins_cost(VOLATILE_REF_COST);
9192 format %{ "get_and_addI_acq [$mem], $incr" %}
9193 ins_encode %{
9194 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9195 %}
9196 ins_pipe(pipe_serial);
9197 %}
9198
9199 // Manifest a CmpU result in an integer register.
9200 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9201 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9202 %{
9203 match(Set dst (CmpU3 src1 src2));
9204 effect(KILL flags);
9205
9206 ins_cost(INSN_COST * 3);
9207 format %{
9208 "cmpw $src1, $src2\n\t"
9209 "csetw $dst, ne\n\t"
9210 "cnegw $dst, lo\t# CmpU3(reg)"
9211 %}
9212 ins_encode %{
9213 __ cmpw($src1$$Register, $src2$$Register);
9214 __ csetw($dst$$Register, Assembler::NE);
9215 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9216 %}
9217
9218 ins_pipe(pipe_class_default);
9219 %}
9220
9221 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9222 %{
9223 match(Set dst (CmpU3 src1 src2));
9224 effect(KILL flags);
9225
9226 ins_cost(INSN_COST * 3);
9227 format %{
9228 "subsw zr, $src1, $src2\n\t"
9229 "csetw $dst, ne\n\t"
9230 "cnegw $dst, lo\t# CmpU3(imm)"
9231 %}
9232 ins_encode %{
9233 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9234 __ csetw($dst$$Register, Assembler::NE);
9235 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9236 %}
9237
9238 ins_pipe(pipe_class_default);
9239 %}
9240
9241 // Manifest a CmpUL result in an integer register.
9242 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9243 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9244 %{
9245 match(Set dst (CmpUL3 src1 src2));
9246 effect(KILL flags);
9247
9248 ins_cost(INSN_COST * 3);
9249 format %{
9250 "cmp $src1, $src2\n\t"
9251 "csetw $dst, ne\n\t"
9252 "cnegw $dst, lo\t# CmpUL3(reg)"
9253 %}
9254 ins_encode %{
9255 __ cmp($src1$$Register, $src2$$Register);
9256 __ csetw($dst$$Register, Assembler::NE);
9257 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9258 %}
9259
9260 ins_pipe(pipe_class_default);
9261 %}
9262
9263 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9264 %{
9265 match(Set dst (CmpUL3 src1 src2));
9266 effect(KILL flags);
9267
9268 ins_cost(INSN_COST * 3);
9269 format %{
9270 "subs zr, $src1, $src2\n\t"
9271 "csetw $dst, ne\n\t"
9272 "cnegw $dst, lo\t# CmpUL3(imm)"
9273 %}
9274 ins_encode %{
9275 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9276 __ csetw($dst$$Register, Assembler::NE);
9277 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9278 %}
9279
9280 ins_pipe(pipe_class_default);
9281 %}
9282
9283 // Manifest a CmpL result in an integer register.
9284 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9285 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9286 %{
9287 match(Set dst (CmpL3 src1 src2));
9288 effect(KILL flags);
9289
9290 ins_cost(INSN_COST * 3);
9291 format %{
9292 "cmp $src1, $src2\n\t"
9293 "csetw $dst, ne\n\t"
9294 "cnegw $dst, lt\t# CmpL3(reg)"
9295 %}
9296 ins_encode %{
9297 __ cmp($src1$$Register, $src2$$Register);
9298 __ csetw($dst$$Register, Assembler::NE);
9299 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9300 %}
9301
9302 ins_pipe(pipe_class_default);
9303 %}
9304
9305 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9306 %{
9307 match(Set dst (CmpL3 src1 src2));
9308 effect(KILL flags);
9309
9310 ins_cost(INSN_COST * 3);
9311 format %{
9312 "subs zr, $src1, $src2\n\t"
9313 "csetw $dst, ne\n\t"
9314 "cnegw $dst, lt\t# CmpL3(imm)"
9315 %}
9316 ins_encode %{
9317 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9318 __ csetw($dst$$Register, Assembler::NE);
9319 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9320 %}
9321
9322 ins_pipe(pipe_class_default);
9323 %}
9324
9325 // ============================================================================
9326 // Conditional Move Instructions
9327
9328 // n.b. we have identical rules for both a signed compare op (cmpOp)
9329 // and an unsigned compare op (cmpOpU). it would be nice if we could
9330 // define an op class which merged both inputs and use it to type the
9331 // argument to a single rule. unfortunatelyt his fails because the
9332 // opclass does not live up to the COND_INTER interface of its
9333 // component operands. When the generic code tries to negate the
9334 // operand it ends up running the generci Machoper::negate method
9335 // which throws a ShouldNotHappen. So, we have to provide two flavours
9336 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9337
9338 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9339 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9340
9341 ins_cost(INSN_COST * 2);
9342 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9343
9344 ins_encode %{
9345 __ cselw(as_Register($dst$$reg),
9346 as_Register($src2$$reg),
9347 as_Register($src1$$reg),
9348 (Assembler::Condition)$cmp$$cmpcode);
9349 %}
9350
9351 ins_pipe(icond_reg_reg);
9352 %}
9353
9354 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9355 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9356
9357 ins_cost(INSN_COST * 2);
9358 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9359
9360 ins_encode %{
9361 __ cselw(as_Register($dst$$reg),
9362 as_Register($src2$$reg),
9363 as_Register($src1$$reg),
9364 (Assembler::Condition)$cmp$$cmpcode);
9365 %}
9366
9367 ins_pipe(icond_reg_reg);
9368 %}
9369
9370 // special cases where one arg is zero
9371
9372 // n.b. this is selected in preference to the rule above because it
9373 // avoids loading constant 0 into a source register
9374
9375 // TODO
9376 // we ought only to be able to cull one of these variants as the ideal
9377 // transforms ought always to order the zero consistently (to left/right?)
9378
9379 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9380 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9381
9382 ins_cost(INSN_COST * 2);
9383 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9384
9385 ins_encode %{
9386 __ cselw(as_Register($dst$$reg),
9387 as_Register($src$$reg),
9388 zr,
9389 (Assembler::Condition)$cmp$$cmpcode);
9390 %}
9391
9392 ins_pipe(icond_reg);
9393 %}
9394
9395 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9396 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9397
9398 ins_cost(INSN_COST * 2);
9399 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9400
9401 ins_encode %{
9402 __ cselw(as_Register($dst$$reg),
9403 as_Register($src$$reg),
9404 zr,
9405 (Assembler::Condition)$cmp$$cmpcode);
9406 %}
9407
9408 ins_pipe(icond_reg);
9409 %}
9410
9411 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9412 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9413
9414 ins_cost(INSN_COST * 2);
9415 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9416
9417 ins_encode %{
9418 __ cselw(as_Register($dst$$reg),
9419 zr,
9420 as_Register($src$$reg),
9421 (Assembler::Condition)$cmp$$cmpcode);
9422 %}
9423
9424 ins_pipe(icond_reg);
9425 %}
9426
9427 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9428 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9429
9430 ins_cost(INSN_COST * 2);
9431 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9432
9433 ins_encode %{
9434 __ cselw(as_Register($dst$$reg),
9435 zr,
9436 as_Register($src$$reg),
9437 (Assembler::Condition)$cmp$$cmpcode);
9438 %}
9439
9440 ins_pipe(icond_reg);
9441 %}
9442
9443 // special case for creating a boolean 0 or 1
9444
9445 // n.b. this is selected in preference to the rule above because it
9446 // avoids loading constants 0 and 1 into a source register
9447
9448 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9449 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9450
9451 ins_cost(INSN_COST * 2);
9452 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9453
9454 ins_encode %{
9455 // equivalently
9456 // cset(as_Register($dst$$reg),
9457 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9458 __ csincw(as_Register($dst$$reg),
9459 zr,
9460 zr,
9461 (Assembler::Condition)$cmp$$cmpcode);
9462 %}
9463
9464 ins_pipe(icond_none);
9465 %}
9466
9467 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9468 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9469
9470 ins_cost(INSN_COST * 2);
9471 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9472
9473 ins_encode %{
9474 // equivalently
9475 // cset(as_Register($dst$$reg),
9476 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9477 __ csincw(as_Register($dst$$reg),
9478 zr,
9479 zr,
9480 (Assembler::Condition)$cmp$$cmpcode);
9481 %}
9482
9483 ins_pipe(icond_none);
9484 %}
9485
9486 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9487 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9488
9489 ins_cost(INSN_COST * 2);
9490 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9491
9492 ins_encode %{
9493 __ csel(as_Register($dst$$reg),
9494 as_Register($src2$$reg),
9495 as_Register($src1$$reg),
9496 (Assembler::Condition)$cmp$$cmpcode);
9497 %}
9498
9499 ins_pipe(icond_reg_reg);
9500 %}
9501
9502 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9503 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9504
9505 ins_cost(INSN_COST * 2);
9506 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9507
9508 ins_encode %{
9509 __ csel(as_Register($dst$$reg),
9510 as_Register($src2$$reg),
9511 as_Register($src1$$reg),
9512 (Assembler::Condition)$cmp$$cmpcode);
9513 %}
9514
9515 ins_pipe(icond_reg_reg);
9516 %}
9517
9518 // special cases where one arg is zero
9519
9520 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9521 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9522
9523 ins_cost(INSN_COST * 2);
9524 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9525
9526 ins_encode %{
9527 __ csel(as_Register($dst$$reg),
9528 zr,
9529 as_Register($src$$reg),
9530 (Assembler::Condition)$cmp$$cmpcode);
9531 %}
9532
9533 ins_pipe(icond_reg);
9534 %}
9535
9536 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9537 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9538
9539 ins_cost(INSN_COST * 2);
9540 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9541
9542 ins_encode %{
9543 __ csel(as_Register($dst$$reg),
9544 zr,
9545 as_Register($src$$reg),
9546 (Assembler::Condition)$cmp$$cmpcode);
9547 %}
9548
9549 ins_pipe(icond_reg);
9550 %}
9551
9552 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9553 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9554
9555 ins_cost(INSN_COST * 2);
9556 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9557
9558 ins_encode %{
9559 __ csel(as_Register($dst$$reg),
9560 as_Register($src$$reg),
9561 zr,
9562 (Assembler::Condition)$cmp$$cmpcode);
9563 %}
9564
9565 ins_pipe(icond_reg);
9566 %}
9567
9568 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9569 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9570
9571 ins_cost(INSN_COST * 2);
9572 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9573
9574 ins_encode %{
9575 __ csel(as_Register($dst$$reg),
9576 as_Register($src$$reg),
9577 zr,
9578 (Assembler::Condition)$cmp$$cmpcode);
9579 %}
9580
9581 ins_pipe(icond_reg);
9582 %}
9583
9584 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9585 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9586
9587 ins_cost(INSN_COST * 2);
9588 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9589
9590 ins_encode %{
9591 __ csel(as_Register($dst$$reg),
9592 as_Register($src2$$reg),
9593 as_Register($src1$$reg),
9594 (Assembler::Condition)$cmp$$cmpcode);
9595 %}
9596
9597 ins_pipe(icond_reg_reg);
9598 %}
9599
9600 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9601 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9602
9603 ins_cost(INSN_COST * 2);
9604 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9605
9606 ins_encode %{
9607 __ csel(as_Register($dst$$reg),
9608 as_Register($src2$$reg),
9609 as_Register($src1$$reg),
9610 (Assembler::Condition)$cmp$$cmpcode);
9611 %}
9612
9613 ins_pipe(icond_reg_reg);
9614 %}
9615
9616 // special cases where one arg is zero
9617
9618 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9619 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9620
9621 ins_cost(INSN_COST * 2);
9622 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9623
9624 ins_encode %{
9625 __ csel(as_Register($dst$$reg),
9626 zr,
9627 as_Register($src$$reg),
9628 (Assembler::Condition)$cmp$$cmpcode);
9629 %}
9630
9631 ins_pipe(icond_reg);
9632 %}
9633
9634 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9635 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9636
9637 ins_cost(INSN_COST * 2);
9638 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9639
9640 ins_encode %{
9641 __ csel(as_Register($dst$$reg),
9642 zr,
9643 as_Register($src$$reg),
9644 (Assembler::Condition)$cmp$$cmpcode);
9645 %}
9646
9647 ins_pipe(icond_reg);
9648 %}
9649
9650 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9651 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9652
9653 ins_cost(INSN_COST * 2);
9654 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9655
9656 ins_encode %{
9657 __ csel(as_Register($dst$$reg),
9658 as_Register($src$$reg),
9659 zr,
9660 (Assembler::Condition)$cmp$$cmpcode);
9661 %}
9662
9663 ins_pipe(icond_reg);
9664 %}
9665
9666 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9667 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9668
9669 ins_cost(INSN_COST * 2);
9670 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9671
9672 ins_encode %{
9673 __ csel(as_Register($dst$$reg),
9674 as_Register($src$$reg),
9675 zr,
9676 (Assembler::Condition)$cmp$$cmpcode);
9677 %}
9678
9679 ins_pipe(icond_reg);
9680 %}
9681
9682 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9683 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9684
9685 ins_cost(INSN_COST * 2);
9686 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9687
9688 ins_encode %{
9689 __ cselw(as_Register($dst$$reg),
9690 as_Register($src2$$reg),
9691 as_Register($src1$$reg),
9692 (Assembler::Condition)$cmp$$cmpcode);
9693 %}
9694
9695 ins_pipe(icond_reg_reg);
9696 %}
9697
9698 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9699 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9700
9701 ins_cost(INSN_COST * 2);
9702 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9703
9704 ins_encode %{
9705 __ cselw(as_Register($dst$$reg),
9706 as_Register($src2$$reg),
9707 as_Register($src1$$reg),
9708 (Assembler::Condition)$cmp$$cmpcode);
9709 %}
9710
9711 ins_pipe(icond_reg_reg);
9712 %}
9713
9714 // special cases where one arg is zero
9715
9716 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9717 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9718
9719 ins_cost(INSN_COST * 2);
9720 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9721
9722 ins_encode %{
9723 __ cselw(as_Register($dst$$reg),
9724 zr,
9725 as_Register($src$$reg),
9726 (Assembler::Condition)$cmp$$cmpcode);
9727 %}
9728
9729 ins_pipe(icond_reg);
9730 %}
9731
9732 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9733 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9734
9735 ins_cost(INSN_COST * 2);
9736 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9737
9738 ins_encode %{
9739 __ cselw(as_Register($dst$$reg),
9740 zr,
9741 as_Register($src$$reg),
9742 (Assembler::Condition)$cmp$$cmpcode);
9743 %}
9744
9745 ins_pipe(icond_reg);
9746 %}
9747
9748 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9749 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9750
9751 ins_cost(INSN_COST * 2);
9752 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9753
9754 ins_encode %{
9755 __ cselw(as_Register($dst$$reg),
9756 as_Register($src$$reg),
9757 zr,
9758 (Assembler::Condition)$cmp$$cmpcode);
9759 %}
9760
9761 ins_pipe(icond_reg);
9762 %}
9763
9764 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9765 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9766
9767 ins_cost(INSN_COST * 2);
9768 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9769
9770 ins_encode %{
9771 __ cselw(as_Register($dst$$reg),
9772 as_Register($src$$reg),
9773 zr,
9774 (Assembler::Condition)$cmp$$cmpcode);
9775 %}
9776
9777 ins_pipe(icond_reg);
9778 %}
9779
9780 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9781 %{
9782 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9783
9784 ins_cost(INSN_COST * 3);
9785
9786 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9787 ins_encode %{
9788 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9789 __ fcsels(as_FloatRegister($dst$$reg),
9790 as_FloatRegister($src2$$reg),
9791 as_FloatRegister($src1$$reg),
9792 cond);
9793 %}
9794
9795 ins_pipe(fp_cond_reg_reg_s);
9796 %}
9797
9798 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9799 %{
9800 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9801
9802 ins_cost(INSN_COST * 3);
9803
9804 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9805 ins_encode %{
9806 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9807 __ fcsels(as_FloatRegister($dst$$reg),
9808 as_FloatRegister($src2$$reg),
9809 as_FloatRegister($src1$$reg),
9810 cond);
9811 %}
9812
9813 ins_pipe(fp_cond_reg_reg_s);
9814 %}
9815
9816 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9817 %{
9818 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9819
9820 ins_cost(INSN_COST * 3);
9821
9822 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9823 ins_encode %{
9824 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9825 __ fcseld(as_FloatRegister($dst$$reg),
9826 as_FloatRegister($src2$$reg),
9827 as_FloatRegister($src1$$reg),
9828 cond);
9829 %}
9830
9831 ins_pipe(fp_cond_reg_reg_d);
9832 %}
9833
9834 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9835 %{
9836 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9837
9838 ins_cost(INSN_COST * 3);
9839
9840 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9841 ins_encode %{
9842 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9843 __ fcseld(as_FloatRegister($dst$$reg),
9844 as_FloatRegister($src2$$reg),
9845 as_FloatRegister($src1$$reg),
9846 cond);
9847 %}
9848
9849 ins_pipe(fp_cond_reg_reg_d);
9850 %}
9851
9852 // ============================================================================
9853 // Arithmetic Instructions
9854 //
9855
9856 // Integer Addition
9857
9858 // TODO
9859 // these currently employ operations which do not set CR and hence are
9860 // not flagged as killing CR but we would like to isolate the cases
9861 // where we want to set flags from those where we don't. need to work
9862 // out how to do that.
9863
9864 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9865 match(Set dst (AddI src1 src2));
9866
9867 ins_cost(INSN_COST);
9868 format %{ "addw $dst, $src1, $src2" %}
9869
9870 ins_encode %{
9871 __ addw(as_Register($dst$$reg),
9872 as_Register($src1$$reg),
9873 as_Register($src2$$reg));
9874 %}
9875
9876 ins_pipe(ialu_reg_reg);
9877 %}
9878
9879 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9880 match(Set dst (AddI src1 src2));
9881
9882 ins_cost(INSN_COST);
9883 format %{ "addw $dst, $src1, $src2" %}
9884
9885 // use opcode to indicate that this is an add not a sub
9886 opcode(0x0);
9887
9888 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9889
9890 ins_pipe(ialu_reg_imm);
9891 %}
9892
9893 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9894 match(Set dst (AddI (ConvL2I src1) src2));
9895
9896 ins_cost(INSN_COST);
9897 format %{ "addw $dst, $src1, $src2" %}
9898
9899 // use opcode to indicate that this is an add not a sub
9900 opcode(0x0);
9901
9902 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9903
9904 ins_pipe(ialu_reg_imm);
9905 %}
9906
9907 // Pointer Addition
9908 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9909 match(Set dst (AddP src1 src2));
9910
9911 ins_cost(INSN_COST);
9912 format %{ "add $dst, $src1, $src2\t# ptr" %}
9913
9914 ins_encode %{
9915 __ add(as_Register($dst$$reg),
9916 as_Register($src1$$reg),
9917 as_Register($src2$$reg));
9918 %}
9919
9920 ins_pipe(ialu_reg_reg);
9921 %}
9922
9923 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9924 match(Set dst (AddP src1 (ConvI2L src2)));
9925
9926 ins_cost(1.9 * INSN_COST);
9927 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9928
9929 ins_encode %{
9930 __ add(as_Register($dst$$reg),
9931 as_Register($src1$$reg),
9932 as_Register($src2$$reg), ext::sxtw);
9933 %}
9934
9935 ins_pipe(ialu_reg_reg);
9936 %}
9937
9938 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9939 match(Set dst (AddP src1 (LShiftL src2 scale)));
9940
9941 ins_cost(1.9 * INSN_COST);
9942 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9943
9944 ins_encode %{
9945 __ lea(as_Register($dst$$reg),
9946 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9947 Address::lsl($scale$$constant)));
9948 %}
9949
9950 ins_pipe(ialu_reg_reg_shift);
9951 %}
9952
9953 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9954 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9955
9956 ins_cost(1.9 * INSN_COST);
9957 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9958
9959 ins_encode %{
9960 __ lea(as_Register($dst$$reg),
9961 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9962 Address::sxtw($scale$$constant)));
9963 %}
9964
9965 ins_pipe(ialu_reg_reg_shift);
9966 %}
9967
9968 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9969 match(Set dst (LShiftL (ConvI2L src) scale));
9970
9971 ins_cost(INSN_COST);
9972 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9973
9974 ins_encode %{
9975 __ sbfiz(as_Register($dst$$reg),
9976 as_Register($src$$reg),
9977 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9978 %}
9979
9980 ins_pipe(ialu_reg_shift);
9981 %}
9982
9983 // Pointer Immediate Addition
9984 // n.b. this needs to be more expensive than using an indirect memory
9985 // operand
9986 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9987 match(Set dst (AddP src1 src2));
9988
9989 ins_cost(INSN_COST);
9990 format %{ "add $dst, $src1, $src2\t# ptr" %}
9991
9992 // use opcode to indicate that this is an add not a sub
9993 opcode(0x0);
9994
9995 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9996
9997 ins_pipe(ialu_reg_imm);
9998 %}
9999
10000 // Long Addition
10001 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10002
10003 match(Set dst (AddL src1 src2));
10004
10005 ins_cost(INSN_COST);
10006 format %{ "add $dst, $src1, $src2" %}
10007
10008 ins_encode %{
10009 __ add(as_Register($dst$$reg),
10010 as_Register($src1$$reg),
10011 as_Register($src2$$reg));
10012 %}
10013
10014 ins_pipe(ialu_reg_reg);
10015 %}
10016
10017 // No constant pool entries requiredLong Immediate Addition.
10018 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10019 match(Set dst (AddL src1 src2));
10020
10021 ins_cost(INSN_COST);
10022 format %{ "add $dst, $src1, $src2" %}
10023
10024 // use opcode to indicate that this is an add not a sub
10025 opcode(0x0);
10026
10027 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10028
10029 ins_pipe(ialu_reg_imm);
10030 %}
10031
10032 // Integer Subtraction
10033 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10034 match(Set dst (SubI src1 src2));
10035
10036 ins_cost(INSN_COST);
10037 format %{ "subw $dst, $src1, $src2" %}
10038
10039 ins_encode %{
10040 __ subw(as_Register($dst$$reg),
10041 as_Register($src1$$reg),
10042 as_Register($src2$$reg));
10043 %}
10044
10045 ins_pipe(ialu_reg_reg);
10046 %}
10047
10048 // Immediate Subtraction
10049 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10050 match(Set dst (SubI src1 src2));
10051
10052 ins_cost(INSN_COST);
10053 format %{ "subw $dst, $src1, $src2" %}
10054
10055 // use opcode to indicate that this is a sub not an add
10056 opcode(0x1);
10057
10058 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10059
10060 ins_pipe(ialu_reg_imm);
10061 %}
10062
10063 // Long Subtraction
10064 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10065
10066 match(Set dst (SubL src1 src2));
10067
10068 ins_cost(INSN_COST);
10069 format %{ "sub $dst, $src1, $src2" %}
10070
10071 ins_encode %{
10072 __ sub(as_Register($dst$$reg),
10073 as_Register($src1$$reg),
10074 as_Register($src2$$reg));
10075 %}
10076
10077 ins_pipe(ialu_reg_reg);
10078 %}
10079
10080 // No constant pool entries requiredLong Immediate Subtraction.
10081 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10082 match(Set dst (SubL src1 src2));
10083
10084 ins_cost(INSN_COST);
10085 format %{ "sub$dst, $src1, $src2" %}
10086
10087 // use opcode to indicate that this is a sub not an add
10088 opcode(0x1);
10089
10090 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10091
10092 ins_pipe(ialu_reg_imm);
10093 %}
10094
10095 // Integer Negation (special case for sub)
10096
10097 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10098 match(Set dst (SubI zero src));
10099
10100 ins_cost(INSN_COST);
10101 format %{ "negw $dst, $src\t# int" %}
10102
10103 ins_encode %{
10104 __ negw(as_Register($dst$$reg),
10105 as_Register($src$$reg));
10106 %}
10107
10108 ins_pipe(ialu_reg);
10109 %}
10110
10111 // Long Negation
10112
10113 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10114 match(Set dst (SubL zero src));
10115
10116 ins_cost(INSN_COST);
10117 format %{ "neg $dst, $src\t# long" %}
10118
10119 ins_encode %{
10120 __ neg(as_Register($dst$$reg),
10121 as_Register($src$$reg));
10122 %}
10123
10124 ins_pipe(ialu_reg);
10125 %}
10126
10127 // Integer Multiply
10128
10129 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10130 match(Set dst (MulI src1 src2));
10131
10132 ins_cost(INSN_COST * 3);
10133 format %{ "mulw $dst, $src1, $src2" %}
10134
10135 ins_encode %{
10136 __ mulw(as_Register($dst$$reg),
10137 as_Register($src1$$reg),
10138 as_Register($src2$$reg));
10139 %}
10140
10141 ins_pipe(imul_reg_reg);
10142 %}
10143
10144 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10145 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10146
10147 ins_cost(INSN_COST * 3);
10148 format %{ "smull $dst, $src1, $src2" %}
10149
10150 ins_encode %{
10151 __ smull(as_Register($dst$$reg),
10152 as_Register($src1$$reg),
10153 as_Register($src2$$reg));
10154 %}
10155
10156 ins_pipe(imul_reg_reg);
10157 %}
10158
10159 // Long Multiply
10160
10161 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10162 match(Set dst (MulL src1 src2));
10163
10164 ins_cost(INSN_COST * 5);
10165 format %{ "mul $dst, $src1, $src2" %}
10166
10167 ins_encode %{
10168 __ mul(as_Register($dst$$reg),
10169 as_Register($src1$$reg),
10170 as_Register($src2$$reg));
10171 %}
10172
10173 ins_pipe(lmul_reg_reg);
10174 %}
10175
10176 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10177 %{
10178 match(Set dst (MulHiL src1 src2));
10179
10180 ins_cost(INSN_COST * 7);
10181 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10182
10183 ins_encode %{
10184 __ smulh(as_Register($dst$$reg),
10185 as_Register($src1$$reg),
10186 as_Register($src2$$reg));
10187 %}
10188
10189 ins_pipe(lmul_reg_reg);
10190 %}
10191
10192 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10193 %{
10194 match(Set dst (UMulHiL src1 src2));
10195
10196 ins_cost(INSN_COST * 7);
10197 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10198
10199 ins_encode %{
10200 __ umulh(as_Register($dst$$reg),
10201 as_Register($src1$$reg),
10202 as_Register($src2$$reg));
10203 %}
10204
10205 ins_pipe(lmul_reg_reg);
10206 %}
10207
10208 // Combined Integer Multiply & Add/Sub
10209
10210 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10211 match(Set dst (AddI src3 (MulI src1 src2)));
10212
10213 ins_cost(INSN_COST * 3);
10214 format %{ "madd $dst, $src1, $src2, $src3" %}
10215
10216 ins_encode %{
10217 __ maddw(as_Register($dst$$reg),
10218 as_Register($src1$$reg),
10219 as_Register($src2$$reg),
10220 as_Register($src3$$reg));
10221 %}
10222
10223 ins_pipe(imac_reg_reg);
10224 %}
10225
10226 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10227 match(Set dst (SubI src3 (MulI src1 src2)));
10228
10229 ins_cost(INSN_COST * 3);
10230 format %{ "msub $dst, $src1, $src2, $src3" %}
10231
10232 ins_encode %{
10233 __ msubw(as_Register($dst$$reg),
10234 as_Register($src1$$reg),
10235 as_Register($src2$$reg),
10236 as_Register($src3$$reg));
10237 %}
10238
10239 ins_pipe(imac_reg_reg);
10240 %}
10241
10242 // Combined Integer Multiply & Neg
10243
10244 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10245 match(Set dst (MulI (SubI zero src1) src2));
10246
10247 ins_cost(INSN_COST * 3);
10248 format %{ "mneg $dst, $src1, $src2" %}
10249
10250 ins_encode %{
10251 __ mnegw(as_Register($dst$$reg),
10252 as_Register($src1$$reg),
10253 as_Register($src2$$reg));
10254 %}
10255
10256 ins_pipe(imac_reg_reg);
10257 %}
10258
10259 // Combined Long Multiply & Add/Sub
10260
10261 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10262 match(Set dst (AddL src3 (MulL src1 src2)));
10263
10264 ins_cost(INSN_COST * 5);
10265 format %{ "madd $dst, $src1, $src2, $src3" %}
10266
10267 ins_encode %{
10268 __ madd(as_Register($dst$$reg),
10269 as_Register($src1$$reg),
10270 as_Register($src2$$reg),
10271 as_Register($src3$$reg));
10272 %}
10273
10274 ins_pipe(lmac_reg_reg);
10275 %}
10276
10277 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10278 match(Set dst (SubL src3 (MulL src1 src2)));
10279
10280 ins_cost(INSN_COST * 5);
10281 format %{ "msub $dst, $src1, $src2, $src3" %}
10282
10283 ins_encode %{
10284 __ msub(as_Register($dst$$reg),
10285 as_Register($src1$$reg),
10286 as_Register($src2$$reg),
10287 as_Register($src3$$reg));
10288 %}
10289
10290 ins_pipe(lmac_reg_reg);
10291 %}
10292
10293 // Combined Long Multiply & Neg
10294
10295 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10296 match(Set dst (MulL (SubL zero src1) src2));
10297
10298 ins_cost(INSN_COST * 5);
10299 format %{ "mneg $dst, $src1, $src2" %}
10300
10301 ins_encode %{
10302 __ mneg(as_Register($dst$$reg),
10303 as_Register($src1$$reg),
10304 as_Register($src2$$reg));
10305 %}
10306
10307 ins_pipe(lmac_reg_reg);
10308 %}
10309
10310 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10311
10312 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10313 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10314
10315 ins_cost(INSN_COST * 3);
10316 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10317
10318 ins_encode %{
10319 __ smaddl(as_Register($dst$$reg),
10320 as_Register($src1$$reg),
10321 as_Register($src2$$reg),
10322 as_Register($src3$$reg));
10323 %}
10324
10325 ins_pipe(imac_reg_reg);
10326 %}
10327
10328 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10329 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10330
10331 ins_cost(INSN_COST * 3);
10332 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10333
10334 ins_encode %{
10335 __ smsubl(as_Register($dst$$reg),
10336 as_Register($src1$$reg),
10337 as_Register($src2$$reg),
10338 as_Register($src3$$reg));
10339 %}
10340
10341 ins_pipe(imac_reg_reg);
10342 %}
10343
10344 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10345 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10346
10347 ins_cost(INSN_COST * 3);
10348 format %{ "smnegl $dst, $src1, $src2" %}
10349
10350 ins_encode %{
10351 __ smnegl(as_Register($dst$$reg),
10352 as_Register($src1$$reg),
10353 as_Register($src2$$reg));
10354 %}
10355
10356 ins_pipe(imac_reg_reg);
10357 %}
10358
10359 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10360
10361 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10362 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10363
10364 ins_cost(INSN_COST * 5);
10365 format %{ "mulw rscratch1, $src1, $src2\n\t"
10366 "maddw $dst, $src3, $src4, rscratch1" %}
10367
10368 ins_encode %{
10369 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10370 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10371
10372 ins_pipe(imac_reg_reg);
10373 %}
10374
10375 // Integer Divide
10376
10377 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10378 match(Set dst (DivI src1 src2));
10379
10380 ins_cost(INSN_COST * 19);
10381 format %{ "sdivw $dst, $src1, $src2" %}
10382
10383 ins_encode(aarch64_enc_divw(dst, src1, src2));
10384 ins_pipe(idiv_reg_reg);
10385 %}
10386
10387 // Long Divide
10388
10389 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10390 match(Set dst (DivL src1 src2));
10391
10392 ins_cost(INSN_COST * 35);
10393 format %{ "sdiv $dst, $src1, $src2" %}
10394
10395 ins_encode(aarch64_enc_div(dst, src1, src2));
10396 ins_pipe(ldiv_reg_reg);
10397 %}
10398
10399 // Integer Remainder
10400
10401 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10402 match(Set dst (ModI src1 src2));
10403
10404 ins_cost(INSN_COST * 22);
10405 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10406 "msubw $dst, rscratch1, $src2, $src1" %}
10407
10408 ins_encode(aarch64_enc_modw(dst, src1, src2));
10409 ins_pipe(idiv_reg_reg);
10410 %}
10411
10412 // Long Remainder
10413
10414 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10415 match(Set dst (ModL src1 src2));
10416
10417 ins_cost(INSN_COST * 38);
10418 format %{ "sdiv rscratch1, $src1, $src2\n"
10419 "msub $dst, rscratch1, $src2, $src1" %}
10420
10421 ins_encode(aarch64_enc_mod(dst, src1, src2));
10422 ins_pipe(ldiv_reg_reg);
10423 %}
10424
10425 // Unsigned Integer Divide
10426
10427 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10428 match(Set dst (UDivI src1 src2));
10429
10430 ins_cost(INSN_COST * 19);
10431 format %{ "udivw $dst, $src1, $src2" %}
10432
10433 ins_encode %{
10434 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10435 %}
10436
10437 ins_pipe(idiv_reg_reg);
10438 %}
10439
10440 // Unsigned Long Divide
10441
10442 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10443 match(Set dst (UDivL src1 src2));
10444
10445 ins_cost(INSN_COST * 35);
10446 format %{ "udiv $dst, $src1, $src2" %}
10447
10448 ins_encode %{
10449 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10450 %}
10451
10452 ins_pipe(ldiv_reg_reg);
10453 %}
10454
10455 // Unsigned Integer Remainder
10456
10457 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10458 match(Set dst (UModI src1 src2));
10459
10460 ins_cost(INSN_COST * 22);
10461 format %{ "udivw rscratch1, $src1, $src2\n\t"
10462 "msubw $dst, rscratch1, $src2, $src1" %}
10463
10464 ins_encode %{
10465 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10466 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10467 %}
10468
10469 ins_pipe(idiv_reg_reg);
10470 %}
10471
10472 // Unsigned Long Remainder
10473
10474 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10475 match(Set dst (UModL src1 src2));
10476
10477 ins_cost(INSN_COST * 38);
10478 format %{ "udiv rscratch1, $src1, $src2\n"
10479 "msub $dst, rscratch1, $src2, $src1" %}
10480
10481 ins_encode %{
10482 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10483 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10484 %}
10485
10486 ins_pipe(ldiv_reg_reg);
10487 %}
10488
10489 // Integer Shifts
10490
10491 // Shift Left Register
10492 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10493 match(Set dst (LShiftI src1 src2));
10494
10495 ins_cost(INSN_COST * 2);
10496 format %{ "lslvw $dst, $src1, $src2" %}
10497
10498 ins_encode %{
10499 __ lslvw(as_Register($dst$$reg),
10500 as_Register($src1$$reg),
10501 as_Register($src2$$reg));
10502 %}
10503
10504 ins_pipe(ialu_reg_reg_vshift);
10505 %}
10506
10507 // Shift Left Immediate
10508 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10509 match(Set dst (LShiftI src1 src2));
10510
10511 ins_cost(INSN_COST);
10512 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10513
10514 ins_encode %{
10515 __ lslw(as_Register($dst$$reg),
10516 as_Register($src1$$reg),
10517 $src2$$constant & 0x1f);
10518 %}
10519
10520 ins_pipe(ialu_reg_shift);
10521 %}
10522
10523 // Shift Right Logical Register
10524 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10525 match(Set dst (URShiftI src1 src2));
10526
10527 ins_cost(INSN_COST * 2);
10528 format %{ "lsrvw $dst, $src1, $src2" %}
10529
10530 ins_encode %{
10531 __ lsrvw(as_Register($dst$$reg),
10532 as_Register($src1$$reg),
10533 as_Register($src2$$reg));
10534 %}
10535
10536 ins_pipe(ialu_reg_reg_vshift);
10537 %}
10538
10539 // Shift Right Logical Immediate
10540 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10541 match(Set dst (URShiftI src1 src2));
10542
10543 ins_cost(INSN_COST);
10544 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10545
10546 ins_encode %{
10547 __ lsrw(as_Register($dst$$reg),
10548 as_Register($src1$$reg),
10549 $src2$$constant & 0x1f);
10550 %}
10551
10552 ins_pipe(ialu_reg_shift);
10553 %}
10554
10555 // Shift Right Arithmetic Register
10556 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10557 match(Set dst (RShiftI src1 src2));
10558
10559 ins_cost(INSN_COST * 2);
10560 format %{ "asrvw $dst, $src1, $src2" %}
10561
10562 ins_encode %{
10563 __ asrvw(as_Register($dst$$reg),
10564 as_Register($src1$$reg),
10565 as_Register($src2$$reg));
10566 %}
10567
10568 ins_pipe(ialu_reg_reg_vshift);
10569 %}
10570
10571 // Shift Right Arithmetic Immediate
10572 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10573 match(Set dst (RShiftI src1 src2));
10574
10575 ins_cost(INSN_COST);
10576 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10577
10578 ins_encode %{
10579 __ asrw(as_Register($dst$$reg),
10580 as_Register($src1$$reg),
10581 $src2$$constant & 0x1f);
10582 %}
10583
10584 ins_pipe(ialu_reg_shift);
10585 %}
10586
10587 // Combined Int Mask and Right Shift (using UBFM)
10588 // TODO
10589
10590 // Long Shifts
10591
10592 // Shift Left Register
10593 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10594 match(Set dst (LShiftL src1 src2));
10595
10596 ins_cost(INSN_COST * 2);
10597 format %{ "lslv $dst, $src1, $src2" %}
10598
10599 ins_encode %{
10600 __ lslv(as_Register($dst$$reg),
10601 as_Register($src1$$reg),
10602 as_Register($src2$$reg));
10603 %}
10604
10605 ins_pipe(ialu_reg_reg_vshift);
10606 %}
10607
10608 // Shift Left Immediate
10609 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10610 match(Set dst (LShiftL src1 src2));
10611
10612 ins_cost(INSN_COST);
10613 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10614
10615 ins_encode %{
10616 __ lsl(as_Register($dst$$reg),
10617 as_Register($src1$$reg),
10618 $src2$$constant & 0x3f);
10619 %}
10620
10621 ins_pipe(ialu_reg_shift);
10622 %}
10623
10624 // Shift Right Logical Register
10625 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10626 match(Set dst (URShiftL src1 src2));
10627
10628 ins_cost(INSN_COST * 2);
10629 format %{ "lsrv $dst, $src1, $src2" %}
10630
10631 ins_encode %{
10632 __ lsrv(as_Register($dst$$reg),
10633 as_Register($src1$$reg),
10634 as_Register($src2$$reg));
10635 %}
10636
10637 ins_pipe(ialu_reg_reg_vshift);
10638 %}
10639
10640 // Shift Right Logical Immediate
10641 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10642 match(Set dst (URShiftL src1 src2));
10643
10644 ins_cost(INSN_COST);
10645 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10646
10647 ins_encode %{
10648 __ lsr(as_Register($dst$$reg),
10649 as_Register($src1$$reg),
10650 $src2$$constant & 0x3f);
10651 %}
10652
10653 ins_pipe(ialu_reg_shift);
10654 %}
10655
10656 // A special-case pattern for card table stores.
10657 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10658 match(Set dst (URShiftL (CastP2X src1) src2));
10659
10660 ins_cost(INSN_COST);
10661 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10662
10663 ins_encode %{
10664 __ lsr(as_Register($dst$$reg),
10665 as_Register($src1$$reg),
10666 $src2$$constant & 0x3f);
10667 %}
10668
10669 ins_pipe(ialu_reg_shift);
10670 %}
10671
10672 // Shift Right Arithmetic Register
10673 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10674 match(Set dst (RShiftL src1 src2));
10675
10676 ins_cost(INSN_COST * 2);
10677 format %{ "asrv $dst, $src1, $src2" %}
10678
10679 ins_encode %{
10680 __ asrv(as_Register($dst$$reg),
10681 as_Register($src1$$reg),
10682 as_Register($src2$$reg));
10683 %}
10684
10685 ins_pipe(ialu_reg_reg_vshift);
10686 %}
10687
10688 // Shift Right Arithmetic Immediate
10689 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10690 match(Set dst (RShiftL src1 src2));
10691
10692 ins_cost(INSN_COST);
10693 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10694
10695 ins_encode %{
10696 __ asr(as_Register($dst$$reg),
10697 as_Register($src1$$reg),
10698 $src2$$constant & 0x3f);
10699 %}
10700
10701 ins_pipe(ialu_reg_shift);
10702 %}
10703
10704 // BEGIN This section of the file is automatically generated. Do not edit --------------
10705 // This section is generated from aarch64_ad.m4
10706
10707 // This pattern is automatically generated from aarch64_ad.m4
10708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10709 instruct regL_not_reg(iRegLNoSp dst,
10710 iRegL src1, immL_M1 m1,
10711 rFlagsReg cr) %{
10712 match(Set dst (XorL src1 m1));
10713 ins_cost(INSN_COST);
10714 format %{ "eon $dst, $src1, zr" %}
10715
10716 ins_encode %{
10717 __ eon(as_Register($dst$$reg),
10718 as_Register($src1$$reg),
10719 zr,
10720 Assembler::LSL, 0);
10721 %}
10722
10723 ins_pipe(ialu_reg);
10724 %}
10725
10726 // This pattern is automatically generated from aarch64_ad.m4
10727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10728 instruct regI_not_reg(iRegINoSp dst,
10729 iRegIorL2I src1, immI_M1 m1,
10730 rFlagsReg cr) %{
10731 match(Set dst (XorI src1 m1));
10732 ins_cost(INSN_COST);
10733 format %{ "eonw $dst, $src1, zr" %}
10734
10735 ins_encode %{
10736 __ eonw(as_Register($dst$$reg),
10737 as_Register($src1$$reg),
10738 zr,
10739 Assembler::LSL, 0);
10740 %}
10741
10742 ins_pipe(ialu_reg);
10743 %}
10744
10745 // This pattern is automatically generated from aarch64_ad.m4
10746 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10747 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10748 immI0 zero, iRegIorL2I src1, immI src2) %{
10749 match(Set dst (SubI zero (URShiftI src1 src2)));
10750
10751 ins_cost(1.9 * INSN_COST);
10752 format %{ "negw $dst, $src1, LSR $src2" %}
10753
10754 ins_encode %{
10755 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10756 Assembler::LSR, $src2$$constant & 0x1f);
10757 %}
10758
10759 ins_pipe(ialu_reg_shift);
10760 %}
10761
10762 // This pattern is automatically generated from aarch64_ad.m4
10763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10764 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10765 immI0 zero, iRegIorL2I src1, immI src2) %{
10766 match(Set dst (SubI zero (RShiftI src1 src2)));
10767
10768 ins_cost(1.9 * INSN_COST);
10769 format %{ "negw $dst, $src1, ASR $src2" %}
10770
10771 ins_encode %{
10772 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10773 Assembler::ASR, $src2$$constant & 0x1f);
10774 %}
10775
10776 ins_pipe(ialu_reg_shift);
10777 %}
10778
10779 // This pattern is automatically generated from aarch64_ad.m4
10780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10781 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10782 immI0 zero, iRegIorL2I src1, immI src2) %{
10783 match(Set dst (SubI zero (LShiftI src1 src2)));
10784
10785 ins_cost(1.9 * INSN_COST);
10786 format %{ "negw $dst, $src1, LSL $src2" %}
10787
10788 ins_encode %{
10789 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10790 Assembler::LSL, $src2$$constant & 0x1f);
10791 %}
10792
10793 ins_pipe(ialu_reg_shift);
10794 %}
10795
10796 // This pattern is automatically generated from aarch64_ad.m4
10797 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10798 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10799 immL0 zero, iRegL src1, immI src2) %{
10800 match(Set dst (SubL zero (URShiftL src1 src2)));
10801
10802 ins_cost(1.9 * INSN_COST);
10803 format %{ "neg $dst, $src1, LSR $src2" %}
10804
10805 ins_encode %{
10806 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10807 Assembler::LSR, $src2$$constant & 0x3f);
10808 %}
10809
10810 ins_pipe(ialu_reg_shift);
10811 %}
10812
10813 // This pattern is automatically generated from aarch64_ad.m4
10814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10815 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10816 immL0 zero, iRegL src1, immI src2) %{
10817 match(Set dst (SubL zero (RShiftL src1 src2)));
10818
10819 ins_cost(1.9 * INSN_COST);
10820 format %{ "neg $dst, $src1, ASR $src2" %}
10821
10822 ins_encode %{
10823 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10824 Assembler::ASR, $src2$$constant & 0x3f);
10825 %}
10826
10827 ins_pipe(ialu_reg_shift);
10828 %}
10829
10830 // This pattern is automatically generated from aarch64_ad.m4
10831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10832 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10833 immL0 zero, iRegL src1, immI src2) %{
10834 match(Set dst (SubL zero (LShiftL src1 src2)));
10835
10836 ins_cost(1.9 * INSN_COST);
10837 format %{ "neg $dst, $src1, LSL $src2" %}
10838
10839 ins_encode %{
10840 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10841 Assembler::LSL, $src2$$constant & 0x3f);
10842 %}
10843
10844 ins_pipe(ialu_reg_shift);
10845 %}
10846
10847 // This pattern is automatically generated from aarch64_ad.m4
10848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10849 instruct AndI_reg_not_reg(iRegINoSp dst,
10850 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10851 match(Set dst (AndI src1 (XorI src2 m1)));
10852 ins_cost(INSN_COST);
10853 format %{ "bicw $dst, $src1, $src2" %}
10854
10855 ins_encode %{
10856 __ bicw(as_Register($dst$$reg),
10857 as_Register($src1$$reg),
10858 as_Register($src2$$reg),
10859 Assembler::LSL, 0);
10860 %}
10861
10862 ins_pipe(ialu_reg_reg);
10863 %}
10864
10865 // This pattern is automatically generated from aarch64_ad.m4
10866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10867 instruct AndL_reg_not_reg(iRegLNoSp dst,
10868 iRegL src1, iRegL src2, immL_M1 m1) %{
10869 match(Set dst (AndL src1 (XorL src2 m1)));
10870 ins_cost(INSN_COST);
10871 format %{ "bic $dst, $src1, $src2" %}
10872
10873 ins_encode %{
10874 __ bic(as_Register($dst$$reg),
10875 as_Register($src1$$reg),
10876 as_Register($src2$$reg),
10877 Assembler::LSL, 0);
10878 %}
10879
10880 ins_pipe(ialu_reg_reg);
10881 %}
10882
10883 // This pattern is automatically generated from aarch64_ad.m4
10884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10885 instruct OrI_reg_not_reg(iRegINoSp dst,
10886 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10887 match(Set dst (OrI src1 (XorI src2 m1)));
10888 ins_cost(INSN_COST);
10889 format %{ "ornw $dst, $src1, $src2" %}
10890
10891 ins_encode %{
10892 __ ornw(as_Register($dst$$reg),
10893 as_Register($src1$$reg),
10894 as_Register($src2$$reg),
10895 Assembler::LSL, 0);
10896 %}
10897
10898 ins_pipe(ialu_reg_reg);
10899 %}
10900
10901 // This pattern is automatically generated from aarch64_ad.m4
10902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10903 instruct OrL_reg_not_reg(iRegLNoSp dst,
10904 iRegL src1, iRegL src2, immL_M1 m1) %{
10905 match(Set dst (OrL src1 (XorL src2 m1)));
10906 ins_cost(INSN_COST);
10907 format %{ "orn $dst, $src1, $src2" %}
10908
10909 ins_encode %{
10910 __ orn(as_Register($dst$$reg),
10911 as_Register($src1$$reg),
10912 as_Register($src2$$reg),
10913 Assembler::LSL, 0);
10914 %}
10915
10916 ins_pipe(ialu_reg_reg);
10917 %}
10918
10919 // This pattern is automatically generated from aarch64_ad.m4
10920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10921 instruct XorI_reg_not_reg(iRegINoSp dst,
10922 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10923 match(Set dst (XorI m1 (XorI src2 src1)));
10924 ins_cost(INSN_COST);
10925 format %{ "eonw $dst, $src1, $src2" %}
10926
10927 ins_encode %{
10928 __ eonw(as_Register($dst$$reg),
10929 as_Register($src1$$reg),
10930 as_Register($src2$$reg),
10931 Assembler::LSL, 0);
10932 %}
10933
10934 ins_pipe(ialu_reg_reg);
10935 %}
10936
10937 // This pattern is automatically generated from aarch64_ad.m4
10938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10939 instruct XorL_reg_not_reg(iRegLNoSp dst,
10940 iRegL src1, iRegL src2, immL_M1 m1) %{
10941 match(Set dst (XorL m1 (XorL src2 src1)));
10942 ins_cost(INSN_COST);
10943 format %{ "eon $dst, $src1, $src2" %}
10944
10945 ins_encode %{
10946 __ eon(as_Register($dst$$reg),
10947 as_Register($src1$$reg),
10948 as_Register($src2$$reg),
10949 Assembler::LSL, 0);
10950 %}
10951
10952 ins_pipe(ialu_reg_reg);
10953 %}
10954
10955 // This pattern is automatically generated from aarch64_ad.m4
10956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10957 // val & (-1 ^ (val >>> shift)) ==> bicw
10958 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10959 iRegIorL2I src1, iRegIorL2I src2,
10960 immI src3, immI_M1 src4) %{
10961 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10962 ins_cost(1.9 * INSN_COST);
10963 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10964
10965 ins_encode %{
10966 __ bicw(as_Register($dst$$reg),
10967 as_Register($src1$$reg),
10968 as_Register($src2$$reg),
10969 Assembler::LSR,
10970 $src3$$constant & 0x1f);
10971 %}
10972
10973 ins_pipe(ialu_reg_reg_shift);
10974 %}
10975
10976 // This pattern is automatically generated from aarch64_ad.m4
10977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10978 // val & (-1 ^ (val >>> shift)) ==> bic
10979 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10980 iRegL src1, iRegL src2,
10981 immI src3, immL_M1 src4) %{
10982 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10983 ins_cost(1.9 * INSN_COST);
10984 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10985
10986 ins_encode %{
10987 __ bic(as_Register($dst$$reg),
10988 as_Register($src1$$reg),
10989 as_Register($src2$$reg),
10990 Assembler::LSR,
10991 $src3$$constant & 0x3f);
10992 %}
10993
10994 ins_pipe(ialu_reg_reg_shift);
10995 %}
10996
10997 // This pattern is automatically generated from aarch64_ad.m4
10998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10999 // val & (-1 ^ (val >> shift)) ==> bicw
11000 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11001 iRegIorL2I src1, iRegIorL2I src2,
11002 immI src3, immI_M1 src4) %{
11003 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11004 ins_cost(1.9 * INSN_COST);
11005 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11006
11007 ins_encode %{
11008 __ bicw(as_Register($dst$$reg),
11009 as_Register($src1$$reg),
11010 as_Register($src2$$reg),
11011 Assembler::ASR,
11012 $src3$$constant & 0x1f);
11013 %}
11014
11015 ins_pipe(ialu_reg_reg_shift);
11016 %}
11017
11018 // This pattern is automatically generated from aarch64_ad.m4
11019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11020 // val & (-1 ^ (val >> shift)) ==> bic
11021 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11022 iRegL src1, iRegL src2,
11023 immI src3, immL_M1 src4) %{
11024 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11025 ins_cost(1.9 * INSN_COST);
11026 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11027
11028 ins_encode %{
11029 __ bic(as_Register($dst$$reg),
11030 as_Register($src1$$reg),
11031 as_Register($src2$$reg),
11032 Assembler::ASR,
11033 $src3$$constant & 0x3f);
11034 %}
11035
11036 ins_pipe(ialu_reg_reg_shift);
11037 %}
11038
11039 // This pattern is automatically generated from aarch64_ad.m4
11040 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11041 // val & (-1 ^ (val ror shift)) ==> bicw
11042 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11043 iRegIorL2I src1, iRegIorL2I src2,
11044 immI src3, immI_M1 src4) %{
11045 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11046 ins_cost(1.9 * INSN_COST);
11047 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11048
11049 ins_encode %{
11050 __ bicw(as_Register($dst$$reg),
11051 as_Register($src1$$reg),
11052 as_Register($src2$$reg),
11053 Assembler::ROR,
11054 $src3$$constant & 0x1f);
11055 %}
11056
11057 ins_pipe(ialu_reg_reg_shift);
11058 %}
11059
11060 // This pattern is automatically generated from aarch64_ad.m4
11061 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11062 // val & (-1 ^ (val ror shift)) ==> bic
11063 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11064 iRegL src1, iRegL src2,
11065 immI src3, immL_M1 src4) %{
11066 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11067 ins_cost(1.9 * INSN_COST);
11068 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11069
11070 ins_encode %{
11071 __ bic(as_Register($dst$$reg),
11072 as_Register($src1$$reg),
11073 as_Register($src2$$reg),
11074 Assembler::ROR,
11075 $src3$$constant & 0x3f);
11076 %}
11077
11078 ins_pipe(ialu_reg_reg_shift);
11079 %}
11080
11081 // This pattern is automatically generated from aarch64_ad.m4
11082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11083 // val & (-1 ^ (val << shift)) ==> bicw
11084 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11085 iRegIorL2I src1, iRegIorL2I src2,
11086 immI src3, immI_M1 src4) %{
11087 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11088 ins_cost(1.9 * INSN_COST);
11089 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11090
11091 ins_encode %{
11092 __ bicw(as_Register($dst$$reg),
11093 as_Register($src1$$reg),
11094 as_Register($src2$$reg),
11095 Assembler::LSL,
11096 $src3$$constant & 0x1f);
11097 %}
11098
11099 ins_pipe(ialu_reg_reg_shift);
11100 %}
11101
11102 // This pattern is automatically generated from aarch64_ad.m4
11103 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11104 // val & (-1 ^ (val << shift)) ==> bic
11105 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11106 iRegL src1, iRegL src2,
11107 immI src3, immL_M1 src4) %{
11108 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11109 ins_cost(1.9 * INSN_COST);
11110 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11111
11112 ins_encode %{
11113 __ bic(as_Register($dst$$reg),
11114 as_Register($src1$$reg),
11115 as_Register($src2$$reg),
11116 Assembler::LSL,
11117 $src3$$constant & 0x3f);
11118 %}
11119
11120 ins_pipe(ialu_reg_reg_shift);
11121 %}
11122
11123 // This pattern is automatically generated from aarch64_ad.m4
11124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11125 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11126 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11127 iRegIorL2I src1, iRegIorL2I src2,
11128 immI src3, immI_M1 src4) %{
11129 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11130 ins_cost(1.9 * INSN_COST);
11131 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11132
11133 ins_encode %{
11134 __ eonw(as_Register($dst$$reg),
11135 as_Register($src1$$reg),
11136 as_Register($src2$$reg),
11137 Assembler::LSR,
11138 $src3$$constant & 0x1f);
11139 %}
11140
11141 ins_pipe(ialu_reg_reg_shift);
11142 %}
11143
11144 // This pattern is automatically generated from aarch64_ad.m4
11145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11146 // val ^ (-1 ^ (val >>> shift)) ==> eon
11147 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11148 iRegL src1, iRegL src2,
11149 immI src3, immL_M1 src4) %{
11150 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11151 ins_cost(1.9 * INSN_COST);
11152 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11153
11154 ins_encode %{
11155 __ eon(as_Register($dst$$reg),
11156 as_Register($src1$$reg),
11157 as_Register($src2$$reg),
11158 Assembler::LSR,
11159 $src3$$constant & 0x3f);
11160 %}
11161
11162 ins_pipe(ialu_reg_reg_shift);
11163 %}
11164
11165 // This pattern is automatically generated from aarch64_ad.m4
11166 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11167 // val ^ (-1 ^ (val >> shift)) ==> eonw
11168 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11169 iRegIorL2I src1, iRegIorL2I src2,
11170 immI src3, immI_M1 src4) %{
11171 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11172 ins_cost(1.9 * INSN_COST);
11173 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11174
11175 ins_encode %{
11176 __ eonw(as_Register($dst$$reg),
11177 as_Register($src1$$reg),
11178 as_Register($src2$$reg),
11179 Assembler::ASR,
11180 $src3$$constant & 0x1f);
11181 %}
11182
11183 ins_pipe(ialu_reg_reg_shift);
11184 %}
11185
11186 // This pattern is automatically generated from aarch64_ad.m4
11187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11188 // val ^ (-1 ^ (val >> shift)) ==> eon
11189 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11190 iRegL src1, iRegL src2,
11191 immI src3, immL_M1 src4) %{
11192 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11193 ins_cost(1.9 * INSN_COST);
11194 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11195
11196 ins_encode %{
11197 __ eon(as_Register($dst$$reg),
11198 as_Register($src1$$reg),
11199 as_Register($src2$$reg),
11200 Assembler::ASR,
11201 $src3$$constant & 0x3f);
11202 %}
11203
11204 ins_pipe(ialu_reg_reg_shift);
11205 %}
11206
11207 // This pattern is automatically generated from aarch64_ad.m4
11208 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11209 // val ^ (-1 ^ (val ror shift)) ==> eonw
11210 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11211 iRegIorL2I src1, iRegIorL2I src2,
11212 immI src3, immI_M1 src4) %{
11213 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11214 ins_cost(1.9 * INSN_COST);
11215 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11216
11217 ins_encode %{
11218 __ eonw(as_Register($dst$$reg),
11219 as_Register($src1$$reg),
11220 as_Register($src2$$reg),
11221 Assembler::ROR,
11222 $src3$$constant & 0x1f);
11223 %}
11224
11225 ins_pipe(ialu_reg_reg_shift);
11226 %}
11227
11228 // This pattern is automatically generated from aarch64_ad.m4
11229 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11230 // val ^ (-1 ^ (val ror shift)) ==> eon
11231 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11232 iRegL src1, iRegL src2,
11233 immI src3, immL_M1 src4) %{
11234 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11235 ins_cost(1.9 * INSN_COST);
11236 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11237
11238 ins_encode %{
11239 __ eon(as_Register($dst$$reg),
11240 as_Register($src1$$reg),
11241 as_Register($src2$$reg),
11242 Assembler::ROR,
11243 $src3$$constant & 0x3f);
11244 %}
11245
11246 ins_pipe(ialu_reg_reg_shift);
11247 %}
11248
11249 // This pattern is automatically generated from aarch64_ad.m4
11250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11251 // val ^ (-1 ^ (val << shift)) ==> eonw
11252 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11253 iRegIorL2I src1, iRegIorL2I src2,
11254 immI src3, immI_M1 src4) %{
11255 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11256 ins_cost(1.9 * INSN_COST);
11257 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11258
11259 ins_encode %{
11260 __ eonw(as_Register($dst$$reg),
11261 as_Register($src1$$reg),
11262 as_Register($src2$$reg),
11263 Assembler::LSL,
11264 $src3$$constant & 0x1f);
11265 %}
11266
11267 ins_pipe(ialu_reg_reg_shift);
11268 %}
11269
11270 // This pattern is automatically generated from aarch64_ad.m4
11271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11272 // val ^ (-1 ^ (val << shift)) ==> eon
11273 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11274 iRegL src1, iRegL src2,
11275 immI src3, immL_M1 src4) %{
11276 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11277 ins_cost(1.9 * INSN_COST);
11278 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11279
11280 ins_encode %{
11281 __ eon(as_Register($dst$$reg),
11282 as_Register($src1$$reg),
11283 as_Register($src2$$reg),
11284 Assembler::LSL,
11285 $src3$$constant & 0x3f);
11286 %}
11287
11288 ins_pipe(ialu_reg_reg_shift);
11289 %}
11290
11291 // This pattern is automatically generated from aarch64_ad.m4
11292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11293 // val | (-1 ^ (val >>> shift)) ==> ornw
11294 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11295 iRegIorL2I src1, iRegIorL2I src2,
11296 immI src3, immI_M1 src4) %{
11297 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11298 ins_cost(1.9 * INSN_COST);
11299 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11300
11301 ins_encode %{
11302 __ ornw(as_Register($dst$$reg),
11303 as_Register($src1$$reg),
11304 as_Register($src2$$reg),
11305 Assembler::LSR,
11306 $src3$$constant & 0x1f);
11307 %}
11308
11309 ins_pipe(ialu_reg_reg_shift);
11310 %}
11311
11312 // This pattern is automatically generated from aarch64_ad.m4
11313 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11314 // val | (-1 ^ (val >>> shift)) ==> orn
11315 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11316 iRegL src1, iRegL src2,
11317 immI src3, immL_M1 src4) %{
11318 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11319 ins_cost(1.9 * INSN_COST);
11320 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11321
11322 ins_encode %{
11323 __ orn(as_Register($dst$$reg),
11324 as_Register($src1$$reg),
11325 as_Register($src2$$reg),
11326 Assembler::LSR,
11327 $src3$$constant & 0x3f);
11328 %}
11329
11330 ins_pipe(ialu_reg_reg_shift);
11331 %}
11332
11333 // This pattern is automatically generated from aarch64_ad.m4
11334 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11335 // val | (-1 ^ (val >> shift)) ==> ornw
11336 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11337 iRegIorL2I src1, iRegIorL2I src2,
11338 immI src3, immI_M1 src4) %{
11339 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11340 ins_cost(1.9 * INSN_COST);
11341 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11342
11343 ins_encode %{
11344 __ ornw(as_Register($dst$$reg),
11345 as_Register($src1$$reg),
11346 as_Register($src2$$reg),
11347 Assembler::ASR,
11348 $src3$$constant & 0x1f);
11349 %}
11350
11351 ins_pipe(ialu_reg_reg_shift);
11352 %}
11353
11354 // This pattern is automatically generated from aarch64_ad.m4
11355 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11356 // val | (-1 ^ (val >> shift)) ==> orn
11357 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11358 iRegL src1, iRegL src2,
11359 immI src3, immL_M1 src4) %{
11360 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11361 ins_cost(1.9 * INSN_COST);
11362 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11363
11364 ins_encode %{
11365 __ orn(as_Register($dst$$reg),
11366 as_Register($src1$$reg),
11367 as_Register($src2$$reg),
11368 Assembler::ASR,
11369 $src3$$constant & 0x3f);
11370 %}
11371
11372 ins_pipe(ialu_reg_reg_shift);
11373 %}
11374
11375 // This pattern is automatically generated from aarch64_ad.m4
11376 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11377 // val | (-1 ^ (val ror shift)) ==> ornw
11378 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11379 iRegIorL2I src1, iRegIorL2I src2,
11380 immI src3, immI_M1 src4) %{
11381 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11382 ins_cost(1.9 * INSN_COST);
11383 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11384
11385 ins_encode %{
11386 __ ornw(as_Register($dst$$reg),
11387 as_Register($src1$$reg),
11388 as_Register($src2$$reg),
11389 Assembler::ROR,
11390 $src3$$constant & 0x1f);
11391 %}
11392
11393 ins_pipe(ialu_reg_reg_shift);
11394 %}
11395
11396 // This pattern is automatically generated from aarch64_ad.m4
11397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11398 // val | (-1 ^ (val ror shift)) ==> orn
11399 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11400 iRegL src1, iRegL src2,
11401 immI src3, immL_M1 src4) %{
11402 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11403 ins_cost(1.9 * INSN_COST);
11404 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11405
11406 ins_encode %{
11407 __ orn(as_Register($dst$$reg),
11408 as_Register($src1$$reg),
11409 as_Register($src2$$reg),
11410 Assembler::ROR,
11411 $src3$$constant & 0x3f);
11412 %}
11413
11414 ins_pipe(ialu_reg_reg_shift);
11415 %}
11416
11417 // This pattern is automatically generated from aarch64_ad.m4
11418 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11419 // val | (-1 ^ (val << shift)) ==> ornw
11420 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11421 iRegIorL2I src1, iRegIorL2I src2,
11422 immI src3, immI_M1 src4) %{
11423 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11424 ins_cost(1.9 * INSN_COST);
11425 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11426
11427 ins_encode %{
11428 __ ornw(as_Register($dst$$reg),
11429 as_Register($src1$$reg),
11430 as_Register($src2$$reg),
11431 Assembler::LSL,
11432 $src3$$constant & 0x1f);
11433 %}
11434
11435 ins_pipe(ialu_reg_reg_shift);
11436 %}
11437
11438 // This pattern is automatically generated from aarch64_ad.m4
11439 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11440 // val | (-1 ^ (val << shift)) ==> orn
11441 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11442 iRegL src1, iRegL src2,
11443 immI src3, immL_M1 src4) %{
11444 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11445 ins_cost(1.9 * INSN_COST);
11446 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11447
11448 ins_encode %{
11449 __ orn(as_Register($dst$$reg),
11450 as_Register($src1$$reg),
11451 as_Register($src2$$reg),
11452 Assembler::LSL,
11453 $src3$$constant & 0x3f);
11454 %}
11455
11456 ins_pipe(ialu_reg_reg_shift);
11457 %}
11458
11459 // This pattern is automatically generated from aarch64_ad.m4
11460 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11461 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11462 iRegIorL2I src1, iRegIorL2I src2,
11463 immI src3) %{
11464 match(Set dst (AndI src1 (URShiftI src2 src3)));
11465
11466 ins_cost(1.9 * INSN_COST);
11467 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11468
11469 ins_encode %{
11470 __ andw(as_Register($dst$$reg),
11471 as_Register($src1$$reg),
11472 as_Register($src2$$reg),
11473 Assembler::LSR,
11474 $src3$$constant & 0x1f);
11475 %}
11476
11477 ins_pipe(ialu_reg_reg_shift);
11478 %}
11479
11480 // This pattern is automatically generated from aarch64_ad.m4
11481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11482 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11483 iRegL src1, iRegL src2,
11484 immI src3) %{
11485 match(Set dst (AndL src1 (URShiftL src2 src3)));
11486
11487 ins_cost(1.9 * INSN_COST);
11488 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11489
11490 ins_encode %{
11491 __ andr(as_Register($dst$$reg),
11492 as_Register($src1$$reg),
11493 as_Register($src2$$reg),
11494 Assembler::LSR,
11495 $src3$$constant & 0x3f);
11496 %}
11497
11498 ins_pipe(ialu_reg_reg_shift);
11499 %}
11500
11501 // This pattern is automatically generated from aarch64_ad.m4
11502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11503 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11504 iRegIorL2I src1, iRegIorL2I src2,
11505 immI src3) %{
11506 match(Set dst (AndI src1 (RShiftI src2 src3)));
11507
11508 ins_cost(1.9 * INSN_COST);
11509 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11510
11511 ins_encode %{
11512 __ andw(as_Register($dst$$reg),
11513 as_Register($src1$$reg),
11514 as_Register($src2$$reg),
11515 Assembler::ASR,
11516 $src3$$constant & 0x1f);
11517 %}
11518
11519 ins_pipe(ialu_reg_reg_shift);
11520 %}
11521
11522 // This pattern is automatically generated from aarch64_ad.m4
11523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11524 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11525 iRegL src1, iRegL src2,
11526 immI src3) %{
11527 match(Set dst (AndL src1 (RShiftL src2 src3)));
11528
11529 ins_cost(1.9 * INSN_COST);
11530 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11531
11532 ins_encode %{
11533 __ andr(as_Register($dst$$reg),
11534 as_Register($src1$$reg),
11535 as_Register($src2$$reg),
11536 Assembler::ASR,
11537 $src3$$constant & 0x3f);
11538 %}
11539
11540 ins_pipe(ialu_reg_reg_shift);
11541 %}
11542
11543 // This pattern is automatically generated from aarch64_ad.m4
11544 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11545 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11546 iRegIorL2I src1, iRegIorL2I src2,
11547 immI src3) %{
11548 match(Set dst (AndI src1 (LShiftI src2 src3)));
11549
11550 ins_cost(1.9 * INSN_COST);
11551 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11552
11553 ins_encode %{
11554 __ andw(as_Register($dst$$reg),
11555 as_Register($src1$$reg),
11556 as_Register($src2$$reg),
11557 Assembler::LSL,
11558 $src3$$constant & 0x1f);
11559 %}
11560
11561 ins_pipe(ialu_reg_reg_shift);
11562 %}
11563
11564 // This pattern is automatically generated from aarch64_ad.m4
11565 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11566 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11567 iRegL src1, iRegL src2,
11568 immI src3) %{
11569 match(Set dst (AndL src1 (LShiftL src2 src3)));
11570
11571 ins_cost(1.9 * INSN_COST);
11572 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11573
11574 ins_encode %{
11575 __ andr(as_Register($dst$$reg),
11576 as_Register($src1$$reg),
11577 as_Register($src2$$reg),
11578 Assembler::LSL,
11579 $src3$$constant & 0x3f);
11580 %}
11581
11582 ins_pipe(ialu_reg_reg_shift);
11583 %}
11584
11585 // This pattern is automatically generated from aarch64_ad.m4
11586 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11587 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11588 iRegIorL2I src1, iRegIorL2I src2,
11589 immI src3) %{
11590 match(Set dst (AndI src1 (RotateRight src2 src3)));
11591
11592 ins_cost(1.9 * INSN_COST);
11593 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11594
11595 ins_encode %{
11596 __ andw(as_Register($dst$$reg),
11597 as_Register($src1$$reg),
11598 as_Register($src2$$reg),
11599 Assembler::ROR,
11600 $src3$$constant & 0x1f);
11601 %}
11602
11603 ins_pipe(ialu_reg_reg_shift);
11604 %}
11605
11606 // This pattern is automatically generated from aarch64_ad.m4
11607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11608 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11609 iRegL src1, iRegL src2,
11610 immI src3) %{
11611 match(Set dst (AndL src1 (RotateRight src2 src3)));
11612
11613 ins_cost(1.9 * INSN_COST);
11614 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11615
11616 ins_encode %{
11617 __ andr(as_Register($dst$$reg),
11618 as_Register($src1$$reg),
11619 as_Register($src2$$reg),
11620 Assembler::ROR,
11621 $src3$$constant & 0x3f);
11622 %}
11623
11624 ins_pipe(ialu_reg_reg_shift);
11625 %}
11626
11627 // This pattern is automatically generated from aarch64_ad.m4
11628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11629 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11630 iRegIorL2I src1, iRegIorL2I src2,
11631 immI src3) %{
11632 match(Set dst (XorI src1 (URShiftI src2 src3)));
11633
11634 ins_cost(1.9 * INSN_COST);
11635 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11636
11637 ins_encode %{
11638 __ eorw(as_Register($dst$$reg),
11639 as_Register($src1$$reg),
11640 as_Register($src2$$reg),
11641 Assembler::LSR,
11642 $src3$$constant & 0x1f);
11643 %}
11644
11645 ins_pipe(ialu_reg_reg_shift);
11646 %}
11647
11648 // This pattern is automatically generated from aarch64_ad.m4
11649 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11650 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11651 iRegL src1, iRegL src2,
11652 immI src3) %{
11653 match(Set dst (XorL src1 (URShiftL src2 src3)));
11654
11655 ins_cost(1.9 * INSN_COST);
11656 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11657
11658 ins_encode %{
11659 __ eor(as_Register($dst$$reg),
11660 as_Register($src1$$reg),
11661 as_Register($src2$$reg),
11662 Assembler::LSR,
11663 $src3$$constant & 0x3f);
11664 %}
11665
11666 ins_pipe(ialu_reg_reg_shift);
11667 %}
11668
11669 // This pattern is automatically generated from aarch64_ad.m4
11670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11671 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11672 iRegIorL2I src1, iRegIorL2I src2,
11673 immI src3) %{
11674 match(Set dst (XorI src1 (RShiftI src2 src3)));
11675
11676 ins_cost(1.9 * INSN_COST);
11677 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11678
11679 ins_encode %{
11680 __ eorw(as_Register($dst$$reg),
11681 as_Register($src1$$reg),
11682 as_Register($src2$$reg),
11683 Assembler::ASR,
11684 $src3$$constant & 0x1f);
11685 %}
11686
11687 ins_pipe(ialu_reg_reg_shift);
11688 %}
11689
11690 // This pattern is automatically generated from aarch64_ad.m4
11691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11692 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11693 iRegL src1, iRegL src2,
11694 immI src3) %{
11695 match(Set dst (XorL src1 (RShiftL src2 src3)));
11696
11697 ins_cost(1.9 * INSN_COST);
11698 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11699
11700 ins_encode %{
11701 __ eor(as_Register($dst$$reg),
11702 as_Register($src1$$reg),
11703 as_Register($src2$$reg),
11704 Assembler::ASR,
11705 $src3$$constant & 0x3f);
11706 %}
11707
11708 ins_pipe(ialu_reg_reg_shift);
11709 %}
11710
11711 // This pattern is automatically generated from aarch64_ad.m4
11712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11713 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11714 iRegIorL2I src1, iRegIorL2I src2,
11715 immI src3) %{
11716 match(Set dst (XorI src1 (LShiftI src2 src3)));
11717
11718 ins_cost(1.9 * INSN_COST);
11719 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11720
11721 ins_encode %{
11722 __ eorw(as_Register($dst$$reg),
11723 as_Register($src1$$reg),
11724 as_Register($src2$$reg),
11725 Assembler::LSL,
11726 $src3$$constant & 0x1f);
11727 %}
11728
11729 ins_pipe(ialu_reg_reg_shift);
11730 %}
11731
11732 // This pattern is automatically generated from aarch64_ad.m4
11733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11734 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11735 iRegL src1, iRegL src2,
11736 immI src3) %{
11737 match(Set dst (XorL src1 (LShiftL src2 src3)));
11738
11739 ins_cost(1.9 * INSN_COST);
11740 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11741
11742 ins_encode %{
11743 __ eor(as_Register($dst$$reg),
11744 as_Register($src1$$reg),
11745 as_Register($src2$$reg),
11746 Assembler::LSL,
11747 $src3$$constant & 0x3f);
11748 %}
11749
11750 ins_pipe(ialu_reg_reg_shift);
11751 %}
11752
11753 // This pattern is automatically generated from aarch64_ad.m4
11754 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11755 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11756 iRegIorL2I src1, iRegIorL2I src2,
11757 immI src3) %{
11758 match(Set dst (XorI src1 (RotateRight src2 src3)));
11759
11760 ins_cost(1.9 * INSN_COST);
11761 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11762
11763 ins_encode %{
11764 __ eorw(as_Register($dst$$reg),
11765 as_Register($src1$$reg),
11766 as_Register($src2$$reg),
11767 Assembler::ROR,
11768 $src3$$constant & 0x1f);
11769 %}
11770
11771 ins_pipe(ialu_reg_reg_shift);
11772 %}
11773
11774 // This pattern is automatically generated from aarch64_ad.m4
11775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11776 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11777 iRegL src1, iRegL src2,
11778 immI src3) %{
11779 match(Set dst (XorL src1 (RotateRight src2 src3)));
11780
11781 ins_cost(1.9 * INSN_COST);
11782 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11783
11784 ins_encode %{
11785 __ eor(as_Register($dst$$reg),
11786 as_Register($src1$$reg),
11787 as_Register($src2$$reg),
11788 Assembler::ROR,
11789 $src3$$constant & 0x3f);
11790 %}
11791
11792 ins_pipe(ialu_reg_reg_shift);
11793 %}
11794
11795 // This pattern is automatically generated from aarch64_ad.m4
11796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11797 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11798 iRegIorL2I src1, iRegIorL2I src2,
11799 immI src3) %{
11800 match(Set dst (OrI src1 (URShiftI src2 src3)));
11801
11802 ins_cost(1.9 * INSN_COST);
11803 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11804
11805 ins_encode %{
11806 __ orrw(as_Register($dst$$reg),
11807 as_Register($src1$$reg),
11808 as_Register($src2$$reg),
11809 Assembler::LSR,
11810 $src3$$constant & 0x1f);
11811 %}
11812
11813 ins_pipe(ialu_reg_reg_shift);
11814 %}
11815
11816 // This pattern is automatically generated from aarch64_ad.m4
11817 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11818 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11819 iRegL src1, iRegL src2,
11820 immI src3) %{
11821 match(Set dst (OrL src1 (URShiftL src2 src3)));
11822
11823 ins_cost(1.9 * INSN_COST);
11824 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11825
11826 ins_encode %{
11827 __ orr(as_Register($dst$$reg),
11828 as_Register($src1$$reg),
11829 as_Register($src2$$reg),
11830 Assembler::LSR,
11831 $src3$$constant & 0x3f);
11832 %}
11833
11834 ins_pipe(ialu_reg_reg_shift);
11835 %}
11836
11837 // This pattern is automatically generated from aarch64_ad.m4
11838 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11839 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11840 iRegIorL2I src1, iRegIorL2I src2,
11841 immI src3) %{
11842 match(Set dst (OrI src1 (RShiftI src2 src3)));
11843
11844 ins_cost(1.9 * INSN_COST);
11845 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11846
11847 ins_encode %{
11848 __ orrw(as_Register($dst$$reg),
11849 as_Register($src1$$reg),
11850 as_Register($src2$$reg),
11851 Assembler::ASR,
11852 $src3$$constant & 0x1f);
11853 %}
11854
11855 ins_pipe(ialu_reg_reg_shift);
11856 %}
11857
11858 // This pattern is automatically generated from aarch64_ad.m4
11859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11860 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11861 iRegL src1, iRegL src2,
11862 immI src3) %{
11863 match(Set dst (OrL src1 (RShiftL src2 src3)));
11864
11865 ins_cost(1.9 * INSN_COST);
11866 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11867
11868 ins_encode %{
11869 __ orr(as_Register($dst$$reg),
11870 as_Register($src1$$reg),
11871 as_Register($src2$$reg),
11872 Assembler::ASR,
11873 $src3$$constant & 0x3f);
11874 %}
11875
11876 ins_pipe(ialu_reg_reg_shift);
11877 %}
11878
11879 // This pattern is automatically generated from aarch64_ad.m4
11880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11881 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11882 iRegIorL2I src1, iRegIorL2I src2,
11883 immI src3) %{
11884 match(Set dst (OrI src1 (LShiftI src2 src3)));
11885
11886 ins_cost(1.9 * INSN_COST);
11887 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11888
11889 ins_encode %{
11890 __ orrw(as_Register($dst$$reg),
11891 as_Register($src1$$reg),
11892 as_Register($src2$$reg),
11893 Assembler::LSL,
11894 $src3$$constant & 0x1f);
11895 %}
11896
11897 ins_pipe(ialu_reg_reg_shift);
11898 %}
11899
11900 // This pattern is automatically generated from aarch64_ad.m4
11901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11902 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11903 iRegL src1, iRegL src2,
11904 immI src3) %{
11905 match(Set dst (OrL src1 (LShiftL src2 src3)));
11906
11907 ins_cost(1.9 * INSN_COST);
11908 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11909
11910 ins_encode %{
11911 __ orr(as_Register($dst$$reg),
11912 as_Register($src1$$reg),
11913 as_Register($src2$$reg),
11914 Assembler::LSL,
11915 $src3$$constant & 0x3f);
11916 %}
11917
11918 ins_pipe(ialu_reg_reg_shift);
11919 %}
11920
11921 // This pattern is automatically generated from aarch64_ad.m4
11922 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11923 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11924 iRegIorL2I src1, iRegIorL2I src2,
11925 immI src3) %{
11926 match(Set dst (OrI src1 (RotateRight src2 src3)));
11927
11928 ins_cost(1.9 * INSN_COST);
11929 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11930
11931 ins_encode %{
11932 __ orrw(as_Register($dst$$reg),
11933 as_Register($src1$$reg),
11934 as_Register($src2$$reg),
11935 Assembler::ROR,
11936 $src3$$constant & 0x1f);
11937 %}
11938
11939 ins_pipe(ialu_reg_reg_shift);
11940 %}
11941
11942 // This pattern is automatically generated from aarch64_ad.m4
11943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11944 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11945 iRegL src1, iRegL src2,
11946 immI src3) %{
11947 match(Set dst (OrL src1 (RotateRight src2 src3)));
11948
11949 ins_cost(1.9 * INSN_COST);
11950 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11951
11952 ins_encode %{
11953 __ orr(as_Register($dst$$reg),
11954 as_Register($src1$$reg),
11955 as_Register($src2$$reg),
11956 Assembler::ROR,
11957 $src3$$constant & 0x3f);
11958 %}
11959
11960 ins_pipe(ialu_reg_reg_shift);
11961 %}
11962
11963 // This pattern is automatically generated from aarch64_ad.m4
11964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11965 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11966 iRegIorL2I src1, iRegIorL2I src2,
11967 immI src3) %{
11968 match(Set dst (AddI src1 (URShiftI src2 src3)));
11969
11970 ins_cost(1.9 * INSN_COST);
11971 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11972
11973 ins_encode %{
11974 __ addw(as_Register($dst$$reg),
11975 as_Register($src1$$reg),
11976 as_Register($src2$$reg),
11977 Assembler::LSR,
11978 $src3$$constant & 0x1f);
11979 %}
11980
11981 ins_pipe(ialu_reg_reg_shift);
11982 %}
11983
11984 // This pattern is automatically generated from aarch64_ad.m4
11985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11986 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11987 iRegL src1, iRegL src2,
11988 immI src3) %{
11989 match(Set dst (AddL src1 (URShiftL src2 src3)));
11990
11991 ins_cost(1.9 * INSN_COST);
11992 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11993
11994 ins_encode %{
11995 __ add(as_Register($dst$$reg),
11996 as_Register($src1$$reg),
11997 as_Register($src2$$reg),
11998 Assembler::LSR,
11999 $src3$$constant & 0x3f);
12000 %}
12001
12002 ins_pipe(ialu_reg_reg_shift);
12003 %}
12004
12005 // This pattern is automatically generated from aarch64_ad.m4
12006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12007 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12008 iRegIorL2I src1, iRegIorL2I src2,
12009 immI src3) %{
12010 match(Set dst (AddI src1 (RShiftI src2 src3)));
12011
12012 ins_cost(1.9 * INSN_COST);
12013 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12014
12015 ins_encode %{
12016 __ addw(as_Register($dst$$reg),
12017 as_Register($src1$$reg),
12018 as_Register($src2$$reg),
12019 Assembler::ASR,
12020 $src3$$constant & 0x1f);
12021 %}
12022
12023 ins_pipe(ialu_reg_reg_shift);
12024 %}
12025
12026 // This pattern is automatically generated from aarch64_ad.m4
12027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12028 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12029 iRegL src1, iRegL src2,
12030 immI src3) %{
12031 match(Set dst (AddL src1 (RShiftL src2 src3)));
12032
12033 ins_cost(1.9 * INSN_COST);
12034 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12035
12036 ins_encode %{
12037 __ add(as_Register($dst$$reg),
12038 as_Register($src1$$reg),
12039 as_Register($src2$$reg),
12040 Assembler::ASR,
12041 $src3$$constant & 0x3f);
12042 %}
12043
12044 ins_pipe(ialu_reg_reg_shift);
12045 %}
12046
12047 // This pattern is automatically generated from aarch64_ad.m4
12048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12049 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12050 iRegIorL2I src1, iRegIorL2I src2,
12051 immI src3) %{
12052 match(Set dst (AddI src1 (LShiftI src2 src3)));
12053
12054 ins_cost(1.9 * INSN_COST);
12055 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12056
12057 ins_encode %{
12058 __ addw(as_Register($dst$$reg),
12059 as_Register($src1$$reg),
12060 as_Register($src2$$reg),
12061 Assembler::LSL,
12062 $src3$$constant & 0x1f);
12063 %}
12064
12065 ins_pipe(ialu_reg_reg_shift);
12066 %}
12067
12068 // This pattern is automatically generated from aarch64_ad.m4
12069 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12070 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12071 iRegL src1, iRegL src2,
12072 immI src3) %{
12073 match(Set dst (AddL src1 (LShiftL src2 src3)));
12074
12075 ins_cost(1.9 * INSN_COST);
12076 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12077
12078 ins_encode %{
12079 __ add(as_Register($dst$$reg),
12080 as_Register($src1$$reg),
12081 as_Register($src2$$reg),
12082 Assembler::LSL,
12083 $src3$$constant & 0x3f);
12084 %}
12085
12086 ins_pipe(ialu_reg_reg_shift);
12087 %}
12088
12089 // This pattern is automatically generated from aarch64_ad.m4
12090 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12091 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12092 iRegIorL2I src1, iRegIorL2I src2,
12093 immI src3) %{
12094 match(Set dst (SubI src1 (URShiftI src2 src3)));
12095
12096 ins_cost(1.9 * INSN_COST);
12097 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12098
12099 ins_encode %{
12100 __ subw(as_Register($dst$$reg),
12101 as_Register($src1$$reg),
12102 as_Register($src2$$reg),
12103 Assembler::LSR,
12104 $src3$$constant & 0x1f);
12105 %}
12106
12107 ins_pipe(ialu_reg_reg_shift);
12108 %}
12109
12110 // This pattern is automatically generated from aarch64_ad.m4
12111 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12112 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12113 iRegL src1, iRegL src2,
12114 immI src3) %{
12115 match(Set dst (SubL src1 (URShiftL src2 src3)));
12116
12117 ins_cost(1.9 * INSN_COST);
12118 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12119
12120 ins_encode %{
12121 __ sub(as_Register($dst$$reg),
12122 as_Register($src1$$reg),
12123 as_Register($src2$$reg),
12124 Assembler::LSR,
12125 $src3$$constant & 0x3f);
12126 %}
12127
12128 ins_pipe(ialu_reg_reg_shift);
12129 %}
12130
12131 // This pattern is automatically generated from aarch64_ad.m4
12132 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12133 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12134 iRegIorL2I src1, iRegIorL2I src2,
12135 immI src3) %{
12136 match(Set dst (SubI src1 (RShiftI src2 src3)));
12137
12138 ins_cost(1.9 * INSN_COST);
12139 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12140
12141 ins_encode %{
12142 __ subw(as_Register($dst$$reg),
12143 as_Register($src1$$reg),
12144 as_Register($src2$$reg),
12145 Assembler::ASR,
12146 $src3$$constant & 0x1f);
12147 %}
12148
12149 ins_pipe(ialu_reg_reg_shift);
12150 %}
12151
12152 // This pattern is automatically generated from aarch64_ad.m4
12153 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12154 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12155 iRegL src1, iRegL src2,
12156 immI src3) %{
12157 match(Set dst (SubL src1 (RShiftL src2 src3)));
12158
12159 ins_cost(1.9 * INSN_COST);
12160 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12161
12162 ins_encode %{
12163 __ sub(as_Register($dst$$reg),
12164 as_Register($src1$$reg),
12165 as_Register($src2$$reg),
12166 Assembler::ASR,
12167 $src3$$constant & 0x3f);
12168 %}
12169
12170 ins_pipe(ialu_reg_reg_shift);
12171 %}
12172
12173 // This pattern is automatically generated from aarch64_ad.m4
12174 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12175 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12176 iRegIorL2I src1, iRegIorL2I src2,
12177 immI src3) %{
12178 match(Set dst (SubI src1 (LShiftI src2 src3)));
12179
12180 ins_cost(1.9 * INSN_COST);
12181 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12182
12183 ins_encode %{
12184 __ subw(as_Register($dst$$reg),
12185 as_Register($src1$$reg),
12186 as_Register($src2$$reg),
12187 Assembler::LSL,
12188 $src3$$constant & 0x1f);
12189 %}
12190
12191 ins_pipe(ialu_reg_reg_shift);
12192 %}
12193
12194 // This pattern is automatically generated from aarch64_ad.m4
12195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12196 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12197 iRegL src1, iRegL src2,
12198 immI src3) %{
12199 match(Set dst (SubL src1 (LShiftL src2 src3)));
12200
12201 ins_cost(1.9 * INSN_COST);
12202 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12203
12204 ins_encode %{
12205 __ sub(as_Register($dst$$reg),
12206 as_Register($src1$$reg),
12207 as_Register($src2$$reg),
12208 Assembler::LSL,
12209 $src3$$constant & 0x3f);
12210 %}
12211
12212 ins_pipe(ialu_reg_reg_shift);
12213 %}
12214
12215 // This pattern is automatically generated from aarch64_ad.m4
12216 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12217
12218 // Shift Left followed by Shift Right.
12219 // This idiom is used by the compiler for the i2b bytecode etc.
12220 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12221 %{
12222 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12223 ins_cost(INSN_COST * 2);
12224 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12225 ins_encode %{
12226 int lshift = $lshift_count$$constant & 63;
12227 int rshift = $rshift_count$$constant & 63;
12228 int s = 63 - lshift;
12229 int r = (rshift - lshift) & 63;
12230 __ sbfm(as_Register($dst$$reg),
12231 as_Register($src$$reg),
12232 r, s);
12233 %}
12234
12235 ins_pipe(ialu_reg_shift);
12236 %}
12237
12238 // This pattern is automatically generated from aarch64_ad.m4
12239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12240
12241 // Shift Left followed by Shift Right.
12242 // This idiom is used by the compiler for the i2b bytecode etc.
12243 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12244 %{
12245 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12246 ins_cost(INSN_COST * 2);
12247 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12248 ins_encode %{
12249 int lshift = $lshift_count$$constant & 31;
12250 int rshift = $rshift_count$$constant & 31;
12251 int s = 31 - lshift;
12252 int r = (rshift - lshift) & 31;
12253 __ sbfmw(as_Register($dst$$reg),
12254 as_Register($src$$reg),
12255 r, s);
12256 %}
12257
12258 ins_pipe(ialu_reg_shift);
12259 %}
12260
12261 // This pattern is automatically generated from aarch64_ad.m4
12262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12263
12264 // Shift Left followed by Shift Right.
12265 // This idiom is used by the compiler for the i2b bytecode etc.
12266 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12267 %{
12268 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12269 ins_cost(INSN_COST * 2);
12270 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12271 ins_encode %{
12272 int lshift = $lshift_count$$constant & 63;
12273 int rshift = $rshift_count$$constant & 63;
12274 int s = 63 - lshift;
12275 int r = (rshift - lshift) & 63;
12276 __ ubfm(as_Register($dst$$reg),
12277 as_Register($src$$reg),
12278 r, s);
12279 %}
12280
12281 ins_pipe(ialu_reg_shift);
12282 %}
12283
12284 // This pattern is automatically generated from aarch64_ad.m4
12285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12286
12287 // Shift Left followed by Shift Right.
12288 // This idiom is used by the compiler for the i2b bytecode etc.
12289 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12290 %{
12291 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12292 ins_cost(INSN_COST * 2);
12293 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12294 ins_encode %{
12295 int lshift = $lshift_count$$constant & 31;
12296 int rshift = $rshift_count$$constant & 31;
12297 int s = 31 - lshift;
12298 int r = (rshift - lshift) & 31;
12299 __ ubfmw(as_Register($dst$$reg),
12300 as_Register($src$$reg),
12301 r, s);
12302 %}
12303
12304 ins_pipe(ialu_reg_shift);
12305 %}
12306
12307 // Bitfield extract with shift & mask
12308
12309 // This pattern is automatically generated from aarch64_ad.m4
12310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12311 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12312 %{
12313 match(Set dst (AndI (URShiftI src rshift) mask));
12314 // Make sure we are not going to exceed what ubfxw can do.
12315 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12316
12317 ins_cost(INSN_COST);
12318 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12319 ins_encode %{
12320 int rshift = $rshift$$constant & 31;
12321 intptr_t mask = $mask$$constant;
12322 int width = exact_log2(mask+1);
12323 __ ubfxw(as_Register($dst$$reg),
12324 as_Register($src$$reg), rshift, width);
12325 %}
12326 ins_pipe(ialu_reg_shift);
12327 %}
12328
12329 // This pattern is automatically generated from aarch64_ad.m4
12330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12331 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12332 %{
12333 match(Set dst (AndL (URShiftL src rshift) mask));
12334 // Make sure we are not going to exceed what ubfx can do.
12335 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12336
12337 ins_cost(INSN_COST);
12338 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12339 ins_encode %{
12340 int rshift = $rshift$$constant & 63;
12341 intptr_t mask = $mask$$constant;
12342 int width = exact_log2_long(mask+1);
12343 __ ubfx(as_Register($dst$$reg),
12344 as_Register($src$$reg), rshift, width);
12345 %}
12346 ins_pipe(ialu_reg_shift);
12347 %}
12348
12349
12350 // This pattern is automatically generated from aarch64_ad.m4
12351 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12352
12353 // We can use ubfx when extending an And with a mask when we know mask
12354 // is positive. We know that because immI_bitmask guarantees it.
12355 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12356 %{
12357 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12358 // Make sure we are not going to exceed what ubfxw can do.
12359 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12360
12361 ins_cost(INSN_COST * 2);
12362 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12363 ins_encode %{
12364 int rshift = $rshift$$constant & 31;
12365 intptr_t mask = $mask$$constant;
12366 int width = exact_log2(mask+1);
12367 __ ubfx(as_Register($dst$$reg),
12368 as_Register($src$$reg), rshift, width);
12369 %}
12370 ins_pipe(ialu_reg_shift);
12371 %}
12372
12373
12374 // This pattern is automatically generated from aarch64_ad.m4
12375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12376
12377 // We can use ubfiz when masking by a positive number and then left shifting the result.
12378 // We know that the mask is positive because immI_bitmask guarantees it.
12379 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12380 %{
12381 match(Set dst (LShiftI (AndI src mask) lshift));
12382 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12383
12384 ins_cost(INSN_COST);
12385 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12386 ins_encode %{
12387 int lshift = $lshift$$constant & 31;
12388 intptr_t mask = $mask$$constant;
12389 int width = exact_log2(mask+1);
12390 __ ubfizw(as_Register($dst$$reg),
12391 as_Register($src$$reg), lshift, width);
12392 %}
12393 ins_pipe(ialu_reg_shift);
12394 %}
12395
12396 // This pattern is automatically generated from aarch64_ad.m4
12397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12398
12399 // We can use ubfiz when masking by a positive number and then left shifting the result.
12400 // We know that the mask is positive because immL_bitmask guarantees it.
12401 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12402 %{
12403 match(Set dst (LShiftL (AndL src mask) lshift));
12404 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12405
12406 ins_cost(INSN_COST);
12407 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12408 ins_encode %{
12409 int lshift = $lshift$$constant & 63;
12410 intptr_t mask = $mask$$constant;
12411 int width = exact_log2_long(mask+1);
12412 __ ubfiz(as_Register($dst$$reg),
12413 as_Register($src$$reg), lshift, width);
12414 %}
12415 ins_pipe(ialu_reg_shift);
12416 %}
12417
12418 // This pattern is automatically generated from aarch64_ad.m4
12419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12420
12421 // We can use ubfiz when masking by a positive number and then left shifting the result.
12422 // We know that the mask is positive because immI_bitmask guarantees it.
12423 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12424 %{
12425 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12426 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12427
12428 ins_cost(INSN_COST);
12429 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12430 ins_encode %{
12431 int lshift = $lshift$$constant & 31;
12432 intptr_t mask = $mask$$constant;
12433 int width = exact_log2(mask+1);
12434 __ ubfizw(as_Register($dst$$reg),
12435 as_Register($src$$reg), lshift, width);
12436 %}
12437 ins_pipe(ialu_reg_shift);
12438 %}
12439
12440 // This pattern is automatically generated from aarch64_ad.m4
12441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12442
12443 // We can use ubfiz when masking by a positive number and then left shifting the result.
12444 // We know that the mask is positive because immL_bitmask guarantees it.
12445 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12446 %{
12447 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12448 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12449
12450 ins_cost(INSN_COST);
12451 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12452 ins_encode %{
12453 int lshift = $lshift$$constant & 63;
12454 intptr_t mask = $mask$$constant;
12455 int width = exact_log2_long(mask+1);
12456 __ ubfiz(as_Register($dst$$reg),
12457 as_Register($src$$reg), lshift, width);
12458 %}
12459 ins_pipe(ialu_reg_shift);
12460 %}
12461
12462
12463 // This pattern is automatically generated from aarch64_ad.m4
12464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12465
12466 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12467 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12468 %{
12469 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12470 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12471
12472 ins_cost(INSN_COST);
12473 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12474 ins_encode %{
12475 int lshift = $lshift$$constant & 63;
12476 intptr_t mask = $mask$$constant;
12477 int width = exact_log2(mask+1);
12478 __ ubfiz(as_Register($dst$$reg),
12479 as_Register($src$$reg), lshift, width);
12480 %}
12481 ins_pipe(ialu_reg_shift);
12482 %}
12483
12484 // This pattern is automatically generated from aarch64_ad.m4
12485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12486
12487 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12488 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12489 %{
12490 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12491 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12492
12493 ins_cost(INSN_COST);
12494 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12495 ins_encode %{
12496 int lshift = $lshift$$constant & 31;
12497 intptr_t mask = $mask$$constant;
12498 int width = exact_log2(mask+1);
12499 __ ubfiz(as_Register($dst$$reg),
12500 as_Register($src$$reg), lshift, width);
12501 %}
12502 ins_pipe(ialu_reg_shift);
12503 %}
12504
12505 // This pattern is automatically generated from aarch64_ad.m4
12506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12507
12508 // Can skip int2long conversions after AND with small bitmask
12509 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12510 %{
12511 match(Set dst (ConvI2L (AndI src msk)));
12512 ins_cost(INSN_COST);
12513 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12514 ins_encode %{
12515 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12516 %}
12517 ins_pipe(ialu_reg_shift);
12518 %}
12519
12520
12521 // Rotations
12522
12523 // This pattern is automatically generated from aarch64_ad.m4
12524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12525 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12526 %{
12527 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12528 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12529
12530 ins_cost(INSN_COST);
12531 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12532
12533 ins_encode %{
12534 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12535 $rshift$$constant & 63);
12536 %}
12537 ins_pipe(ialu_reg_reg_extr);
12538 %}
12539
12540
12541 // This pattern is automatically generated from aarch64_ad.m4
12542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12543 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12544 %{
12545 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12546 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12547
12548 ins_cost(INSN_COST);
12549 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12550
12551 ins_encode %{
12552 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12553 $rshift$$constant & 31);
12554 %}
12555 ins_pipe(ialu_reg_reg_extr);
12556 %}
12557
12558
12559 // This pattern is automatically generated from aarch64_ad.m4
12560 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12561 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12562 %{
12563 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12564 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12565
12566 ins_cost(INSN_COST);
12567 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12568
12569 ins_encode %{
12570 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12571 $rshift$$constant & 63);
12572 %}
12573 ins_pipe(ialu_reg_reg_extr);
12574 %}
12575
12576
12577 // This pattern is automatically generated from aarch64_ad.m4
12578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12579 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12580 %{
12581 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12582 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12583
12584 ins_cost(INSN_COST);
12585 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12586
12587 ins_encode %{
12588 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12589 $rshift$$constant & 31);
12590 %}
12591 ins_pipe(ialu_reg_reg_extr);
12592 %}
12593
12594 // This pattern is automatically generated from aarch64_ad.m4
12595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12596 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12597 %{
12598 match(Set dst (RotateRight src shift));
12599
12600 ins_cost(INSN_COST);
12601 format %{ "ror $dst, $src, $shift" %}
12602
12603 ins_encode %{
12604 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12605 $shift$$constant & 0x1f);
12606 %}
12607 ins_pipe(ialu_reg_reg_vshift);
12608 %}
12609
12610 // This pattern is automatically generated from aarch64_ad.m4
12611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12612 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12613 %{
12614 match(Set dst (RotateRight src shift));
12615
12616 ins_cost(INSN_COST);
12617 format %{ "ror $dst, $src, $shift" %}
12618
12619 ins_encode %{
12620 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12621 $shift$$constant & 0x3f);
12622 %}
12623 ins_pipe(ialu_reg_reg_vshift);
12624 %}
12625
12626 // This pattern is automatically generated from aarch64_ad.m4
12627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12628 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12629 %{
12630 match(Set dst (RotateRight src shift));
12631
12632 ins_cost(INSN_COST);
12633 format %{ "ror $dst, $src, $shift" %}
12634
12635 ins_encode %{
12636 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12637 %}
12638 ins_pipe(ialu_reg_reg_vshift);
12639 %}
12640
12641 // This pattern is automatically generated from aarch64_ad.m4
12642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12643 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12644 %{
12645 match(Set dst (RotateRight src shift));
12646
12647 ins_cost(INSN_COST);
12648 format %{ "ror $dst, $src, $shift" %}
12649
12650 ins_encode %{
12651 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12652 %}
12653 ins_pipe(ialu_reg_reg_vshift);
12654 %}
12655
12656 // This pattern is automatically generated from aarch64_ad.m4
12657 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12658 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12659 %{
12660 match(Set dst (RotateLeft src shift));
12661
12662 ins_cost(INSN_COST);
12663 format %{ "rol $dst, $src, $shift" %}
12664
12665 ins_encode %{
12666 __ subw(rscratch1, zr, as_Register($shift$$reg));
12667 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12668 %}
12669 ins_pipe(ialu_reg_reg_vshift);
12670 %}
12671
12672 // This pattern is automatically generated from aarch64_ad.m4
12673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12674 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12675 %{
12676 match(Set dst (RotateLeft src shift));
12677
12678 ins_cost(INSN_COST);
12679 format %{ "rol $dst, $src, $shift" %}
12680
12681 ins_encode %{
12682 __ subw(rscratch1, zr, as_Register($shift$$reg));
12683 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12684 %}
12685 ins_pipe(ialu_reg_reg_vshift);
12686 %}
12687
12688
12689 // Add/subtract (extended)
12690
12691 // This pattern is automatically generated from aarch64_ad.m4
12692 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12693 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12694 %{
12695 match(Set dst (AddL src1 (ConvI2L src2)));
12696 ins_cost(INSN_COST);
12697 format %{ "add $dst, $src1, $src2, sxtw" %}
12698
12699 ins_encode %{
12700 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12701 as_Register($src2$$reg), ext::sxtw);
12702 %}
12703 ins_pipe(ialu_reg_reg);
12704 %}
12705
12706 // This pattern is automatically generated from aarch64_ad.m4
12707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12708 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12709 %{
12710 match(Set dst (SubL src1 (ConvI2L src2)));
12711 ins_cost(INSN_COST);
12712 format %{ "sub $dst, $src1, $src2, sxtw" %}
12713
12714 ins_encode %{
12715 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12716 as_Register($src2$$reg), ext::sxtw);
12717 %}
12718 ins_pipe(ialu_reg_reg);
12719 %}
12720
12721 // This pattern is automatically generated from aarch64_ad.m4
12722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12723 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12724 %{
12725 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12726 ins_cost(INSN_COST);
12727 format %{ "add $dst, $src1, $src2, sxth" %}
12728
12729 ins_encode %{
12730 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12731 as_Register($src2$$reg), ext::sxth);
12732 %}
12733 ins_pipe(ialu_reg_reg);
12734 %}
12735
12736 // This pattern is automatically generated from aarch64_ad.m4
12737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12738 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12739 %{
12740 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12741 ins_cost(INSN_COST);
12742 format %{ "add $dst, $src1, $src2, sxtb" %}
12743
12744 ins_encode %{
12745 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12746 as_Register($src2$$reg), ext::sxtb);
12747 %}
12748 ins_pipe(ialu_reg_reg);
12749 %}
12750
12751 // This pattern is automatically generated from aarch64_ad.m4
12752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12753 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12754 %{
12755 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12756 ins_cost(INSN_COST);
12757 format %{ "add $dst, $src1, $src2, uxtb" %}
12758
12759 ins_encode %{
12760 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12761 as_Register($src2$$reg), ext::uxtb);
12762 %}
12763 ins_pipe(ialu_reg_reg);
12764 %}
12765
12766 // This pattern is automatically generated from aarch64_ad.m4
12767 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12768 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12769 %{
12770 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12771 ins_cost(INSN_COST);
12772 format %{ "add $dst, $src1, $src2, sxth" %}
12773
12774 ins_encode %{
12775 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12776 as_Register($src2$$reg), ext::sxth);
12777 %}
12778 ins_pipe(ialu_reg_reg);
12779 %}
12780
12781 // This pattern is automatically generated from aarch64_ad.m4
12782 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12783 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12784 %{
12785 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12786 ins_cost(INSN_COST);
12787 format %{ "add $dst, $src1, $src2, sxtw" %}
12788
12789 ins_encode %{
12790 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12791 as_Register($src2$$reg), ext::sxtw);
12792 %}
12793 ins_pipe(ialu_reg_reg);
12794 %}
12795
12796 // This pattern is automatically generated from aarch64_ad.m4
12797 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12798 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12799 %{
12800 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12801 ins_cost(INSN_COST);
12802 format %{ "add $dst, $src1, $src2, sxtb" %}
12803
12804 ins_encode %{
12805 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12806 as_Register($src2$$reg), ext::sxtb);
12807 %}
12808 ins_pipe(ialu_reg_reg);
12809 %}
12810
12811 // This pattern is automatically generated from aarch64_ad.m4
12812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12813 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12814 %{
12815 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12816 ins_cost(INSN_COST);
12817 format %{ "add $dst, $src1, $src2, uxtb" %}
12818
12819 ins_encode %{
12820 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12821 as_Register($src2$$reg), ext::uxtb);
12822 %}
12823 ins_pipe(ialu_reg_reg);
12824 %}
12825
12826 // This pattern is automatically generated from aarch64_ad.m4
12827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12828 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12829 %{
12830 match(Set dst (AddI src1 (AndI src2 mask)));
12831 ins_cost(INSN_COST);
12832 format %{ "addw $dst, $src1, $src2, uxtb" %}
12833
12834 ins_encode %{
12835 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12836 as_Register($src2$$reg), ext::uxtb);
12837 %}
12838 ins_pipe(ialu_reg_reg);
12839 %}
12840
12841 // This pattern is automatically generated from aarch64_ad.m4
12842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12843 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12844 %{
12845 match(Set dst (AddI src1 (AndI src2 mask)));
12846 ins_cost(INSN_COST);
12847 format %{ "addw $dst, $src1, $src2, uxth" %}
12848
12849 ins_encode %{
12850 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12851 as_Register($src2$$reg), ext::uxth);
12852 %}
12853 ins_pipe(ialu_reg_reg);
12854 %}
12855
12856 // This pattern is automatically generated from aarch64_ad.m4
12857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12858 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12859 %{
12860 match(Set dst (AddL src1 (AndL src2 mask)));
12861 ins_cost(INSN_COST);
12862 format %{ "add $dst, $src1, $src2, uxtb" %}
12863
12864 ins_encode %{
12865 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12866 as_Register($src2$$reg), ext::uxtb);
12867 %}
12868 ins_pipe(ialu_reg_reg);
12869 %}
12870
12871 // This pattern is automatically generated from aarch64_ad.m4
12872 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12873 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12874 %{
12875 match(Set dst (AddL src1 (AndL src2 mask)));
12876 ins_cost(INSN_COST);
12877 format %{ "add $dst, $src1, $src2, uxth" %}
12878
12879 ins_encode %{
12880 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12881 as_Register($src2$$reg), ext::uxth);
12882 %}
12883 ins_pipe(ialu_reg_reg);
12884 %}
12885
12886 // This pattern is automatically generated from aarch64_ad.m4
12887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12888 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12889 %{
12890 match(Set dst (AddL src1 (AndL src2 mask)));
12891 ins_cost(INSN_COST);
12892 format %{ "add $dst, $src1, $src2, uxtw" %}
12893
12894 ins_encode %{
12895 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12896 as_Register($src2$$reg), ext::uxtw);
12897 %}
12898 ins_pipe(ialu_reg_reg);
12899 %}
12900
12901 // This pattern is automatically generated from aarch64_ad.m4
12902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12903 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12904 %{
12905 match(Set dst (SubI src1 (AndI src2 mask)));
12906 ins_cost(INSN_COST);
12907 format %{ "subw $dst, $src1, $src2, uxtb" %}
12908
12909 ins_encode %{
12910 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12911 as_Register($src2$$reg), ext::uxtb);
12912 %}
12913 ins_pipe(ialu_reg_reg);
12914 %}
12915
12916 // This pattern is automatically generated from aarch64_ad.m4
12917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12918 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12919 %{
12920 match(Set dst (SubI src1 (AndI src2 mask)));
12921 ins_cost(INSN_COST);
12922 format %{ "subw $dst, $src1, $src2, uxth" %}
12923
12924 ins_encode %{
12925 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12926 as_Register($src2$$reg), ext::uxth);
12927 %}
12928 ins_pipe(ialu_reg_reg);
12929 %}
12930
12931 // This pattern is automatically generated from aarch64_ad.m4
12932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12933 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12934 %{
12935 match(Set dst (SubL src1 (AndL src2 mask)));
12936 ins_cost(INSN_COST);
12937 format %{ "sub $dst, $src1, $src2, uxtb" %}
12938
12939 ins_encode %{
12940 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12941 as_Register($src2$$reg), ext::uxtb);
12942 %}
12943 ins_pipe(ialu_reg_reg);
12944 %}
12945
12946 // This pattern is automatically generated from aarch64_ad.m4
12947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12948 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12949 %{
12950 match(Set dst (SubL src1 (AndL src2 mask)));
12951 ins_cost(INSN_COST);
12952 format %{ "sub $dst, $src1, $src2, uxth" %}
12953
12954 ins_encode %{
12955 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12956 as_Register($src2$$reg), ext::uxth);
12957 %}
12958 ins_pipe(ialu_reg_reg);
12959 %}
12960
12961 // This pattern is automatically generated from aarch64_ad.m4
12962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12963 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12964 %{
12965 match(Set dst (SubL src1 (AndL src2 mask)));
12966 ins_cost(INSN_COST);
12967 format %{ "sub $dst, $src1, $src2, uxtw" %}
12968
12969 ins_encode %{
12970 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12971 as_Register($src2$$reg), ext::uxtw);
12972 %}
12973 ins_pipe(ialu_reg_reg);
12974 %}
12975
12976
12977 // This pattern is automatically generated from aarch64_ad.m4
12978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12979 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12980 %{
12981 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12982 ins_cost(1.9 * INSN_COST);
12983 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12984
12985 ins_encode %{
12986 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12987 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12988 %}
12989 ins_pipe(ialu_reg_reg_shift);
12990 %}
12991
12992 // This pattern is automatically generated from aarch64_ad.m4
12993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12994 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12995 %{
12996 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12997 ins_cost(1.9 * INSN_COST);
12998 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12999
13000 ins_encode %{
13001 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13002 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13003 %}
13004 ins_pipe(ialu_reg_reg_shift);
13005 %}
13006
13007 // This pattern is automatically generated from aarch64_ad.m4
13008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13009 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13010 %{
13011 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13012 ins_cost(1.9 * INSN_COST);
13013 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13014
13015 ins_encode %{
13016 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13017 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13018 %}
13019 ins_pipe(ialu_reg_reg_shift);
13020 %}
13021
13022 // This pattern is automatically generated from aarch64_ad.m4
13023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13024 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13025 %{
13026 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13027 ins_cost(1.9 * INSN_COST);
13028 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13029
13030 ins_encode %{
13031 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13032 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13033 %}
13034 ins_pipe(ialu_reg_reg_shift);
13035 %}
13036
13037 // This pattern is automatically generated from aarch64_ad.m4
13038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13039 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13040 %{
13041 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13042 ins_cost(1.9 * INSN_COST);
13043 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13044
13045 ins_encode %{
13046 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13047 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13048 %}
13049 ins_pipe(ialu_reg_reg_shift);
13050 %}
13051
13052 // This pattern is automatically generated from aarch64_ad.m4
13053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13054 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13055 %{
13056 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13057 ins_cost(1.9 * INSN_COST);
13058 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13059
13060 ins_encode %{
13061 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13062 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13063 %}
13064 ins_pipe(ialu_reg_reg_shift);
13065 %}
13066
13067 // This pattern is automatically generated from aarch64_ad.m4
13068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13069 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13070 %{
13071 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13072 ins_cost(1.9 * INSN_COST);
13073 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13074
13075 ins_encode %{
13076 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13077 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13078 %}
13079 ins_pipe(ialu_reg_reg_shift);
13080 %}
13081
13082 // This pattern is automatically generated from aarch64_ad.m4
13083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13084 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13085 %{
13086 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13087 ins_cost(1.9 * INSN_COST);
13088 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13089
13090 ins_encode %{
13091 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13092 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13093 %}
13094 ins_pipe(ialu_reg_reg_shift);
13095 %}
13096
13097 // This pattern is automatically generated from aarch64_ad.m4
13098 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13099 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13100 %{
13101 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13102 ins_cost(1.9 * INSN_COST);
13103 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13104
13105 ins_encode %{
13106 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13107 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13108 %}
13109 ins_pipe(ialu_reg_reg_shift);
13110 %}
13111
13112 // This pattern is automatically generated from aarch64_ad.m4
13113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13114 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13115 %{
13116 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13117 ins_cost(1.9 * INSN_COST);
13118 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13119
13120 ins_encode %{
13121 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13122 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13123 %}
13124 ins_pipe(ialu_reg_reg_shift);
13125 %}
13126
13127 // This pattern is automatically generated from aarch64_ad.m4
13128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13129 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13130 %{
13131 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13132 ins_cost(1.9 * INSN_COST);
13133 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13134
13135 ins_encode %{
13136 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13137 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13138 %}
13139 ins_pipe(ialu_reg_reg_shift);
13140 %}
13141
13142 // This pattern is automatically generated from aarch64_ad.m4
13143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13144 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13145 %{
13146 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13147 ins_cost(1.9 * INSN_COST);
13148 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13149
13150 ins_encode %{
13151 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13152 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13153 %}
13154 ins_pipe(ialu_reg_reg_shift);
13155 %}
13156
13157 // This pattern is automatically generated from aarch64_ad.m4
13158 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13159 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13160 %{
13161 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13162 ins_cost(1.9 * INSN_COST);
13163 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13164
13165 ins_encode %{
13166 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13167 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13168 %}
13169 ins_pipe(ialu_reg_reg_shift);
13170 %}
13171
13172 // This pattern is automatically generated from aarch64_ad.m4
13173 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13174 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13175 %{
13176 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13177 ins_cost(1.9 * INSN_COST);
13178 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13179
13180 ins_encode %{
13181 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13182 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13183 %}
13184 ins_pipe(ialu_reg_reg_shift);
13185 %}
13186
13187 // This pattern is automatically generated from aarch64_ad.m4
13188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13189 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13190 %{
13191 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13192 ins_cost(1.9 * INSN_COST);
13193 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13194
13195 ins_encode %{
13196 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13197 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13198 %}
13199 ins_pipe(ialu_reg_reg_shift);
13200 %}
13201
13202 // This pattern is automatically generated from aarch64_ad.m4
13203 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13204 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13205 %{
13206 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13207 ins_cost(1.9 * INSN_COST);
13208 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13209
13210 ins_encode %{
13211 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13212 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13213 %}
13214 ins_pipe(ialu_reg_reg_shift);
13215 %}
13216
13217 // This pattern is automatically generated from aarch64_ad.m4
13218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13219 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13220 %{
13221 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13222 ins_cost(1.9 * INSN_COST);
13223 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13224
13225 ins_encode %{
13226 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13227 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13228 %}
13229 ins_pipe(ialu_reg_reg_shift);
13230 %}
13231
13232 // This pattern is automatically generated from aarch64_ad.m4
13233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13234 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13235 %{
13236 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13237 ins_cost(1.9 * INSN_COST);
13238 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13239
13240 ins_encode %{
13241 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13242 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13243 %}
13244 ins_pipe(ialu_reg_reg_shift);
13245 %}
13246
13247 // This pattern is automatically generated from aarch64_ad.m4
13248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13249 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13250 %{
13251 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13252 ins_cost(1.9 * INSN_COST);
13253 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13254
13255 ins_encode %{
13256 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13257 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13258 %}
13259 ins_pipe(ialu_reg_reg_shift);
13260 %}
13261
13262 // This pattern is automatically generated from aarch64_ad.m4
13263 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13264 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13265 %{
13266 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13267 ins_cost(1.9 * INSN_COST);
13268 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13269
13270 ins_encode %{
13271 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13272 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13273 %}
13274 ins_pipe(ialu_reg_reg_shift);
13275 %}
13276
13277 // This pattern is automatically generated from aarch64_ad.m4
13278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13279 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13280 %{
13281 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13282 ins_cost(1.9 * INSN_COST);
13283 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13284
13285 ins_encode %{
13286 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13287 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13288 %}
13289 ins_pipe(ialu_reg_reg_shift);
13290 %}
13291
13292 // This pattern is automatically generated from aarch64_ad.m4
13293 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13294 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13295 %{
13296 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13297 ins_cost(1.9 * INSN_COST);
13298 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13299
13300 ins_encode %{
13301 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13302 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13303 %}
13304 ins_pipe(ialu_reg_reg_shift);
13305 %}
13306
13307 // This pattern is automatically generated from aarch64_ad.m4
13308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13309 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13310 %{
13311 effect(DEF dst, USE src1, USE src2, USE cr);
13312 ins_cost(INSN_COST * 2);
13313 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13314
13315 ins_encode %{
13316 __ cselw($dst$$Register,
13317 $src1$$Register,
13318 $src2$$Register,
13319 Assembler::LT);
13320 %}
13321 ins_pipe(icond_reg_reg);
13322 %}
13323
13324 // This pattern is automatically generated from aarch64_ad.m4
13325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13326 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13327 %{
13328 effect(DEF dst, USE src1, USE src2, USE cr);
13329 ins_cost(INSN_COST * 2);
13330 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13331
13332 ins_encode %{
13333 __ cselw($dst$$Register,
13334 $src1$$Register,
13335 $src2$$Register,
13336 Assembler::GT);
13337 %}
13338 ins_pipe(icond_reg_reg);
13339 %}
13340
13341 // This pattern is automatically generated from aarch64_ad.m4
13342 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13343 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13344 %{
13345 effect(DEF dst, USE src1, USE cr);
13346 ins_cost(INSN_COST * 2);
13347 format %{ "cselw $dst, $src1, zr lt\t" %}
13348
13349 ins_encode %{
13350 __ cselw($dst$$Register,
13351 $src1$$Register,
13352 zr,
13353 Assembler::LT);
13354 %}
13355 ins_pipe(icond_reg);
13356 %}
13357
13358 // This pattern is automatically generated from aarch64_ad.m4
13359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13360 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13361 %{
13362 effect(DEF dst, USE src1, USE cr);
13363 ins_cost(INSN_COST * 2);
13364 format %{ "cselw $dst, $src1, zr gt\t" %}
13365
13366 ins_encode %{
13367 __ cselw($dst$$Register,
13368 $src1$$Register,
13369 zr,
13370 Assembler::GT);
13371 %}
13372 ins_pipe(icond_reg);
13373 %}
13374
13375 // This pattern is automatically generated from aarch64_ad.m4
13376 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13377 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13378 %{
13379 effect(DEF dst, USE src1, USE cr);
13380 ins_cost(INSN_COST * 2);
13381 format %{ "csincw $dst, $src1, zr le\t" %}
13382
13383 ins_encode %{
13384 __ csincw($dst$$Register,
13385 $src1$$Register,
13386 zr,
13387 Assembler::LE);
13388 %}
13389 ins_pipe(icond_reg);
13390 %}
13391
13392 // This pattern is automatically generated from aarch64_ad.m4
13393 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13394 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13395 %{
13396 effect(DEF dst, USE src1, USE cr);
13397 ins_cost(INSN_COST * 2);
13398 format %{ "csincw $dst, $src1, zr gt\t" %}
13399
13400 ins_encode %{
13401 __ csincw($dst$$Register,
13402 $src1$$Register,
13403 zr,
13404 Assembler::GT);
13405 %}
13406 ins_pipe(icond_reg);
13407 %}
13408
13409 // This pattern is automatically generated from aarch64_ad.m4
13410 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13411 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13412 %{
13413 effect(DEF dst, USE src1, USE cr);
13414 ins_cost(INSN_COST * 2);
13415 format %{ "csinvw $dst, $src1, zr lt\t" %}
13416
13417 ins_encode %{
13418 __ csinvw($dst$$Register,
13419 $src1$$Register,
13420 zr,
13421 Assembler::LT);
13422 %}
13423 ins_pipe(icond_reg);
13424 %}
13425
13426 // This pattern is automatically generated from aarch64_ad.m4
13427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13428 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13429 %{
13430 effect(DEF dst, USE src1, USE cr);
13431 ins_cost(INSN_COST * 2);
13432 format %{ "csinvw $dst, $src1, zr ge\t" %}
13433
13434 ins_encode %{
13435 __ csinvw($dst$$Register,
13436 $src1$$Register,
13437 zr,
13438 Assembler::GE);
13439 %}
13440 ins_pipe(icond_reg);
13441 %}
13442
13443 // This pattern is automatically generated from aarch64_ad.m4
13444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13445 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13446 %{
13447 match(Set dst (MinI src imm));
13448 ins_cost(INSN_COST * 3);
13449 expand %{
13450 rFlagsReg cr;
13451 compI_reg_imm0(cr, src);
13452 cmovI_reg_imm0_lt(dst, src, cr);
13453 %}
13454 %}
13455
13456 // This pattern is automatically generated from aarch64_ad.m4
13457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13458 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13459 %{
13460 match(Set dst (MinI imm src));
13461 ins_cost(INSN_COST * 3);
13462 expand %{
13463 rFlagsReg cr;
13464 compI_reg_imm0(cr, src);
13465 cmovI_reg_imm0_lt(dst, src, cr);
13466 %}
13467 %}
13468
13469 // This pattern is automatically generated from aarch64_ad.m4
13470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13471 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13472 %{
13473 match(Set dst (MinI src imm));
13474 ins_cost(INSN_COST * 3);
13475 expand %{
13476 rFlagsReg cr;
13477 compI_reg_imm0(cr, src);
13478 cmovI_reg_imm1_le(dst, src, cr);
13479 %}
13480 %}
13481
13482 // This pattern is automatically generated from aarch64_ad.m4
13483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13484 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13485 %{
13486 match(Set dst (MinI imm src));
13487 ins_cost(INSN_COST * 3);
13488 expand %{
13489 rFlagsReg cr;
13490 compI_reg_imm0(cr, src);
13491 cmovI_reg_imm1_le(dst, src, cr);
13492 %}
13493 %}
13494
13495 // This pattern is automatically generated from aarch64_ad.m4
13496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13497 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13498 %{
13499 match(Set dst (MinI src imm));
13500 ins_cost(INSN_COST * 3);
13501 expand %{
13502 rFlagsReg cr;
13503 compI_reg_imm0(cr, src);
13504 cmovI_reg_immM1_lt(dst, src, cr);
13505 %}
13506 %}
13507
13508 // This pattern is automatically generated from aarch64_ad.m4
13509 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13510 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13511 %{
13512 match(Set dst (MinI imm src));
13513 ins_cost(INSN_COST * 3);
13514 expand %{
13515 rFlagsReg cr;
13516 compI_reg_imm0(cr, src);
13517 cmovI_reg_immM1_lt(dst, src, cr);
13518 %}
13519 %}
13520
13521 // This pattern is automatically generated from aarch64_ad.m4
13522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13523 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13524 %{
13525 match(Set dst (MaxI src imm));
13526 ins_cost(INSN_COST * 3);
13527 expand %{
13528 rFlagsReg cr;
13529 compI_reg_imm0(cr, src);
13530 cmovI_reg_imm0_gt(dst, src, cr);
13531 %}
13532 %}
13533
13534 // This pattern is automatically generated from aarch64_ad.m4
13535 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13536 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13537 %{
13538 match(Set dst (MaxI imm src));
13539 ins_cost(INSN_COST * 3);
13540 expand %{
13541 rFlagsReg cr;
13542 compI_reg_imm0(cr, src);
13543 cmovI_reg_imm0_gt(dst, src, cr);
13544 %}
13545 %}
13546
13547 // This pattern is automatically generated from aarch64_ad.m4
13548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13549 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13550 %{
13551 match(Set dst (MaxI src imm));
13552 ins_cost(INSN_COST * 3);
13553 expand %{
13554 rFlagsReg cr;
13555 compI_reg_imm0(cr, src);
13556 cmovI_reg_imm1_gt(dst, src, cr);
13557 %}
13558 %}
13559
13560 // This pattern is automatically generated from aarch64_ad.m4
13561 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13562 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13563 %{
13564 match(Set dst (MaxI imm src));
13565 ins_cost(INSN_COST * 3);
13566 expand %{
13567 rFlagsReg cr;
13568 compI_reg_imm0(cr, src);
13569 cmovI_reg_imm1_gt(dst, src, cr);
13570 %}
13571 %}
13572
13573 // This pattern is automatically generated from aarch64_ad.m4
13574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13575 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13576 %{
13577 match(Set dst (MaxI src imm));
13578 ins_cost(INSN_COST * 3);
13579 expand %{
13580 rFlagsReg cr;
13581 compI_reg_imm0(cr, src);
13582 cmovI_reg_immM1_ge(dst, src, cr);
13583 %}
13584 %}
13585
13586 // This pattern is automatically generated from aarch64_ad.m4
13587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13588 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13589 %{
13590 match(Set dst (MaxI imm src));
13591 ins_cost(INSN_COST * 3);
13592 expand %{
13593 rFlagsReg cr;
13594 compI_reg_imm0(cr, src);
13595 cmovI_reg_immM1_ge(dst, src, cr);
13596 %}
13597 %}
13598
13599 // This pattern is automatically generated from aarch64_ad.m4
13600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13601 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13602 %{
13603 match(Set dst (ReverseI src));
13604 ins_cost(INSN_COST);
13605 format %{ "rbitw $dst, $src" %}
13606 ins_encode %{
13607 __ rbitw($dst$$Register, $src$$Register);
13608 %}
13609 ins_pipe(ialu_reg);
13610 %}
13611
13612 // This pattern is automatically generated from aarch64_ad.m4
13613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13614 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13615 %{
13616 match(Set dst (ReverseL src));
13617 ins_cost(INSN_COST);
13618 format %{ "rbit $dst, $src" %}
13619 ins_encode %{
13620 __ rbit($dst$$Register, $src$$Register);
13621 %}
13622 ins_pipe(ialu_reg);
13623 %}
13624
13625
13626 // END This section of the file is automatically generated. Do not edit --------------
13627
13628
13629 // ============================================================================
13630 // Floating Point Arithmetic Instructions
13631
13632 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13633 match(Set dst (AddF src1 src2));
13634
13635 ins_cost(INSN_COST * 5);
13636 format %{ "fadds $dst, $src1, $src2" %}
13637
13638 ins_encode %{
13639 __ fadds(as_FloatRegister($dst$$reg),
13640 as_FloatRegister($src1$$reg),
13641 as_FloatRegister($src2$$reg));
13642 %}
13643
13644 ins_pipe(fp_dop_reg_reg_s);
13645 %}
13646
13647 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13648 match(Set dst (AddD src1 src2));
13649
13650 ins_cost(INSN_COST * 5);
13651 format %{ "faddd $dst, $src1, $src2" %}
13652
13653 ins_encode %{
13654 __ faddd(as_FloatRegister($dst$$reg),
13655 as_FloatRegister($src1$$reg),
13656 as_FloatRegister($src2$$reg));
13657 %}
13658
13659 ins_pipe(fp_dop_reg_reg_d);
13660 %}
13661
13662 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13663 match(Set dst (SubF src1 src2));
13664
13665 ins_cost(INSN_COST * 5);
13666 format %{ "fsubs $dst, $src1, $src2" %}
13667
13668 ins_encode %{
13669 __ fsubs(as_FloatRegister($dst$$reg),
13670 as_FloatRegister($src1$$reg),
13671 as_FloatRegister($src2$$reg));
13672 %}
13673
13674 ins_pipe(fp_dop_reg_reg_s);
13675 %}
13676
13677 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13678 match(Set dst (SubD src1 src2));
13679
13680 ins_cost(INSN_COST * 5);
13681 format %{ "fsubd $dst, $src1, $src2" %}
13682
13683 ins_encode %{
13684 __ fsubd(as_FloatRegister($dst$$reg),
13685 as_FloatRegister($src1$$reg),
13686 as_FloatRegister($src2$$reg));
13687 %}
13688
13689 ins_pipe(fp_dop_reg_reg_d);
13690 %}
13691
13692 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13693 match(Set dst (MulF src1 src2));
13694
13695 ins_cost(INSN_COST * 6);
13696 format %{ "fmuls $dst, $src1, $src2" %}
13697
13698 ins_encode %{
13699 __ fmuls(as_FloatRegister($dst$$reg),
13700 as_FloatRegister($src1$$reg),
13701 as_FloatRegister($src2$$reg));
13702 %}
13703
13704 ins_pipe(fp_dop_reg_reg_s);
13705 %}
13706
13707 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13708 match(Set dst (MulD src1 src2));
13709
13710 ins_cost(INSN_COST * 6);
13711 format %{ "fmuld $dst, $src1, $src2" %}
13712
13713 ins_encode %{
13714 __ fmuld(as_FloatRegister($dst$$reg),
13715 as_FloatRegister($src1$$reg),
13716 as_FloatRegister($src2$$reg));
13717 %}
13718
13719 ins_pipe(fp_dop_reg_reg_d);
13720 %}
13721
13722 // src1 * src2 + src3
13723 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13724 match(Set dst (FmaF src3 (Binary src1 src2)));
13725
13726 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13727
13728 ins_encode %{
13729 assert(UseFMA, "Needs FMA instructions support.");
13730 __ fmadds(as_FloatRegister($dst$$reg),
13731 as_FloatRegister($src1$$reg),
13732 as_FloatRegister($src2$$reg),
13733 as_FloatRegister($src3$$reg));
13734 %}
13735
13736 ins_pipe(pipe_class_default);
13737 %}
13738
13739 // src1 * src2 + src3
13740 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13741 match(Set dst (FmaD src3 (Binary src1 src2)));
13742
13743 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13744
13745 ins_encode %{
13746 assert(UseFMA, "Needs FMA instructions support.");
13747 __ fmaddd(as_FloatRegister($dst$$reg),
13748 as_FloatRegister($src1$$reg),
13749 as_FloatRegister($src2$$reg),
13750 as_FloatRegister($src3$$reg));
13751 %}
13752
13753 ins_pipe(pipe_class_default);
13754 %}
13755
13756 // src1 * (-src2) + src3
13757 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13758 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13759 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13760
13761 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13762
13763 ins_encode %{
13764 assert(UseFMA, "Needs FMA instructions support.");
13765 __ fmsubs(as_FloatRegister($dst$$reg),
13766 as_FloatRegister($src1$$reg),
13767 as_FloatRegister($src2$$reg),
13768 as_FloatRegister($src3$$reg));
13769 %}
13770
13771 ins_pipe(pipe_class_default);
13772 %}
13773
13774 // src1 * (-src2) + src3
13775 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13776 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13777 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13778
13779 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13780
13781 ins_encode %{
13782 assert(UseFMA, "Needs FMA instructions support.");
13783 __ fmsubd(as_FloatRegister($dst$$reg),
13784 as_FloatRegister($src1$$reg),
13785 as_FloatRegister($src2$$reg),
13786 as_FloatRegister($src3$$reg));
13787 %}
13788
13789 ins_pipe(pipe_class_default);
13790 %}
13791
13792 // src1 * (-src2) - src3
13793 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13794 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13795 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13796
13797 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13798
13799 ins_encode %{
13800 assert(UseFMA, "Needs FMA instructions support.");
13801 __ fnmadds(as_FloatRegister($dst$$reg),
13802 as_FloatRegister($src1$$reg),
13803 as_FloatRegister($src2$$reg),
13804 as_FloatRegister($src3$$reg));
13805 %}
13806
13807 ins_pipe(pipe_class_default);
13808 %}
13809
13810 // src1 * (-src2) - src3
13811 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13812 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13813 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13814
13815 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13816
13817 ins_encode %{
13818 assert(UseFMA, "Needs FMA instructions support.");
13819 __ fnmaddd(as_FloatRegister($dst$$reg),
13820 as_FloatRegister($src1$$reg),
13821 as_FloatRegister($src2$$reg),
13822 as_FloatRegister($src3$$reg));
13823 %}
13824
13825 ins_pipe(pipe_class_default);
13826 %}
13827
13828 // src1 * src2 - src3
13829 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13830 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13831
13832 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13833
13834 ins_encode %{
13835 assert(UseFMA, "Needs FMA instructions support.");
13836 __ fnmsubs(as_FloatRegister($dst$$reg),
13837 as_FloatRegister($src1$$reg),
13838 as_FloatRegister($src2$$reg),
13839 as_FloatRegister($src3$$reg));
13840 %}
13841
13842 ins_pipe(pipe_class_default);
13843 %}
13844
13845 // src1 * src2 - src3
13846 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13847 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13848
13849 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13850
13851 ins_encode %{
13852 assert(UseFMA, "Needs FMA instructions support.");
13853 // n.b. insn name should be fnmsubd
13854 __ fnmsub(as_FloatRegister($dst$$reg),
13855 as_FloatRegister($src1$$reg),
13856 as_FloatRegister($src2$$reg),
13857 as_FloatRegister($src3$$reg));
13858 %}
13859
13860 ins_pipe(pipe_class_default);
13861 %}
13862
13863
13864 // Math.max(FF)F
13865 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13866 match(Set dst (MaxF src1 src2));
13867
13868 format %{ "fmaxs $dst, $src1, $src2" %}
13869 ins_encode %{
13870 __ fmaxs(as_FloatRegister($dst$$reg),
13871 as_FloatRegister($src1$$reg),
13872 as_FloatRegister($src2$$reg));
13873 %}
13874
13875 ins_pipe(fp_dop_reg_reg_s);
13876 %}
13877
13878 // Math.min(FF)F
13879 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13880 match(Set dst (MinF src1 src2));
13881
13882 format %{ "fmins $dst, $src1, $src2" %}
13883 ins_encode %{
13884 __ fmins(as_FloatRegister($dst$$reg),
13885 as_FloatRegister($src1$$reg),
13886 as_FloatRegister($src2$$reg));
13887 %}
13888
13889 ins_pipe(fp_dop_reg_reg_s);
13890 %}
13891
13892 // Math.max(DD)D
13893 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13894 match(Set dst (MaxD src1 src2));
13895
13896 format %{ "fmaxd $dst, $src1, $src2" %}
13897 ins_encode %{
13898 __ fmaxd(as_FloatRegister($dst$$reg),
13899 as_FloatRegister($src1$$reg),
13900 as_FloatRegister($src2$$reg));
13901 %}
13902
13903 ins_pipe(fp_dop_reg_reg_d);
13904 %}
13905
13906 // Math.min(DD)D
13907 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13908 match(Set dst (MinD src1 src2));
13909
13910 format %{ "fmind $dst, $src1, $src2" %}
13911 ins_encode %{
13912 __ fmind(as_FloatRegister($dst$$reg),
13913 as_FloatRegister($src1$$reg),
13914 as_FloatRegister($src2$$reg));
13915 %}
13916
13917 ins_pipe(fp_dop_reg_reg_d);
13918 %}
13919
13920
13921 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13922 match(Set dst (DivF src1 src2));
13923
13924 ins_cost(INSN_COST * 18);
13925 format %{ "fdivs $dst, $src1, $src2" %}
13926
13927 ins_encode %{
13928 __ fdivs(as_FloatRegister($dst$$reg),
13929 as_FloatRegister($src1$$reg),
13930 as_FloatRegister($src2$$reg));
13931 %}
13932
13933 ins_pipe(fp_div_s);
13934 %}
13935
13936 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13937 match(Set dst (DivD src1 src2));
13938
13939 ins_cost(INSN_COST * 32);
13940 format %{ "fdivd $dst, $src1, $src2" %}
13941
13942 ins_encode %{
13943 __ fdivd(as_FloatRegister($dst$$reg),
13944 as_FloatRegister($src1$$reg),
13945 as_FloatRegister($src2$$reg));
13946 %}
13947
13948 ins_pipe(fp_div_d);
13949 %}
13950
13951 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13952 match(Set dst (NegF src));
13953
13954 ins_cost(INSN_COST * 3);
13955 format %{ "fneg $dst, $src" %}
13956
13957 ins_encode %{
13958 __ fnegs(as_FloatRegister($dst$$reg),
13959 as_FloatRegister($src$$reg));
13960 %}
13961
13962 ins_pipe(fp_uop_s);
13963 %}
13964
13965 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13966 match(Set dst (NegD src));
13967
13968 ins_cost(INSN_COST * 3);
13969 format %{ "fnegd $dst, $src" %}
13970
13971 ins_encode %{
13972 __ fnegd(as_FloatRegister($dst$$reg),
13973 as_FloatRegister($src$$reg));
13974 %}
13975
13976 ins_pipe(fp_uop_d);
13977 %}
13978
13979 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13980 %{
13981 match(Set dst (AbsI src));
13982
13983 effect(KILL cr);
13984 ins_cost(INSN_COST * 2);
13985 format %{ "cmpw $src, zr\n\t"
13986 "cnegw $dst, $src, Assembler::LT\t# int abs"
13987 %}
13988
13989 ins_encode %{
13990 __ cmpw(as_Register($src$$reg), zr);
13991 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13992 %}
13993 ins_pipe(pipe_class_default);
13994 %}
13995
13996 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13997 %{
13998 match(Set dst (AbsL src));
13999
14000 effect(KILL cr);
14001 ins_cost(INSN_COST * 2);
14002 format %{ "cmp $src, zr\n\t"
14003 "cneg $dst, $src, Assembler::LT\t# long abs"
14004 %}
14005
14006 ins_encode %{
14007 __ cmp(as_Register($src$$reg), zr);
14008 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14009 %}
14010 ins_pipe(pipe_class_default);
14011 %}
14012
14013 instruct absF_reg(vRegF dst, vRegF src) %{
14014 match(Set dst (AbsF src));
14015
14016 ins_cost(INSN_COST * 3);
14017 format %{ "fabss $dst, $src" %}
14018 ins_encode %{
14019 __ fabss(as_FloatRegister($dst$$reg),
14020 as_FloatRegister($src$$reg));
14021 %}
14022
14023 ins_pipe(fp_uop_s);
14024 %}
14025
14026 instruct absD_reg(vRegD dst, vRegD src) %{
14027 match(Set dst (AbsD src));
14028
14029 ins_cost(INSN_COST * 3);
14030 format %{ "fabsd $dst, $src" %}
14031 ins_encode %{
14032 __ fabsd(as_FloatRegister($dst$$reg),
14033 as_FloatRegister($src$$reg));
14034 %}
14035
14036 ins_pipe(fp_uop_d);
14037 %}
14038
14039 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14040 match(Set dst (AbsF (SubF src1 src2)));
14041
14042 ins_cost(INSN_COST * 3);
14043 format %{ "fabds $dst, $src1, $src2" %}
14044 ins_encode %{
14045 __ fabds(as_FloatRegister($dst$$reg),
14046 as_FloatRegister($src1$$reg),
14047 as_FloatRegister($src2$$reg));
14048 %}
14049
14050 ins_pipe(fp_uop_s);
14051 %}
14052
14053 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14054 match(Set dst (AbsD (SubD src1 src2)));
14055
14056 ins_cost(INSN_COST * 3);
14057 format %{ "fabdd $dst, $src1, $src2" %}
14058 ins_encode %{
14059 __ fabdd(as_FloatRegister($dst$$reg),
14060 as_FloatRegister($src1$$reg),
14061 as_FloatRegister($src2$$reg));
14062 %}
14063
14064 ins_pipe(fp_uop_d);
14065 %}
14066
14067 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14068 match(Set dst (SqrtD src));
14069
14070 ins_cost(INSN_COST * 50);
14071 format %{ "fsqrtd $dst, $src" %}
14072 ins_encode %{
14073 __ fsqrtd(as_FloatRegister($dst$$reg),
14074 as_FloatRegister($src$$reg));
14075 %}
14076
14077 ins_pipe(fp_div_s);
14078 %}
14079
14080 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14081 match(Set dst (SqrtF src));
14082
14083 ins_cost(INSN_COST * 50);
14084 format %{ "fsqrts $dst, $src" %}
14085 ins_encode %{
14086 __ fsqrts(as_FloatRegister($dst$$reg),
14087 as_FloatRegister($src$$reg));
14088 %}
14089
14090 ins_pipe(fp_div_d);
14091 %}
14092
14093 // Math.rint, floor, ceil
14094 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14095 match(Set dst (RoundDoubleMode src rmode));
14096 format %{ "frint $dst, $src, $rmode" %}
14097 ins_encode %{
14098 switch ($rmode$$constant) {
14099 case RoundDoubleModeNode::rmode_rint:
14100 __ frintnd(as_FloatRegister($dst$$reg),
14101 as_FloatRegister($src$$reg));
14102 break;
14103 case RoundDoubleModeNode::rmode_floor:
14104 __ frintmd(as_FloatRegister($dst$$reg),
14105 as_FloatRegister($src$$reg));
14106 break;
14107 case RoundDoubleModeNode::rmode_ceil:
14108 __ frintpd(as_FloatRegister($dst$$reg),
14109 as_FloatRegister($src$$reg));
14110 break;
14111 }
14112 %}
14113 ins_pipe(fp_uop_d);
14114 %}
14115
14116 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14117 match(Set dst (CopySignD src1 (Binary src2 zero)));
14118 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14119 format %{ "CopySignD $dst $src1 $src2" %}
14120 ins_encode %{
14121 FloatRegister dst = as_FloatRegister($dst$$reg),
14122 src1 = as_FloatRegister($src1$$reg),
14123 src2 = as_FloatRegister($src2$$reg),
14124 zero = as_FloatRegister($zero$$reg);
14125 __ fnegd(dst, zero);
14126 __ bsl(dst, __ T8B, src2, src1);
14127 %}
14128 ins_pipe(fp_uop_d);
14129 %}
14130
14131 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14132 match(Set dst (CopySignF src1 src2));
14133 effect(TEMP_DEF dst, USE src1, USE src2);
14134 format %{ "CopySignF $dst $src1 $src2" %}
14135 ins_encode %{
14136 FloatRegister dst = as_FloatRegister($dst$$reg),
14137 src1 = as_FloatRegister($src1$$reg),
14138 src2 = as_FloatRegister($src2$$reg);
14139 __ movi(dst, __ T2S, 0x80, 24);
14140 __ bsl(dst, __ T8B, src2, src1);
14141 %}
14142 ins_pipe(fp_uop_d);
14143 %}
14144
14145 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14146 match(Set dst (SignumD src (Binary zero one)));
14147 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14148 format %{ "signumD $dst, $src" %}
14149 ins_encode %{
14150 FloatRegister src = as_FloatRegister($src$$reg),
14151 dst = as_FloatRegister($dst$$reg),
14152 zero = as_FloatRegister($zero$$reg),
14153 one = as_FloatRegister($one$$reg);
14154 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14155 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14156 // Bit selection instruction gets bit from "one" for each enabled bit in
14157 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14158 // NaN the whole "src" will be copied because "dst" is zero. For all other
14159 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14160 // from "src", and all other bits are copied from 1.0.
14161 __ bsl(dst, __ T8B, one, src);
14162 %}
14163 ins_pipe(fp_uop_d);
14164 %}
14165
14166 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14167 match(Set dst (SignumF src (Binary zero one)));
14168 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14169 format %{ "signumF $dst, $src" %}
14170 ins_encode %{
14171 FloatRegister src = as_FloatRegister($src$$reg),
14172 dst = as_FloatRegister($dst$$reg),
14173 zero = as_FloatRegister($zero$$reg),
14174 one = as_FloatRegister($one$$reg);
14175 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14176 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14177 // Bit selection instruction gets bit from "one" for each enabled bit in
14178 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14179 // NaN the whole "src" will be copied because "dst" is zero. For all other
14180 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14181 // from "src", and all other bits are copied from 1.0.
14182 __ bsl(dst, __ T8B, one, src);
14183 %}
14184 ins_pipe(fp_uop_d);
14185 %}
14186
14187 instruct onspinwait() %{
14188 match(OnSpinWait);
14189 ins_cost(INSN_COST);
14190
14191 format %{ "onspinwait" %}
14192
14193 ins_encode %{
14194 __ spin_wait();
14195 %}
14196 ins_pipe(pipe_class_empty);
14197 %}
14198
14199 // ============================================================================
14200 // Logical Instructions
14201
14202 // Integer Logical Instructions
14203
14204 // And Instructions
14205
14206
14207 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14208 match(Set dst (AndI src1 src2));
14209
14210 format %{ "andw $dst, $src1, $src2\t# int" %}
14211
14212 ins_cost(INSN_COST);
14213 ins_encode %{
14214 __ andw(as_Register($dst$$reg),
14215 as_Register($src1$$reg),
14216 as_Register($src2$$reg));
14217 %}
14218
14219 ins_pipe(ialu_reg_reg);
14220 %}
14221
14222 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14223 match(Set dst (AndI src1 src2));
14224
14225 format %{ "andsw $dst, $src1, $src2\t# int" %}
14226
14227 ins_cost(INSN_COST);
14228 ins_encode %{
14229 __ andw(as_Register($dst$$reg),
14230 as_Register($src1$$reg),
14231 (uint64_t)($src2$$constant));
14232 %}
14233
14234 ins_pipe(ialu_reg_imm);
14235 %}
14236
14237 // Or Instructions
14238
14239 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14240 match(Set dst (OrI src1 src2));
14241
14242 format %{ "orrw $dst, $src1, $src2\t# int" %}
14243
14244 ins_cost(INSN_COST);
14245 ins_encode %{
14246 __ orrw(as_Register($dst$$reg),
14247 as_Register($src1$$reg),
14248 as_Register($src2$$reg));
14249 %}
14250
14251 ins_pipe(ialu_reg_reg);
14252 %}
14253
14254 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14255 match(Set dst (OrI src1 src2));
14256
14257 format %{ "orrw $dst, $src1, $src2\t# int" %}
14258
14259 ins_cost(INSN_COST);
14260 ins_encode %{
14261 __ orrw(as_Register($dst$$reg),
14262 as_Register($src1$$reg),
14263 (uint64_t)($src2$$constant));
14264 %}
14265
14266 ins_pipe(ialu_reg_imm);
14267 %}
14268
14269 // Xor Instructions
14270
14271 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14272 match(Set dst (XorI src1 src2));
14273
14274 format %{ "eorw $dst, $src1, $src2\t# int" %}
14275
14276 ins_cost(INSN_COST);
14277 ins_encode %{
14278 __ eorw(as_Register($dst$$reg),
14279 as_Register($src1$$reg),
14280 as_Register($src2$$reg));
14281 %}
14282
14283 ins_pipe(ialu_reg_reg);
14284 %}
14285
14286 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14287 match(Set dst (XorI src1 src2));
14288
14289 format %{ "eorw $dst, $src1, $src2\t# int" %}
14290
14291 ins_cost(INSN_COST);
14292 ins_encode %{
14293 __ eorw(as_Register($dst$$reg),
14294 as_Register($src1$$reg),
14295 (uint64_t)($src2$$constant));
14296 %}
14297
14298 ins_pipe(ialu_reg_imm);
14299 %}
14300
14301 // Long Logical Instructions
14302 // TODO
14303
14304 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14305 match(Set dst (AndL src1 src2));
14306
14307 format %{ "and $dst, $src1, $src2\t# int" %}
14308
14309 ins_cost(INSN_COST);
14310 ins_encode %{
14311 __ andr(as_Register($dst$$reg),
14312 as_Register($src1$$reg),
14313 as_Register($src2$$reg));
14314 %}
14315
14316 ins_pipe(ialu_reg_reg);
14317 %}
14318
14319 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14320 match(Set dst (AndL src1 src2));
14321
14322 format %{ "and $dst, $src1, $src2\t# int" %}
14323
14324 ins_cost(INSN_COST);
14325 ins_encode %{
14326 __ andr(as_Register($dst$$reg),
14327 as_Register($src1$$reg),
14328 (uint64_t)($src2$$constant));
14329 %}
14330
14331 ins_pipe(ialu_reg_imm);
14332 %}
14333
14334 // Or Instructions
14335
14336 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14337 match(Set dst (OrL src1 src2));
14338
14339 format %{ "orr $dst, $src1, $src2\t# int" %}
14340
14341 ins_cost(INSN_COST);
14342 ins_encode %{
14343 __ orr(as_Register($dst$$reg),
14344 as_Register($src1$$reg),
14345 as_Register($src2$$reg));
14346 %}
14347
14348 ins_pipe(ialu_reg_reg);
14349 %}
14350
14351 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14352 match(Set dst (OrL src1 src2));
14353
14354 format %{ "orr $dst, $src1, $src2\t# int" %}
14355
14356 ins_cost(INSN_COST);
14357 ins_encode %{
14358 __ orr(as_Register($dst$$reg),
14359 as_Register($src1$$reg),
14360 (uint64_t)($src2$$constant));
14361 %}
14362
14363 ins_pipe(ialu_reg_imm);
14364 %}
14365
14366 // Xor Instructions
14367
14368 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14369 match(Set dst (XorL src1 src2));
14370
14371 format %{ "eor $dst, $src1, $src2\t# int" %}
14372
14373 ins_cost(INSN_COST);
14374 ins_encode %{
14375 __ eor(as_Register($dst$$reg),
14376 as_Register($src1$$reg),
14377 as_Register($src2$$reg));
14378 %}
14379
14380 ins_pipe(ialu_reg_reg);
14381 %}
14382
14383 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14384 match(Set dst (XorL src1 src2));
14385
14386 ins_cost(INSN_COST);
14387 format %{ "eor $dst, $src1, $src2\t# int" %}
14388
14389 ins_encode %{
14390 __ eor(as_Register($dst$$reg),
14391 as_Register($src1$$reg),
14392 (uint64_t)($src2$$constant));
14393 %}
14394
14395 ins_pipe(ialu_reg_imm);
14396 %}
14397
14398 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14399 %{
14400 match(Set dst (ConvI2L src));
14401
14402 ins_cost(INSN_COST);
14403 format %{ "sxtw $dst, $src\t# i2l" %}
14404 ins_encode %{
14405 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14406 %}
14407 ins_pipe(ialu_reg_shift);
14408 %}
14409
14410 // this pattern occurs in bigmath arithmetic
14411 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14412 %{
14413 match(Set dst (AndL (ConvI2L src) mask));
14414
14415 ins_cost(INSN_COST);
14416 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14417 ins_encode %{
14418 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14419 %}
14420
14421 ins_pipe(ialu_reg_shift);
14422 %}
14423
14424 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14425 match(Set dst (ConvL2I src));
14426
14427 ins_cost(INSN_COST);
14428 format %{ "movw $dst, $src \t// l2i" %}
14429
14430 ins_encode %{
14431 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14432 %}
14433
14434 ins_pipe(ialu_reg);
14435 %}
14436
14437 instruct convD2F_reg(vRegF dst, vRegD src) %{
14438 match(Set dst (ConvD2F src));
14439
14440 ins_cost(INSN_COST * 5);
14441 format %{ "fcvtd $dst, $src \t// d2f" %}
14442
14443 ins_encode %{
14444 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14445 %}
14446
14447 ins_pipe(fp_d2f);
14448 %}
14449
14450 instruct convF2D_reg(vRegD dst, vRegF src) %{
14451 match(Set dst (ConvF2D src));
14452
14453 ins_cost(INSN_COST * 5);
14454 format %{ "fcvts $dst, $src \t// f2d" %}
14455
14456 ins_encode %{
14457 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14458 %}
14459
14460 ins_pipe(fp_f2d);
14461 %}
14462
14463 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14464 match(Set dst (ConvF2I src));
14465
14466 ins_cost(INSN_COST * 5);
14467 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14468
14469 ins_encode %{
14470 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14471 %}
14472
14473 ins_pipe(fp_f2i);
14474 %}
14475
14476 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14477 match(Set dst (ConvF2L src));
14478
14479 ins_cost(INSN_COST * 5);
14480 format %{ "fcvtzs $dst, $src \t// f2l" %}
14481
14482 ins_encode %{
14483 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14484 %}
14485
14486 ins_pipe(fp_f2l);
14487 %}
14488
14489 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14490 match(Set dst (ConvF2HF src));
14491 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14492 "smov $dst, $tmp\t# move result from $tmp to $dst"
14493 %}
14494 effect(TEMP tmp);
14495 ins_encode %{
14496 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14497 %}
14498 ins_pipe(pipe_slow);
14499 %}
14500
14501 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14502 match(Set dst (ConvHF2F src));
14503 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14504 "fcvt $dst, $tmp\t# convert half to single precision"
14505 %}
14506 effect(TEMP tmp);
14507 ins_encode %{
14508 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14509 %}
14510 ins_pipe(pipe_slow);
14511 %}
14512
14513 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14514 match(Set dst (ConvI2F src));
14515
14516 ins_cost(INSN_COST * 5);
14517 format %{ "scvtfws $dst, $src \t// i2f" %}
14518
14519 ins_encode %{
14520 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14521 %}
14522
14523 ins_pipe(fp_i2f);
14524 %}
14525
14526 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14527 match(Set dst (ConvL2F src));
14528
14529 ins_cost(INSN_COST * 5);
14530 format %{ "scvtfs $dst, $src \t// l2f" %}
14531
14532 ins_encode %{
14533 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14534 %}
14535
14536 ins_pipe(fp_l2f);
14537 %}
14538
14539 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14540 match(Set dst (ConvD2I src));
14541
14542 ins_cost(INSN_COST * 5);
14543 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14544
14545 ins_encode %{
14546 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14547 %}
14548
14549 ins_pipe(fp_d2i);
14550 %}
14551
14552 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14553 match(Set dst (ConvD2L src));
14554
14555 ins_cost(INSN_COST * 5);
14556 format %{ "fcvtzd $dst, $src \t// d2l" %}
14557
14558 ins_encode %{
14559 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14560 %}
14561
14562 ins_pipe(fp_d2l);
14563 %}
14564
14565 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14566 match(Set dst (ConvI2D src));
14567
14568 ins_cost(INSN_COST * 5);
14569 format %{ "scvtfwd $dst, $src \t// i2d" %}
14570
14571 ins_encode %{
14572 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14573 %}
14574
14575 ins_pipe(fp_i2d);
14576 %}
14577
14578 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14579 match(Set dst (ConvL2D src));
14580
14581 ins_cost(INSN_COST * 5);
14582 format %{ "scvtfd $dst, $src \t// l2d" %}
14583
14584 ins_encode %{
14585 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14586 %}
14587
14588 ins_pipe(fp_l2d);
14589 %}
14590
14591 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14592 %{
14593 match(Set dst (RoundD src));
14594 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14595 format %{ "java_round_double $dst,$src"%}
14596 ins_encode %{
14597 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14598 as_FloatRegister($ftmp$$reg));
14599 %}
14600 ins_pipe(pipe_slow);
14601 %}
14602
14603 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14604 %{
14605 match(Set dst (RoundF src));
14606 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14607 format %{ "java_round_float $dst,$src"%}
14608 ins_encode %{
14609 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14610 as_FloatRegister($ftmp$$reg));
14611 %}
14612 ins_pipe(pipe_slow);
14613 %}
14614
14615 // stack <-> reg and reg <-> reg shuffles with no conversion
14616
14617 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14618
14619 match(Set dst (MoveF2I src));
14620
14621 effect(DEF dst, USE src);
14622
14623 ins_cost(4 * INSN_COST);
14624
14625 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14626
14627 ins_encode %{
14628 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14629 %}
14630
14631 ins_pipe(iload_reg_reg);
14632
14633 %}
14634
14635 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14636
14637 match(Set dst (MoveI2F src));
14638
14639 effect(DEF dst, USE src);
14640
14641 ins_cost(4 * INSN_COST);
14642
14643 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14644
14645 ins_encode %{
14646 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14647 %}
14648
14649 ins_pipe(pipe_class_memory);
14650
14651 %}
14652
14653 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14654
14655 match(Set dst (MoveD2L src));
14656
14657 effect(DEF dst, USE src);
14658
14659 ins_cost(4 * INSN_COST);
14660
14661 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14662
14663 ins_encode %{
14664 __ ldr($dst$$Register, Address(sp, $src$$disp));
14665 %}
14666
14667 ins_pipe(iload_reg_reg);
14668
14669 %}
14670
14671 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14672
14673 match(Set dst (MoveL2D src));
14674
14675 effect(DEF dst, USE src);
14676
14677 ins_cost(4 * INSN_COST);
14678
14679 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14680
14681 ins_encode %{
14682 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14683 %}
14684
14685 ins_pipe(pipe_class_memory);
14686
14687 %}
14688
14689 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14690
14691 match(Set dst (MoveF2I src));
14692
14693 effect(DEF dst, USE src);
14694
14695 ins_cost(INSN_COST);
14696
14697 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14698
14699 ins_encode %{
14700 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14701 %}
14702
14703 ins_pipe(pipe_class_memory);
14704
14705 %}
14706
14707 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14708
14709 match(Set dst (MoveI2F src));
14710
14711 effect(DEF dst, USE src);
14712
14713 ins_cost(INSN_COST);
14714
14715 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14716
14717 ins_encode %{
14718 __ strw($src$$Register, Address(sp, $dst$$disp));
14719 %}
14720
14721 ins_pipe(istore_reg_reg);
14722
14723 %}
14724
14725 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14726
14727 match(Set dst (MoveD2L src));
14728
14729 effect(DEF dst, USE src);
14730
14731 ins_cost(INSN_COST);
14732
14733 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14734
14735 ins_encode %{
14736 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14737 %}
14738
14739 ins_pipe(pipe_class_memory);
14740
14741 %}
14742
14743 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14744
14745 match(Set dst (MoveL2D src));
14746
14747 effect(DEF dst, USE src);
14748
14749 ins_cost(INSN_COST);
14750
14751 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14752
14753 ins_encode %{
14754 __ str($src$$Register, Address(sp, $dst$$disp));
14755 %}
14756
14757 ins_pipe(istore_reg_reg);
14758
14759 %}
14760
14761 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14762
14763 match(Set dst (MoveF2I src));
14764
14765 effect(DEF dst, USE src);
14766
14767 ins_cost(INSN_COST);
14768
14769 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14770
14771 ins_encode %{
14772 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14773 %}
14774
14775 ins_pipe(fp_f2i);
14776
14777 %}
14778
14779 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14780
14781 match(Set dst (MoveI2F src));
14782
14783 effect(DEF dst, USE src);
14784
14785 ins_cost(INSN_COST);
14786
14787 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14788
14789 ins_encode %{
14790 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14791 %}
14792
14793 ins_pipe(fp_i2f);
14794
14795 %}
14796
14797 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14798
14799 match(Set dst (MoveD2L src));
14800
14801 effect(DEF dst, USE src);
14802
14803 ins_cost(INSN_COST);
14804
14805 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14806
14807 ins_encode %{
14808 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14809 %}
14810
14811 ins_pipe(fp_d2l);
14812
14813 %}
14814
14815 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14816
14817 match(Set dst (MoveL2D src));
14818
14819 effect(DEF dst, USE src);
14820
14821 ins_cost(INSN_COST);
14822
14823 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14824
14825 ins_encode %{
14826 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14827 %}
14828
14829 ins_pipe(fp_l2d);
14830
14831 %}
14832
14833 // ============================================================================
14834 // clearing of an array
14835
14836 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14837 %{
14838 match(Set dummy (ClearArray cnt base));
14839 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14840
14841 ins_cost(4 * INSN_COST);
14842 format %{ "ClearArray $cnt, $base" %}
14843
14844 ins_encode %{
14845 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14846 if (tpc == nullptr) {
14847 ciEnv::current()->record_failure("CodeCache is full");
14848 return;
14849 }
14850 %}
14851
14852 ins_pipe(pipe_class_memory);
14853 %}
14854
14855 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14856 %{
14857 predicate((uint64_t)n->in(2)->get_long()
14858 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14859 match(Set dummy (ClearArray cnt base));
14860 effect(TEMP temp, USE_KILL base, KILL cr);
14861
14862 ins_cost(4 * INSN_COST);
14863 format %{ "ClearArray $cnt, $base" %}
14864
14865 ins_encode %{
14866 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14867 if (tpc == nullptr) {
14868 ciEnv::current()->record_failure("CodeCache is full");
14869 return;
14870 }
14871 %}
14872
14873 ins_pipe(pipe_class_memory);
14874 %}
14875
14876 // ============================================================================
14877 // Overflow Math Instructions
14878
14879 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14880 %{
14881 match(Set cr (OverflowAddI op1 op2));
14882
14883 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14884 ins_cost(INSN_COST);
14885 ins_encode %{
14886 __ cmnw($op1$$Register, $op2$$Register);
14887 %}
14888
14889 ins_pipe(icmp_reg_reg);
14890 %}
14891
14892 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14893 %{
14894 match(Set cr (OverflowAddI op1 op2));
14895
14896 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14897 ins_cost(INSN_COST);
14898 ins_encode %{
14899 __ cmnw($op1$$Register, $op2$$constant);
14900 %}
14901
14902 ins_pipe(icmp_reg_imm);
14903 %}
14904
14905 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14906 %{
14907 match(Set cr (OverflowAddL op1 op2));
14908
14909 format %{ "cmn $op1, $op2\t# overflow check long" %}
14910 ins_cost(INSN_COST);
14911 ins_encode %{
14912 __ cmn($op1$$Register, $op2$$Register);
14913 %}
14914
14915 ins_pipe(icmp_reg_reg);
14916 %}
14917
14918 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14919 %{
14920 match(Set cr (OverflowAddL op1 op2));
14921
14922 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14923 ins_cost(INSN_COST);
14924 ins_encode %{
14925 __ adds(zr, $op1$$Register, $op2$$constant);
14926 %}
14927
14928 ins_pipe(icmp_reg_imm);
14929 %}
14930
14931 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14932 %{
14933 match(Set cr (OverflowSubI op1 op2));
14934
14935 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14936 ins_cost(INSN_COST);
14937 ins_encode %{
14938 __ cmpw($op1$$Register, $op2$$Register);
14939 %}
14940
14941 ins_pipe(icmp_reg_reg);
14942 %}
14943
14944 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14945 %{
14946 match(Set cr (OverflowSubI op1 op2));
14947
14948 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14949 ins_cost(INSN_COST);
14950 ins_encode %{
14951 __ cmpw($op1$$Register, $op2$$constant);
14952 %}
14953
14954 ins_pipe(icmp_reg_imm);
14955 %}
14956
14957 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14958 %{
14959 match(Set cr (OverflowSubL op1 op2));
14960
14961 format %{ "cmp $op1, $op2\t# overflow check long" %}
14962 ins_cost(INSN_COST);
14963 ins_encode %{
14964 __ cmp($op1$$Register, $op2$$Register);
14965 %}
14966
14967 ins_pipe(icmp_reg_reg);
14968 %}
14969
14970 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14971 %{
14972 match(Set cr (OverflowSubL op1 op2));
14973
14974 format %{ "cmp $op1, $op2\t# overflow check long" %}
14975 ins_cost(INSN_COST);
14976 ins_encode %{
14977 __ subs(zr, $op1$$Register, $op2$$constant);
14978 %}
14979
14980 ins_pipe(icmp_reg_imm);
14981 %}
14982
14983 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14984 %{
14985 match(Set cr (OverflowSubI zero op1));
14986
14987 format %{ "cmpw zr, $op1\t# overflow check int" %}
14988 ins_cost(INSN_COST);
14989 ins_encode %{
14990 __ cmpw(zr, $op1$$Register);
14991 %}
14992
14993 ins_pipe(icmp_reg_imm);
14994 %}
14995
14996 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14997 %{
14998 match(Set cr (OverflowSubL zero op1));
14999
15000 format %{ "cmp zr, $op1\t# overflow check long" %}
15001 ins_cost(INSN_COST);
15002 ins_encode %{
15003 __ cmp(zr, $op1$$Register);
15004 %}
15005
15006 ins_pipe(icmp_reg_imm);
15007 %}
15008
15009 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15010 %{
15011 match(Set cr (OverflowMulI op1 op2));
15012
15013 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15014 "cmp rscratch1, rscratch1, sxtw\n\t"
15015 "movw rscratch1, #0x80000000\n\t"
15016 "cselw rscratch1, rscratch1, zr, NE\n\t"
15017 "cmpw rscratch1, #1" %}
15018 ins_cost(5 * INSN_COST);
15019 ins_encode %{
15020 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15021 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15022 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15023 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15024 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15025 %}
15026
15027 ins_pipe(pipe_slow);
15028 %}
15029
15030 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15031 %{
15032 match(If cmp (OverflowMulI op1 op2));
15033 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15034 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15035 effect(USE labl, KILL cr);
15036
15037 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15038 "cmp rscratch1, rscratch1, sxtw\n\t"
15039 "b$cmp $labl" %}
15040 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15041 ins_encode %{
15042 Label* L = $labl$$label;
15043 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15044 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15045 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15046 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15047 %}
15048
15049 ins_pipe(pipe_serial);
15050 %}
15051
15052 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15053 %{
15054 match(Set cr (OverflowMulL op1 op2));
15055
15056 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15057 "smulh rscratch2, $op1, $op2\n\t"
15058 "cmp rscratch2, rscratch1, ASR #63\n\t"
15059 "movw rscratch1, #0x80000000\n\t"
15060 "cselw rscratch1, rscratch1, zr, NE\n\t"
15061 "cmpw rscratch1, #1" %}
15062 ins_cost(6 * INSN_COST);
15063 ins_encode %{
15064 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15065 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15066 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15067 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15068 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15069 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15070 %}
15071
15072 ins_pipe(pipe_slow);
15073 %}
15074
15075 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15076 %{
15077 match(If cmp (OverflowMulL op1 op2));
15078 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15079 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15080 effect(USE labl, KILL cr);
15081
15082 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15083 "smulh rscratch2, $op1, $op2\n\t"
15084 "cmp rscratch2, rscratch1, ASR #63\n\t"
15085 "b$cmp $labl" %}
15086 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15087 ins_encode %{
15088 Label* L = $labl$$label;
15089 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15090 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15091 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15092 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15093 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15094 %}
15095
15096 ins_pipe(pipe_serial);
15097 %}
15098
15099 // ============================================================================
15100 // Compare Instructions
15101
15102 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15103 %{
15104 match(Set cr (CmpI op1 op2));
15105
15106 effect(DEF cr, USE op1, USE op2);
15107
15108 ins_cost(INSN_COST);
15109 format %{ "cmpw $op1, $op2" %}
15110
15111 ins_encode(aarch64_enc_cmpw(op1, op2));
15112
15113 ins_pipe(icmp_reg_reg);
15114 %}
15115
15116 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15117 %{
15118 match(Set cr (CmpI op1 zero));
15119
15120 effect(DEF cr, USE op1);
15121
15122 ins_cost(INSN_COST);
15123 format %{ "cmpw $op1, 0" %}
15124
15125 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15126
15127 ins_pipe(icmp_reg_imm);
15128 %}
15129
15130 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15131 %{
15132 match(Set cr (CmpI op1 op2));
15133
15134 effect(DEF cr, USE op1);
15135
15136 ins_cost(INSN_COST);
15137 format %{ "cmpw $op1, $op2" %}
15138
15139 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15140
15141 ins_pipe(icmp_reg_imm);
15142 %}
15143
15144 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15145 %{
15146 match(Set cr (CmpI op1 op2));
15147
15148 effect(DEF cr, USE op1);
15149
15150 ins_cost(INSN_COST * 2);
15151 format %{ "cmpw $op1, $op2" %}
15152
15153 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15154
15155 ins_pipe(icmp_reg_imm);
15156 %}
15157
15158 // Unsigned compare Instructions; really, same as signed compare
15159 // except it should only be used to feed an If or a CMovI which takes a
15160 // cmpOpU.
15161
15162 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15163 %{
15164 match(Set cr (CmpU op1 op2));
15165
15166 effect(DEF cr, USE op1, USE op2);
15167
15168 ins_cost(INSN_COST);
15169 format %{ "cmpw $op1, $op2\t# unsigned" %}
15170
15171 ins_encode(aarch64_enc_cmpw(op1, op2));
15172
15173 ins_pipe(icmp_reg_reg);
15174 %}
15175
15176 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15177 %{
15178 match(Set cr (CmpU op1 zero));
15179
15180 effect(DEF cr, USE op1);
15181
15182 ins_cost(INSN_COST);
15183 format %{ "cmpw $op1, #0\t# unsigned" %}
15184
15185 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15186
15187 ins_pipe(icmp_reg_imm);
15188 %}
15189
15190 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15191 %{
15192 match(Set cr (CmpU op1 op2));
15193
15194 effect(DEF cr, USE op1);
15195
15196 ins_cost(INSN_COST);
15197 format %{ "cmpw $op1, $op2\t# unsigned" %}
15198
15199 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15200
15201 ins_pipe(icmp_reg_imm);
15202 %}
15203
15204 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15205 %{
15206 match(Set cr (CmpU op1 op2));
15207
15208 effect(DEF cr, USE op1);
15209
15210 ins_cost(INSN_COST * 2);
15211 format %{ "cmpw $op1, $op2\t# unsigned" %}
15212
15213 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15214
15215 ins_pipe(icmp_reg_imm);
15216 %}
15217
15218 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15219 %{
15220 match(Set cr (CmpL op1 op2));
15221
15222 effect(DEF cr, USE op1, USE op2);
15223
15224 ins_cost(INSN_COST);
15225 format %{ "cmp $op1, $op2" %}
15226
15227 ins_encode(aarch64_enc_cmp(op1, op2));
15228
15229 ins_pipe(icmp_reg_reg);
15230 %}
15231
15232 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15233 %{
15234 match(Set cr (CmpL op1 zero));
15235
15236 effect(DEF cr, USE op1);
15237
15238 ins_cost(INSN_COST);
15239 format %{ "tst $op1" %}
15240
15241 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15242
15243 ins_pipe(icmp_reg_imm);
15244 %}
15245
15246 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15247 %{
15248 match(Set cr (CmpL op1 op2));
15249
15250 effect(DEF cr, USE op1);
15251
15252 ins_cost(INSN_COST);
15253 format %{ "cmp $op1, $op2" %}
15254
15255 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15256
15257 ins_pipe(icmp_reg_imm);
15258 %}
15259
15260 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15261 %{
15262 match(Set cr (CmpL op1 op2));
15263
15264 effect(DEF cr, USE op1);
15265
15266 ins_cost(INSN_COST * 2);
15267 format %{ "cmp $op1, $op2" %}
15268
15269 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15270
15271 ins_pipe(icmp_reg_imm);
15272 %}
15273
15274 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15275 %{
15276 match(Set cr (CmpUL op1 op2));
15277
15278 effect(DEF cr, USE op1, USE op2);
15279
15280 ins_cost(INSN_COST);
15281 format %{ "cmp $op1, $op2" %}
15282
15283 ins_encode(aarch64_enc_cmp(op1, op2));
15284
15285 ins_pipe(icmp_reg_reg);
15286 %}
15287
15288 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15289 %{
15290 match(Set cr (CmpUL op1 zero));
15291
15292 effect(DEF cr, USE op1);
15293
15294 ins_cost(INSN_COST);
15295 format %{ "tst $op1" %}
15296
15297 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15298
15299 ins_pipe(icmp_reg_imm);
15300 %}
15301
15302 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15303 %{
15304 match(Set cr (CmpUL op1 op2));
15305
15306 effect(DEF cr, USE op1);
15307
15308 ins_cost(INSN_COST);
15309 format %{ "cmp $op1, $op2" %}
15310
15311 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15312
15313 ins_pipe(icmp_reg_imm);
15314 %}
15315
15316 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15317 %{
15318 match(Set cr (CmpUL op1 op2));
15319
15320 effect(DEF cr, USE op1);
15321
15322 ins_cost(INSN_COST * 2);
15323 format %{ "cmp $op1, $op2" %}
15324
15325 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15326
15327 ins_pipe(icmp_reg_imm);
15328 %}
15329
15330 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15331 %{
15332 match(Set cr (CmpP op1 op2));
15333
15334 effect(DEF cr, USE op1, USE op2);
15335
15336 ins_cost(INSN_COST);
15337 format %{ "cmp $op1, $op2\t // ptr" %}
15338
15339 ins_encode(aarch64_enc_cmpp(op1, op2));
15340
15341 ins_pipe(icmp_reg_reg);
15342 %}
15343
15344 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15345 %{
15346 match(Set cr (CmpN op1 op2));
15347
15348 effect(DEF cr, USE op1, USE op2);
15349
15350 ins_cost(INSN_COST);
15351 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15352
15353 ins_encode(aarch64_enc_cmpn(op1, op2));
15354
15355 ins_pipe(icmp_reg_reg);
15356 %}
15357
15358 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15359 %{
15360 match(Set cr (CmpP op1 zero));
15361
15362 effect(DEF cr, USE op1, USE zero);
15363
15364 ins_cost(INSN_COST);
15365 format %{ "cmp $op1, 0\t // ptr" %}
15366
15367 ins_encode(aarch64_enc_testp(op1));
15368
15369 ins_pipe(icmp_reg_imm);
15370 %}
15371
15372 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15373 %{
15374 match(Set cr (CmpN op1 zero));
15375
15376 effect(DEF cr, USE op1, USE zero);
15377
15378 ins_cost(INSN_COST);
15379 format %{ "cmp $op1, 0\t // compressed ptr" %}
15380
15381 ins_encode(aarch64_enc_testn(op1));
15382
15383 ins_pipe(icmp_reg_imm);
15384 %}
15385
15386 // FP comparisons
15387 //
15388 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15389 // using normal cmpOp. See declaration of rFlagsReg for details.
15390
15391 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15392 %{
15393 match(Set cr (CmpF src1 src2));
15394
15395 ins_cost(3 * INSN_COST);
15396 format %{ "fcmps $src1, $src2" %}
15397
15398 ins_encode %{
15399 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15400 %}
15401
15402 ins_pipe(pipe_class_compare);
15403 %}
15404
15405 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15406 %{
15407 match(Set cr (CmpF src1 src2));
15408
15409 ins_cost(3 * INSN_COST);
15410 format %{ "fcmps $src1, 0.0" %}
15411
15412 ins_encode %{
15413 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15414 %}
15415
15416 ins_pipe(pipe_class_compare);
15417 %}
15418 // FROM HERE
15419
15420 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15421 %{
15422 match(Set cr (CmpD src1 src2));
15423
15424 ins_cost(3 * INSN_COST);
15425 format %{ "fcmpd $src1, $src2" %}
15426
15427 ins_encode %{
15428 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15429 %}
15430
15431 ins_pipe(pipe_class_compare);
15432 %}
15433
15434 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15435 %{
15436 match(Set cr (CmpD src1 src2));
15437
15438 ins_cost(3 * INSN_COST);
15439 format %{ "fcmpd $src1, 0.0" %}
15440
15441 ins_encode %{
15442 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15443 %}
15444
15445 ins_pipe(pipe_class_compare);
15446 %}
15447
15448 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15449 %{
15450 match(Set dst (CmpF3 src1 src2));
15451 effect(KILL cr);
15452
15453 ins_cost(5 * INSN_COST);
15454 format %{ "fcmps $src1, $src2\n\t"
15455 "csinvw($dst, zr, zr, eq\n\t"
15456 "csnegw($dst, $dst, $dst, lt)"
15457 %}
15458
15459 ins_encode %{
15460 Label done;
15461 FloatRegister s1 = as_FloatRegister($src1$$reg);
15462 FloatRegister s2 = as_FloatRegister($src2$$reg);
15463 Register d = as_Register($dst$$reg);
15464 __ fcmps(s1, s2);
15465 // installs 0 if EQ else -1
15466 __ csinvw(d, zr, zr, Assembler::EQ);
15467 // keeps -1 if less or unordered else installs 1
15468 __ csnegw(d, d, d, Assembler::LT);
15469 __ bind(done);
15470 %}
15471
15472 ins_pipe(pipe_class_default);
15473
15474 %}
15475
15476 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15477 %{
15478 match(Set dst (CmpD3 src1 src2));
15479 effect(KILL cr);
15480
15481 ins_cost(5 * INSN_COST);
15482 format %{ "fcmpd $src1, $src2\n\t"
15483 "csinvw($dst, zr, zr, eq\n\t"
15484 "csnegw($dst, $dst, $dst, lt)"
15485 %}
15486
15487 ins_encode %{
15488 Label done;
15489 FloatRegister s1 = as_FloatRegister($src1$$reg);
15490 FloatRegister s2 = as_FloatRegister($src2$$reg);
15491 Register d = as_Register($dst$$reg);
15492 __ fcmpd(s1, s2);
15493 // installs 0 if EQ else -1
15494 __ csinvw(d, zr, zr, Assembler::EQ);
15495 // keeps -1 if less or unordered else installs 1
15496 __ csnegw(d, d, d, Assembler::LT);
15497 __ bind(done);
15498 %}
15499 ins_pipe(pipe_class_default);
15500
15501 %}
15502
15503 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15504 %{
15505 match(Set dst (CmpF3 src1 zero));
15506 effect(KILL cr);
15507
15508 ins_cost(5 * INSN_COST);
15509 format %{ "fcmps $src1, 0.0\n\t"
15510 "csinvw($dst, zr, zr, eq\n\t"
15511 "csnegw($dst, $dst, $dst, lt)"
15512 %}
15513
15514 ins_encode %{
15515 Label done;
15516 FloatRegister s1 = as_FloatRegister($src1$$reg);
15517 Register d = as_Register($dst$$reg);
15518 __ fcmps(s1, 0.0);
15519 // installs 0 if EQ else -1
15520 __ csinvw(d, zr, zr, Assembler::EQ);
15521 // keeps -1 if less or unordered else installs 1
15522 __ csnegw(d, d, d, Assembler::LT);
15523 __ bind(done);
15524 %}
15525
15526 ins_pipe(pipe_class_default);
15527
15528 %}
15529
15530 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15531 %{
15532 match(Set dst (CmpD3 src1 zero));
15533 effect(KILL cr);
15534
15535 ins_cost(5 * INSN_COST);
15536 format %{ "fcmpd $src1, 0.0\n\t"
15537 "csinvw($dst, zr, zr, eq\n\t"
15538 "csnegw($dst, $dst, $dst, lt)"
15539 %}
15540
15541 ins_encode %{
15542 Label done;
15543 FloatRegister s1 = as_FloatRegister($src1$$reg);
15544 Register d = as_Register($dst$$reg);
15545 __ fcmpd(s1, 0.0);
15546 // installs 0 if EQ else -1
15547 __ csinvw(d, zr, zr, Assembler::EQ);
15548 // keeps -1 if less or unordered else installs 1
15549 __ csnegw(d, d, d, Assembler::LT);
15550 __ bind(done);
15551 %}
15552 ins_pipe(pipe_class_default);
15553
15554 %}
15555
15556 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15557 %{
15558 match(Set dst (CmpLTMask p q));
15559 effect(KILL cr);
15560
15561 ins_cost(3 * INSN_COST);
15562
15563 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15564 "csetw $dst, lt\n\t"
15565 "subw $dst, zr, $dst"
15566 %}
15567
15568 ins_encode %{
15569 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15570 __ csetw(as_Register($dst$$reg), Assembler::LT);
15571 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15572 %}
15573
15574 ins_pipe(ialu_reg_reg);
15575 %}
15576
15577 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15578 %{
15579 match(Set dst (CmpLTMask src zero));
15580 effect(KILL cr);
15581
15582 ins_cost(INSN_COST);
15583
15584 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15585
15586 ins_encode %{
15587 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15588 %}
15589
15590 ins_pipe(ialu_reg_shift);
15591 %}
15592
15593 // ============================================================================
15594 // Max and Min
15595
15596 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15597
15598 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15599 %{
15600 effect(DEF cr, USE src);
15601 ins_cost(INSN_COST);
15602 format %{ "cmpw $src, 0" %}
15603
15604 ins_encode %{
15605 __ cmpw($src$$Register, 0);
15606 %}
15607 ins_pipe(icmp_reg_imm);
15608 %}
15609
15610 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15611 %{
15612 match(Set dst (MinI src1 src2));
15613 ins_cost(INSN_COST * 3);
15614
15615 expand %{
15616 rFlagsReg cr;
15617 compI_reg_reg(cr, src1, src2);
15618 cmovI_reg_reg_lt(dst, src1, src2, cr);
15619 %}
15620 %}
15621
15622 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15623 %{
15624 match(Set dst (MaxI src1 src2));
15625 ins_cost(INSN_COST * 3);
15626
15627 expand %{
15628 rFlagsReg cr;
15629 compI_reg_reg(cr, src1, src2);
15630 cmovI_reg_reg_gt(dst, src1, src2, cr);
15631 %}
15632 %}
15633
15634
15635 // ============================================================================
15636 // Branch Instructions
15637
15638 // Direct Branch.
15639 instruct branch(label lbl)
15640 %{
15641 match(Goto);
15642
15643 effect(USE lbl);
15644
15645 ins_cost(BRANCH_COST);
15646 format %{ "b $lbl" %}
15647
15648 ins_encode(aarch64_enc_b(lbl));
15649
15650 ins_pipe(pipe_branch);
15651 %}
15652
15653 // Conditional Near Branch
15654 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15655 %{
15656 // Same match rule as `branchConFar'.
15657 match(If cmp cr);
15658
15659 effect(USE lbl);
15660
15661 ins_cost(BRANCH_COST);
15662 // If set to 1 this indicates that the current instruction is a
15663 // short variant of a long branch. This avoids using this
15664 // instruction in first-pass matching. It will then only be used in
15665 // the `Shorten_branches' pass.
15666 // ins_short_branch(1);
15667 format %{ "b$cmp $lbl" %}
15668
15669 ins_encode(aarch64_enc_br_con(cmp, lbl));
15670
15671 ins_pipe(pipe_branch_cond);
15672 %}
15673
15674 // Conditional Near Branch Unsigned
15675 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15676 %{
15677 // Same match rule as `branchConFar'.
15678 match(If cmp cr);
15679
15680 effect(USE lbl);
15681
15682 ins_cost(BRANCH_COST);
15683 // If set to 1 this indicates that the current instruction is a
15684 // short variant of a long branch. This avoids using this
15685 // instruction in first-pass matching. It will then only be used in
15686 // the `Shorten_branches' pass.
15687 // ins_short_branch(1);
15688 format %{ "b$cmp $lbl\t# unsigned" %}
15689
15690 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15691
15692 ins_pipe(pipe_branch_cond);
15693 %}
15694
15695 // Make use of CBZ and CBNZ. These instructions, as well as being
15696 // shorter than (cmp; branch), have the additional benefit of not
15697 // killing the flags.
15698
15699 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15700 match(If cmp (CmpI op1 op2));
15701 effect(USE labl);
15702
15703 ins_cost(BRANCH_COST);
15704 format %{ "cbw$cmp $op1, $labl" %}
15705 ins_encode %{
15706 Label* L = $labl$$label;
15707 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15708 if (cond == Assembler::EQ)
15709 __ cbzw($op1$$Register, *L);
15710 else
15711 __ cbnzw($op1$$Register, *L);
15712 %}
15713 ins_pipe(pipe_cmp_branch);
15714 %}
15715
15716 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15717 match(If cmp (CmpL op1 op2));
15718 effect(USE labl);
15719
15720 ins_cost(BRANCH_COST);
15721 format %{ "cb$cmp $op1, $labl" %}
15722 ins_encode %{
15723 Label* L = $labl$$label;
15724 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15725 if (cond == Assembler::EQ)
15726 __ cbz($op1$$Register, *L);
15727 else
15728 __ cbnz($op1$$Register, *L);
15729 %}
15730 ins_pipe(pipe_cmp_branch);
15731 %}
15732
15733 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15734 match(If cmp (CmpP op1 op2));
15735 effect(USE labl);
15736
15737 ins_cost(BRANCH_COST);
15738 format %{ "cb$cmp $op1, $labl" %}
15739 ins_encode %{
15740 Label* L = $labl$$label;
15741 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15742 if (cond == Assembler::EQ)
15743 __ cbz($op1$$Register, *L);
15744 else
15745 __ cbnz($op1$$Register, *L);
15746 %}
15747 ins_pipe(pipe_cmp_branch);
15748 %}
15749
15750 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15751 match(If cmp (CmpN op1 op2));
15752 effect(USE labl);
15753
15754 ins_cost(BRANCH_COST);
15755 format %{ "cbw$cmp $op1, $labl" %}
15756 ins_encode %{
15757 Label* L = $labl$$label;
15758 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15759 if (cond == Assembler::EQ)
15760 __ cbzw($op1$$Register, *L);
15761 else
15762 __ cbnzw($op1$$Register, *L);
15763 %}
15764 ins_pipe(pipe_cmp_branch);
15765 %}
15766
15767 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15768 match(If cmp (CmpP (DecodeN oop) zero));
15769 effect(USE labl);
15770
15771 ins_cost(BRANCH_COST);
15772 format %{ "cb$cmp $oop, $labl" %}
15773 ins_encode %{
15774 Label* L = $labl$$label;
15775 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15776 if (cond == Assembler::EQ)
15777 __ cbzw($oop$$Register, *L);
15778 else
15779 __ cbnzw($oop$$Register, *L);
15780 %}
15781 ins_pipe(pipe_cmp_branch);
15782 %}
15783
15784 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15785 match(If cmp (CmpU op1 op2));
15786 effect(USE labl);
15787
15788 ins_cost(BRANCH_COST);
15789 format %{ "cbw$cmp $op1, $labl" %}
15790 ins_encode %{
15791 Label* L = $labl$$label;
15792 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15793 if (cond == Assembler::EQ || cond == Assembler::LS) {
15794 __ cbzw($op1$$Register, *L);
15795 } else {
15796 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15797 __ cbnzw($op1$$Register, *L);
15798 }
15799 %}
15800 ins_pipe(pipe_cmp_branch);
15801 %}
15802
15803 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15804 match(If cmp (CmpUL op1 op2));
15805 effect(USE labl);
15806
15807 ins_cost(BRANCH_COST);
15808 format %{ "cb$cmp $op1, $labl" %}
15809 ins_encode %{
15810 Label* L = $labl$$label;
15811 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15812 if (cond == Assembler::EQ || cond == Assembler::LS) {
15813 __ cbz($op1$$Register, *L);
15814 } else {
15815 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15816 __ cbnz($op1$$Register, *L);
15817 }
15818 %}
15819 ins_pipe(pipe_cmp_branch);
15820 %}
15821
15822 // Test bit and Branch
15823
15824 // Patterns for short (< 32KiB) variants
15825 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15826 match(If cmp (CmpL op1 op2));
15827 effect(USE labl);
15828
15829 ins_cost(BRANCH_COST);
15830 format %{ "cb$cmp $op1, $labl # long" %}
15831 ins_encode %{
15832 Label* L = $labl$$label;
15833 Assembler::Condition cond =
15834 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15835 __ tbr(cond, $op1$$Register, 63, *L);
15836 %}
15837 ins_pipe(pipe_cmp_branch);
15838 ins_short_branch(1);
15839 %}
15840
15841 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15842 match(If cmp (CmpI op1 op2));
15843 effect(USE labl);
15844
15845 ins_cost(BRANCH_COST);
15846 format %{ "cb$cmp $op1, $labl # int" %}
15847 ins_encode %{
15848 Label* L = $labl$$label;
15849 Assembler::Condition cond =
15850 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15851 __ tbr(cond, $op1$$Register, 31, *L);
15852 %}
15853 ins_pipe(pipe_cmp_branch);
15854 ins_short_branch(1);
15855 %}
15856
15857 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15858 match(If cmp (CmpL (AndL op1 op2) op3));
15859 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15860 effect(USE labl);
15861
15862 ins_cost(BRANCH_COST);
15863 format %{ "tb$cmp $op1, $op2, $labl" %}
15864 ins_encode %{
15865 Label* L = $labl$$label;
15866 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15867 int bit = exact_log2_long($op2$$constant);
15868 __ tbr(cond, $op1$$Register, bit, *L);
15869 %}
15870 ins_pipe(pipe_cmp_branch);
15871 ins_short_branch(1);
15872 %}
15873
15874 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15875 match(If cmp (CmpI (AndI op1 op2) op3));
15876 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15877 effect(USE labl);
15878
15879 ins_cost(BRANCH_COST);
15880 format %{ "tb$cmp $op1, $op2, $labl" %}
15881 ins_encode %{
15882 Label* L = $labl$$label;
15883 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15884 int bit = exact_log2((juint)$op2$$constant);
15885 __ tbr(cond, $op1$$Register, bit, *L);
15886 %}
15887 ins_pipe(pipe_cmp_branch);
15888 ins_short_branch(1);
15889 %}
15890
15891 // And far variants
15892 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15893 match(If cmp (CmpL op1 op2));
15894 effect(USE labl);
15895
15896 ins_cost(BRANCH_COST);
15897 format %{ "cb$cmp $op1, $labl # long" %}
15898 ins_encode %{
15899 Label* L = $labl$$label;
15900 Assembler::Condition cond =
15901 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15902 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15903 %}
15904 ins_pipe(pipe_cmp_branch);
15905 %}
15906
15907 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15908 match(If cmp (CmpI op1 op2));
15909 effect(USE labl);
15910
15911 ins_cost(BRANCH_COST);
15912 format %{ "cb$cmp $op1, $labl # int" %}
15913 ins_encode %{
15914 Label* L = $labl$$label;
15915 Assembler::Condition cond =
15916 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15917 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15918 %}
15919 ins_pipe(pipe_cmp_branch);
15920 %}
15921
15922 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15923 match(If cmp (CmpL (AndL op1 op2) op3));
15924 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15925 effect(USE labl);
15926
15927 ins_cost(BRANCH_COST);
15928 format %{ "tb$cmp $op1, $op2, $labl" %}
15929 ins_encode %{
15930 Label* L = $labl$$label;
15931 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15932 int bit = exact_log2_long($op2$$constant);
15933 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15934 %}
15935 ins_pipe(pipe_cmp_branch);
15936 %}
15937
15938 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15939 match(If cmp (CmpI (AndI op1 op2) op3));
15940 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15941 effect(USE labl);
15942
15943 ins_cost(BRANCH_COST);
15944 format %{ "tb$cmp $op1, $op2, $labl" %}
15945 ins_encode %{
15946 Label* L = $labl$$label;
15947 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15948 int bit = exact_log2((juint)$op2$$constant);
15949 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15950 %}
15951 ins_pipe(pipe_cmp_branch);
15952 %}
15953
15954 // Test bits
15955
15956 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15957 match(Set cr (CmpL (AndL op1 op2) op3));
15958 predicate(Assembler::operand_valid_for_logical_immediate
15959 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15960
15961 ins_cost(INSN_COST);
15962 format %{ "tst $op1, $op2 # long" %}
15963 ins_encode %{
15964 __ tst($op1$$Register, $op2$$constant);
15965 %}
15966 ins_pipe(ialu_reg_reg);
15967 %}
15968
15969 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15970 match(Set cr (CmpI (AndI op1 op2) op3));
15971 predicate(Assembler::operand_valid_for_logical_immediate
15972 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15973
15974 ins_cost(INSN_COST);
15975 format %{ "tst $op1, $op2 # int" %}
15976 ins_encode %{
15977 __ tstw($op1$$Register, $op2$$constant);
15978 %}
15979 ins_pipe(ialu_reg_reg);
15980 %}
15981
15982 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15983 match(Set cr (CmpL (AndL op1 op2) op3));
15984
15985 ins_cost(INSN_COST);
15986 format %{ "tst $op1, $op2 # long" %}
15987 ins_encode %{
15988 __ tst($op1$$Register, $op2$$Register);
15989 %}
15990 ins_pipe(ialu_reg_reg);
15991 %}
15992
15993 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15994 match(Set cr (CmpI (AndI op1 op2) op3));
15995
15996 ins_cost(INSN_COST);
15997 format %{ "tstw $op1, $op2 # int" %}
15998 ins_encode %{
15999 __ tstw($op1$$Register, $op2$$Register);
16000 %}
16001 ins_pipe(ialu_reg_reg);
16002 %}
16003
16004
16005 // Conditional Far Branch
16006 // Conditional Far Branch Unsigned
16007 // TODO: fixme
16008
16009 // counted loop end branch near
16010 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16011 %{
16012 match(CountedLoopEnd cmp cr);
16013
16014 effect(USE lbl);
16015
16016 ins_cost(BRANCH_COST);
16017 // short variant.
16018 // ins_short_branch(1);
16019 format %{ "b$cmp $lbl \t// counted loop end" %}
16020
16021 ins_encode(aarch64_enc_br_con(cmp, lbl));
16022
16023 ins_pipe(pipe_branch);
16024 %}
16025
16026 // counted loop end branch far
16027 // TODO: fixme
16028
16029 // ============================================================================
16030 // inlined locking and unlocking
16031
16032 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16033 %{
16034 predicate(LockingMode != LM_LIGHTWEIGHT);
16035 match(Set cr (FastLock object box));
16036 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16037
16038 ins_cost(5 * INSN_COST);
16039 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16040
16041 ins_encode %{
16042 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16043 %}
16044
16045 ins_pipe(pipe_serial);
16046 %}
16047
16048 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16049 %{
16050 predicate(LockingMode != LM_LIGHTWEIGHT);
16051 match(Set cr (FastUnlock object box));
16052 effect(TEMP tmp, TEMP tmp2);
16053
16054 ins_cost(5 * INSN_COST);
16055 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16056
16057 ins_encode %{
16058 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16059 %}
16060
16061 ins_pipe(pipe_serial);
16062 %}
16063
16064 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16065 %{
16066 predicate(LockingMode == LM_LIGHTWEIGHT);
16067 match(Set cr (FastLock object box));
16068 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16069
16070 ins_cost(5 * INSN_COST);
16071 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16072
16073 ins_encode %{
16074 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16075 %}
16076
16077 ins_pipe(pipe_serial);
16078 %}
16079
16080 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16081 %{
16082 predicate(LockingMode == LM_LIGHTWEIGHT);
16083 match(Set cr (FastUnlock object box));
16084 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16085
16086 ins_cost(5 * INSN_COST);
16087 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16088
16089 ins_encode %{
16090 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16091 %}
16092
16093 ins_pipe(pipe_serial);
16094 %}
16095
16096 // ============================================================================
16097 // Safepoint Instructions
16098
16099 // TODO
16100 // provide a near and far version of this code
16101
16102 instruct safePoint(rFlagsReg cr, iRegP poll)
16103 %{
16104 match(SafePoint poll);
16105 effect(KILL cr);
16106
16107 format %{
16108 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16109 %}
16110 ins_encode %{
16111 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16112 %}
16113 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16114 %}
16115
16116
16117 // ============================================================================
16118 // Procedure Call/Return Instructions
16119
16120 // Call Java Static Instruction
16121
16122 instruct CallStaticJavaDirect(method meth)
16123 %{
16124 match(CallStaticJava);
16125
16126 effect(USE meth);
16127
16128 ins_cost(CALL_COST);
16129
16130 format %{ "call,static $meth \t// ==> " %}
16131
16132 ins_encode(aarch64_enc_java_static_call(meth),
16133 aarch64_enc_call_epilog);
16134
16135 ins_pipe(pipe_class_call);
16136 %}
16137
16138 // TO HERE
16139
16140 // Call Java Dynamic Instruction
16141 instruct CallDynamicJavaDirect(method meth)
16142 %{
16143 match(CallDynamicJava);
16144
16145 effect(USE meth);
16146
16147 ins_cost(CALL_COST);
16148
16149 format %{ "CALL,dynamic $meth \t// ==> " %}
16150
16151 ins_encode(aarch64_enc_java_dynamic_call(meth),
16152 aarch64_enc_call_epilog);
16153
16154 ins_pipe(pipe_class_call);
16155 %}
16156
16157 // Call Runtime Instruction
16158
16159 instruct CallRuntimeDirect(method meth)
16160 %{
16161 match(CallRuntime);
16162
16163 effect(USE meth);
16164
16165 ins_cost(CALL_COST);
16166
16167 format %{ "CALL, runtime $meth" %}
16168
16169 ins_encode( aarch64_enc_java_to_runtime(meth) );
16170
16171 ins_pipe(pipe_class_call);
16172 %}
16173
16174 // Call Runtime Instruction
16175
16176 instruct CallLeafDirect(method meth)
16177 %{
16178 match(CallLeaf);
16179
16180 effect(USE meth);
16181
16182 ins_cost(CALL_COST);
16183
16184 format %{ "CALL, runtime leaf $meth" %}
16185
16186 ins_encode( aarch64_enc_java_to_runtime(meth) );
16187
16188 ins_pipe(pipe_class_call);
16189 %}
16190
16191 // Call Runtime Instruction without safepoint and with vector arguments
16192 instruct CallLeafDirectVector(method meth)
16193 %{
16194 match(CallLeafVector);
16195
16196 effect(USE meth);
16197
16198 ins_cost(CALL_COST);
16199
16200 format %{ "CALL, runtime leaf vector $meth" %}
16201
16202 ins_encode(aarch64_enc_java_to_runtime(meth));
16203
16204 ins_pipe(pipe_class_call);
16205 %}
16206
16207 // Call Runtime Instruction
16208
16209 instruct CallLeafNoFPDirect(method meth)
16210 %{
16211 match(CallLeafNoFP);
16212
16213 effect(USE meth);
16214
16215 ins_cost(CALL_COST);
16216
16217 format %{ "CALL, runtime leaf nofp $meth" %}
16218
16219 ins_encode( aarch64_enc_java_to_runtime(meth) );
16220
16221 ins_pipe(pipe_class_call);
16222 %}
16223
16224 // Tail Call; Jump from runtime stub to Java code.
16225 // Also known as an 'interprocedural jump'.
16226 // Target of jump will eventually return to caller.
16227 // TailJump below removes the return address.
16228 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16229 // emitted just above the TailCall which has reset rfp to the caller state.
16230 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16231 %{
16232 match(TailCall jump_target method_ptr);
16233
16234 ins_cost(CALL_COST);
16235
16236 format %{ "br $jump_target\t# $method_ptr holds method" %}
16237
16238 ins_encode(aarch64_enc_tail_call(jump_target));
16239
16240 ins_pipe(pipe_class_call);
16241 %}
16242
16243 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16244 %{
16245 match(TailJump jump_target ex_oop);
16246
16247 ins_cost(CALL_COST);
16248
16249 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16250
16251 ins_encode(aarch64_enc_tail_jmp(jump_target));
16252
16253 ins_pipe(pipe_class_call);
16254 %}
16255
16256 // Forward exception.
16257 instruct ForwardExceptionjmp()
16258 %{
16259 match(ForwardException);
16260 ins_cost(CALL_COST);
16261
16262 format %{ "b forward_exception_stub" %}
16263 ins_encode %{
16264 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16265 %}
16266 ins_pipe(pipe_class_call);
16267 %}
16268
16269 // Create exception oop: created by stack-crawling runtime code.
16270 // Created exception is now available to this handler, and is setup
16271 // just prior to jumping to this handler. No code emitted.
16272 // TODO check
16273 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16274 instruct CreateException(iRegP_R0 ex_oop)
16275 %{
16276 match(Set ex_oop (CreateEx));
16277
16278 format %{ " -- \t// exception oop; no code emitted" %}
16279
16280 size(0);
16281
16282 ins_encode( /*empty*/ );
16283
16284 ins_pipe(pipe_class_empty);
16285 %}
16286
16287 // Rethrow exception: The exception oop will come in the first
16288 // argument position. Then JUMP (not call) to the rethrow stub code.
16289 instruct RethrowException() %{
16290 match(Rethrow);
16291 ins_cost(CALL_COST);
16292
16293 format %{ "b rethrow_stub" %}
16294
16295 ins_encode( aarch64_enc_rethrow() );
16296
16297 ins_pipe(pipe_class_call);
16298 %}
16299
16300
16301 // Return Instruction
16302 // epilog node loads ret address into lr as part of frame pop
16303 instruct Ret()
16304 %{
16305 match(Return);
16306
16307 format %{ "ret\t// return register" %}
16308
16309 ins_encode( aarch64_enc_ret() );
16310
16311 ins_pipe(pipe_branch);
16312 %}
16313
16314 // Die now.
16315 instruct ShouldNotReachHere() %{
16316 match(Halt);
16317
16318 ins_cost(CALL_COST);
16319 format %{ "ShouldNotReachHere" %}
16320
16321 ins_encode %{
16322 if (is_reachable()) {
16323 __ stop(_halt_reason);
16324 }
16325 %}
16326
16327 ins_pipe(pipe_class_default);
16328 %}
16329
16330 // ============================================================================
16331 // Partial Subtype Check
16332 //
16333 // superklass array for an instance of the superklass. Set a hidden
16334 // internal cache on a hit (cache is checked with exposed code in
16335 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16336 // encoding ALSO sets flags.
16337
16338 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16339 %{
16340 match(Set result (PartialSubtypeCheck sub super));
16341 predicate(!UseSecondarySupersTable);
16342 effect(KILL cr, KILL temp);
16343
16344 ins_cost(20 * INSN_COST); // slightly larger than the next version
16345 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16346
16347 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16348
16349 opcode(0x1); // Force zero of result reg on hit
16350
16351 ins_pipe(pipe_class_memory);
16352 %}
16353
16354 // Two versions of partialSubtypeCheck, both used when we need to
16355 // search for a super class in the secondary supers array. The first
16356 // is used when we don't know _a priori_ the class being searched
16357 // for. The second, far more common, is used when we do know: this is
16358 // used for instanceof, checkcast, and any case where C2 can determine
16359 // it by constant propagation.
16360
16361 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16362 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16363 rFlagsReg cr)
16364 %{
16365 match(Set result (PartialSubtypeCheck sub super));
16366 predicate(UseSecondarySupersTable);
16367 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16368
16369 ins_cost(10 * INSN_COST); // slightly larger than the next version
16370 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16371
16372 ins_encode %{
16373 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16374 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16375 $vtemp$$FloatRegister,
16376 $result$$Register, /*L_success*/nullptr);
16377 %}
16378
16379 ins_pipe(pipe_class_memory);
16380 %}
16381
16382 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16383 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16384 rFlagsReg cr)
16385 %{
16386 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16387 predicate(UseSecondarySupersTable);
16388 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16389
16390 ins_cost(5 * INSN_COST); // smaller than the next version
16391 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16392
16393 ins_encode %{
16394 bool success = false;
16395 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16396 if (InlineSecondarySupersTest) {
16397 success =
16398 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16399 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16400 $vtemp$$FloatRegister,
16401 $result$$Register,
16402 super_klass_slot);
16403 } else {
16404 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16405 success = (call != nullptr);
16406 }
16407 if (!success) {
16408 ciEnv::current()->record_failure("CodeCache is full");
16409 return;
16410 }
16411 %}
16412
16413 ins_pipe(pipe_class_memory);
16414 %}
16415
16416 // Intrisics for String.compareTo()
16417
16418 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16419 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16420 %{
16421 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16422 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16423 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16424
16425 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16426 ins_encode %{
16427 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16428 __ string_compare($str1$$Register, $str2$$Register,
16429 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16430 $tmp1$$Register, $tmp2$$Register,
16431 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16432 %}
16433 ins_pipe(pipe_class_memory);
16434 %}
16435
16436 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16437 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16438 %{
16439 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16440 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16441 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16442
16443 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16444 ins_encode %{
16445 __ string_compare($str1$$Register, $str2$$Register,
16446 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16447 $tmp1$$Register, $tmp2$$Register,
16448 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16449 %}
16450 ins_pipe(pipe_class_memory);
16451 %}
16452
16453 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16454 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16455 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16456 %{
16457 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16458 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16459 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16460 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16461
16462 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16463 ins_encode %{
16464 __ string_compare($str1$$Register, $str2$$Register,
16465 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16466 $tmp1$$Register, $tmp2$$Register,
16467 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16468 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16469 %}
16470 ins_pipe(pipe_class_memory);
16471 %}
16472
16473 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16474 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16475 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16476 %{
16477 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16478 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16479 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16480 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16481
16482 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16483 ins_encode %{
16484 __ string_compare($str1$$Register, $str2$$Register,
16485 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16486 $tmp1$$Register, $tmp2$$Register,
16487 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16488 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16489 %}
16490 ins_pipe(pipe_class_memory);
16491 %}
16492
16493 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16494 // these string_compare variants as NEON register type for convenience so that the prototype of
16495 // string_compare can be shared with all variants.
16496
16497 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16498 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16499 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16500 pRegGov_P1 pgtmp2, rFlagsReg cr)
16501 %{
16502 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16503 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16504 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16505 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16506
16507 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16508 ins_encode %{
16509 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16510 __ string_compare($str1$$Register, $str2$$Register,
16511 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16512 $tmp1$$Register, $tmp2$$Register,
16513 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16514 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16515 StrIntrinsicNode::LL);
16516 %}
16517 ins_pipe(pipe_class_memory);
16518 %}
16519
16520 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16521 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16522 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16523 pRegGov_P1 pgtmp2, rFlagsReg cr)
16524 %{
16525 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16526 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16527 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16528 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16529
16530 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16531 ins_encode %{
16532 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16533 __ string_compare($str1$$Register, $str2$$Register,
16534 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16535 $tmp1$$Register, $tmp2$$Register,
16536 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16537 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16538 StrIntrinsicNode::LU);
16539 %}
16540 ins_pipe(pipe_class_memory);
16541 %}
16542
16543 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16544 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16545 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16546 pRegGov_P1 pgtmp2, rFlagsReg cr)
16547 %{
16548 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16549 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16550 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16551 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16552
16553 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16554 ins_encode %{
16555 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16556 __ string_compare($str1$$Register, $str2$$Register,
16557 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16558 $tmp1$$Register, $tmp2$$Register,
16559 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16560 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16561 StrIntrinsicNode::UL);
16562 %}
16563 ins_pipe(pipe_class_memory);
16564 %}
16565
16566 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16567 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16568 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16569 pRegGov_P1 pgtmp2, rFlagsReg cr)
16570 %{
16571 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16572 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16573 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16574 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16575
16576 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16577 ins_encode %{
16578 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16579 __ string_compare($str1$$Register, $str2$$Register,
16580 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16581 $tmp1$$Register, $tmp2$$Register,
16582 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16583 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16584 StrIntrinsicNode::UU);
16585 %}
16586 ins_pipe(pipe_class_memory);
16587 %}
16588
16589 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16590 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16591 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16592 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16593 %{
16594 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16595 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16596 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16597 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16598 TEMP vtmp0, TEMP vtmp1, KILL cr);
16599 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16600 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16601
16602 ins_encode %{
16603 __ string_indexof($str1$$Register, $str2$$Register,
16604 $cnt1$$Register, $cnt2$$Register,
16605 $tmp1$$Register, $tmp2$$Register,
16606 $tmp3$$Register, $tmp4$$Register,
16607 $tmp5$$Register, $tmp6$$Register,
16608 -1, $result$$Register, StrIntrinsicNode::UU);
16609 %}
16610 ins_pipe(pipe_class_memory);
16611 %}
16612
16613 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16614 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16615 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16616 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16617 %{
16618 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16619 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16620 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16621 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16622 TEMP vtmp0, TEMP vtmp1, KILL cr);
16623 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16624 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16625
16626 ins_encode %{
16627 __ string_indexof($str1$$Register, $str2$$Register,
16628 $cnt1$$Register, $cnt2$$Register,
16629 $tmp1$$Register, $tmp2$$Register,
16630 $tmp3$$Register, $tmp4$$Register,
16631 $tmp5$$Register, $tmp6$$Register,
16632 -1, $result$$Register, StrIntrinsicNode::LL);
16633 %}
16634 ins_pipe(pipe_class_memory);
16635 %}
16636
16637 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16638 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16639 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16640 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16641 %{
16642 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16643 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16644 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16645 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16646 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16647 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16648 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16649
16650 ins_encode %{
16651 __ string_indexof($str1$$Register, $str2$$Register,
16652 $cnt1$$Register, $cnt2$$Register,
16653 $tmp1$$Register, $tmp2$$Register,
16654 $tmp3$$Register, $tmp4$$Register,
16655 $tmp5$$Register, $tmp6$$Register,
16656 -1, $result$$Register, StrIntrinsicNode::UL);
16657 %}
16658 ins_pipe(pipe_class_memory);
16659 %}
16660
16661 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16662 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16663 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16664 %{
16665 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16666 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16667 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16668 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16669 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16670 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16671
16672 ins_encode %{
16673 int icnt2 = (int)$int_cnt2$$constant;
16674 __ string_indexof($str1$$Register, $str2$$Register,
16675 $cnt1$$Register, zr,
16676 $tmp1$$Register, $tmp2$$Register,
16677 $tmp3$$Register, $tmp4$$Register, zr, zr,
16678 icnt2, $result$$Register, StrIntrinsicNode::UU);
16679 %}
16680 ins_pipe(pipe_class_memory);
16681 %}
16682
16683 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16684 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16685 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16686 %{
16687 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16688 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16689 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16690 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16691 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16692 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16693
16694 ins_encode %{
16695 int icnt2 = (int)$int_cnt2$$constant;
16696 __ string_indexof($str1$$Register, $str2$$Register,
16697 $cnt1$$Register, zr,
16698 $tmp1$$Register, $tmp2$$Register,
16699 $tmp3$$Register, $tmp4$$Register, zr, zr,
16700 icnt2, $result$$Register, StrIntrinsicNode::LL);
16701 %}
16702 ins_pipe(pipe_class_memory);
16703 %}
16704
16705 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16706 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16707 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16708 %{
16709 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16710 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16711 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16712 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16713 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16714 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16715
16716 ins_encode %{
16717 int icnt2 = (int)$int_cnt2$$constant;
16718 __ string_indexof($str1$$Register, $str2$$Register,
16719 $cnt1$$Register, zr,
16720 $tmp1$$Register, $tmp2$$Register,
16721 $tmp3$$Register, $tmp4$$Register, zr, zr,
16722 icnt2, $result$$Register, StrIntrinsicNode::UL);
16723 %}
16724 ins_pipe(pipe_class_memory);
16725 %}
16726
16727 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16728 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16729 iRegINoSp tmp3, rFlagsReg cr)
16730 %{
16731 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16732 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16733 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16734 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16735
16736 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16737
16738 ins_encode %{
16739 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16740 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16741 $tmp3$$Register);
16742 %}
16743 ins_pipe(pipe_class_memory);
16744 %}
16745
16746 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16747 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16748 iRegINoSp tmp3, rFlagsReg cr)
16749 %{
16750 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16751 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16752 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16753 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16754
16755 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16756
16757 ins_encode %{
16758 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16759 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16760 $tmp3$$Register);
16761 %}
16762 ins_pipe(pipe_class_memory);
16763 %}
16764
16765 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16766 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16767 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16768 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16769 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16770 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16771 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16772 ins_encode %{
16773 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16774 $result$$Register, $ztmp1$$FloatRegister,
16775 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16776 $ptmp$$PRegister, true /* isL */);
16777 %}
16778 ins_pipe(pipe_class_memory);
16779 %}
16780
16781 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16782 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16783 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16784 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16785 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16786 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16787 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16788 ins_encode %{
16789 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16790 $result$$Register, $ztmp1$$FloatRegister,
16791 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16792 $ptmp$$PRegister, false /* isL */);
16793 %}
16794 ins_pipe(pipe_class_memory);
16795 %}
16796
16797 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16798 iRegI_R0 result, rFlagsReg cr)
16799 %{
16800 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16801 match(Set result (StrEquals (Binary str1 str2) cnt));
16802 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16803
16804 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16805 ins_encode %{
16806 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16807 __ string_equals($str1$$Register, $str2$$Register,
16808 $result$$Register, $cnt$$Register);
16809 %}
16810 ins_pipe(pipe_class_memory);
16811 %}
16812
16813 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16814 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16815 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16816 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16817 iRegP_R10 tmp, rFlagsReg cr)
16818 %{
16819 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16820 match(Set result (AryEq ary1 ary2));
16821 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16822 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16823 TEMP vtmp6, TEMP vtmp7, KILL cr);
16824
16825 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16826 ins_encode %{
16827 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16828 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16829 $result$$Register, $tmp$$Register, 1);
16830 if (tpc == nullptr) {
16831 ciEnv::current()->record_failure("CodeCache is full");
16832 return;
16833 }
16834 %}
16835 ins_pipe(pipe_class_memory);
16836 %}
16837
16838 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16839 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16840 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16841 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16842 iRegP_R10 tmp, rFlagsReg cr)
16843 %{
16844 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16845 match(Set result (AryEq ary1 ary2));
16846 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16847 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16848 TEMP vtmp6, TEMP vtmp7, KILL cr);
16849
16850 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16851 ins_encode %{
16852 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16853 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16854 $result$$Register, $tmp$$Register, 2);
16855 if (tpc == nullptr) {
16856 ciEnv::current()->record_failure("CodeCache is full");
16857 return;
16858 }
16859 %}
16860 ins_pipe(pipe_class_memory);
16861 %}
16862
16863 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16864 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16865 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16866 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16867 %{
16868 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16869 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16870 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16871
16872 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16873 ins_encode %{
16874 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16875 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16876 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16877 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16878 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16879 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16880 (BasicType)$basic_type$$constant);
16881 if (tpc == nullptr) {
16882 ciEnv::current()->record_failure("CodeCache is full");
16883 return;
16884 }
16885 %}
16886 ins_pipe(pipe_class_memory);
16887 %}
16888
16889 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16890 %{
16891 match(Set result (CountPositives ary1 len));
16892 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16893 format %{ "count positives byte[] $ary1,$len -> $result" %}
16894 ins_encode %{
16895 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16896 if (tpc == nullptr) {
16897 ciEnv::current()->record_failure("CodeCache is full");
16898 return;
16899 }
16900 %}
16901 ins_pipe( pipe_slow );
16902 %}
16903
16904 // fast char[] to byte[] compression
16905 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16906 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16907 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16908 iRegI_R0 result, rFlagsReg cr)
16909 %{
16910 match(Set result (StrCompressedCopy src (Binary dst len)));
16911 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16912 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16913
16914 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16915 ins_encode %{
16916 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16917 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16918 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16919 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16920 %}
16921 ins_pipe(pipe_slow);
16922 %}
16923
16924 // fast byte[] to char[] inflation
16925 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16926 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16927 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16928 %{
16929 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16930 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16931 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16932 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16933
16934 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16935 ins_encode %{
16936 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16937 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16938 $vtmp2$$FloatRegister, $tmp$$Register);
16939 if (tpc == nullptr) {
16940 ciEnv::current()->record_failure("CodeCache is full");
16941 return;
16942 }
16943 %}
16944 ins_pipe(pipe_class_memory);
16945 %}
16946
16947 // encode char[] to byte[] in ISO_8859_1
16948 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16949 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16950 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16951 iRegI_R0 result, rFlagsReg cr)
16952 %{
16953 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16954 match(Set result (EncodeISOArray src (Binary dst len)));
16955 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16956 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16957
16958 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16959 ins_encode %{
16960 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16961 $result$$Register, false,
16962 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16963 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16964 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16965 %}
16966 ins_pipe(pipe_class_memory);
16967 %}
16968
16969 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16970 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16971 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16972 iRegI_R0 result, rFlagsReg cr)
16973 %{
16974 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16975 match(Set result (EncodeISOArray src (Binary dst len)));
16976 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16977 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16978
16979 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16980 ins_encode %{
16981 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16982 $result$$Register, true,
16983 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16984 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16985 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16986 %}
16987 ins_pipe(pipe_class_memory);
16988 %}
16989
16990 //----------------------------- CompressBits/ExpandBits ------------------------
16991
16992 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16993 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16994 match(Set dst (CompressBits src mask));
16995 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16996 format %{ "mov $tsrc, $src\n\t"
16997 "mov $tmask, $mask\n\t"
16998 "bext $tdst, $tsrc, $tmask\n\t"
16999 "mov $dst, $tdst"
17000 %}
17001 ins_encode %{
17002 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17003 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17004 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17005 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17006 %}
17007 ins_pipe(pipe_slow);
17008 %}
17009
17010 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17011 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17012 match(Set dst (CompressBits (LoadI mem) mask));
17013 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17014 format %{ "ldrs $tsrc, $mem\n\t"
17015 "ldrs $tmask, $mask\n\t"
17016 "bext $tdst, $tsrc, $tmask\n\t"
17017 "mov $dst, $tdst"
17018 %}
17019 ins_encode %{
17020 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17021 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17022 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17023 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17024 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17025 %}
17026 ins_pipe(pipe_slow);
17027 %}
17028
17029 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17030 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17031 match(Set dst (CompressBits src mask));
17032 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17033 format %{ "mov $tsrc, $src\n\t"
17034 "mov $tmask, $mask\n\t"
17035 "bext $tdst, $tsrc, $tmask\n\t"
17036 "mov $dst, $tdst"
17037 %}
17038 ins_encode %{
17039 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17040 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17041 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17042 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17043 %}
17044 ins_pipe(pipe_slow);
17045 %}
17046
17047 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17048 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17049 match(Set dst (CompressBits (LoadL mem) mask));
17050 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17051 format %{ "ldrd $tsrc, $mem\n\t"
17052 "ldrd $tmask, $mask\n\t"
17053 "bext $tdst, $tsrc, $tmask\n\t"
17054 "mov $dst, $tdst"
17055 %}
17056 ins_encode %{
17057 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17058 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17059 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17060 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17061 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17062 %}
17063 ins_pipe(pipe_slow);
17064 %}
17065
17066 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17067 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17068 match(Set dst (ExpandBits src mask));
17069 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17070 format %{ "mov $tsrc, $src\n\t"
17071 "mov $tmask, $mask\n\t"
17072 "bdep $tdst, $tsrc, $tmask\n\t"
17073 "mov $dst, $tdst"
17074 %}
17075 ins_encode %{
17076 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17077 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17078 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17079 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17080 %}
17081 ins_pipe(pipe_slow);
17082 %}
17083
17084 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17085 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17086 match(Set dst (ExpandBits (LoadI mem) mask));
17087 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17088 format %{ "ldrs $tsrc, $mem\n\t"
17089 "ldrs $tmask, $mask\n\t"
17090 "bdep $tdst, $tsrc, $tmask\n\t"
17091 "mov $dst, $tdst"
17092 %}
17093 ins_encode %{
17094 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17095 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17096 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17097 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17098 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17099 %}
17100 ins_pipe(pipe_slow);
17101 %}
17102
17103 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17104 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17105 match(Set dst (ExpandBits src mask));
17106 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17107 format %{ "mov $tsrc, $src\n\t"
17108 "mov $tmask, $mask\n\t"
17109 "bdep $tdst, $tsrc, $tmask\n\t"
17110 "mov $dst, $tdst"
17111 %}
17112 ins_encode %{
17113 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17114 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17115 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17116 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17117 %}
17118 ins_pipe(pipe_slow);
17119 %}
17120
17121
17122 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17123 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17124 match(Set dst (ExpandBits (LoadL mem) mask));
17125 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17126 format %{ "ldrd $tsrc, $mem\n\t"
17127 "ldrd $tmask, $mask\n\t"
17128 "bdep $tdst, $tsrc, $tmask\n\t"
17129 "mov $dst, $tdst"
17130 %}
17131 ins_encode %{
17132 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17133 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17134 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17135 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17136 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17137 %}
17138 ins_pipe(pipe_slow);
17139 %}
17140
17141 // ============================================================================
17142 // This name is KNOWN by the ADLC and cannot be changed.
17143 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17144 // for this guy.
17145 instruct tlsLoadP(thread_RegP dst)
17146 %{
17147 match(Set dst (ThreadLocal));
17148
17149 ins_cost(0);
17150
17151 format %{ " -- \t// $dst=Thread::current(), empty" %}
17152
17153 size(0);
17154
17155 ins_encode( /*empty*/ );
17156
17157 ins_pipe(pipe_class_empty);
17158 %}
17159
17160 //----------PEEPHOLE RULES-----------------------------------------------------
17161 // These must follow all instruction definitions as they use the names
17162 // defined in the instructions definitions.
17163 //
17164 // peepmatch ( root_instr_name [preceding_instruction]* );
17165 //
17166 // peepconstraint %{
17167 // (instruction_number.operand_name relational_op instruction_number.operand_name
17168 // [, ...] );
17169 // // instruction numbers are zero-based using left to right order in peepmatch
17170 //
17171 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17172 // // provide an instruction_number.operand_name for each operand that appears
17173 // // in the replacement instruction's match rule
17174 //
17175 // ---------VM FLAGS---------------------------------------------------------
17176 //
17177 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17178 //
17179 // Each peephole rule is given an identifying number starting with zero and
17180 // increasing by one in the order seen by the parser. An individual peephole
17181 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17182 // on the command-line.
17183 //
17184 // ---------CURRENT LIMITATIONS----------------------------------------------
17185 //
17186 // Only match adjacent instructions in same basic block
17187 // Only equality constraints
17188 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17189 // Only one replacement instruction
17190 //
17191 // ---------EXAMPLE----------------------------------------------------------
17192 //
17193 // // pertinent parts of existing instructions in architecture description
17194 // instruct movI(iRegINoSp dst, iRegI src)
17195 // %{
17196 // match(Set dst (CopyI src));
17197 // %}
17198 //
17199 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17200 // %{
17201 // match(Set dst (AddI dst src));
17202 // effect(KILL cr);
17203 // %}
17204 //
17205 // // Change (inc mov) to lea
17206 // peephole %{
17207 // // increment preceded by register-register move
17208 // peepmatch ( incI_iReg movI );
17209 // // require that the destination register of the increment
17210 // // match the destination register of the move
17211 // peepconstraint ( 0.dst == 1.dst );
17212 // // construct a replacement instruction that sets
17213 // // the destination to ( move's source register + one )
17214 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17215 // %}
17216 //
17217
17218 // Implementation no longer uses movX instructions since
17219 // machine-independent system no longer uses CopyX nodes.
17220 //
17221 // peephole
17222 // %{
17223 // peepmatch (incI_iReg movI);
17224 // peepconstraint (0.dst == 1.dst);
17225 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17226 // %}
17227
17228 // peephole
17229 // %{
17230 // peepmatch (decI_iReg movI);
17231 // peepconstraint (0.dst == 1.dst);
17232 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17233 // %}
17234
17235 // peephole
17236 // %{
17237 // peepmatch (addI_iReg_imm movI);
17238 // peepconstraint (0.dst == 1.dst);
17239 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17240 // %}
17241
17242 // peephole
17243 // %{
17244 // peepmatch (incL_iReg movL);
17245 // peepconstraint (0.dst == 1.dst);
17246 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17247 // %}
17248
17249 // peephole
17250 // %{
17251 // peepmatch (decL_iReg movL);
17252 // peepconstraint (0.dst == 1.dst);
17253 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17254 // %}
17255
17256 // peephole
17257 // %{
17258 // peepmatch (addL_iReg_imm movL);
17259 // peepconstraint (0.dst == 1.dst);
17260 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17261 // %}
17262
17263 // peephole
17264 // %{
17265 // peepmatch (addP_iReg_imm movP);
17266 // peepconstraint (0.dst == 1.dst);
17267 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17268 // %}
17269
17270 // // Change load of spilled value to only a spill
17271 // instruct storeI(memory mem, iRegI src)
17272 // %{
17273 // match(Set mem (StoreI mem src));
17274 // %}
17275 //
17276 // instruct loadI(iRegINoSp dst, memory mem)
17277 // %{
17278 // match(Set dst (LoadI mem));
17279 // %}
17280 //
17281
17282 //----------SMARTSPILL RULES---------------------------------------------------
17283 // These must follow all instruction definitions as they use the names
17284 // defined in the instructions definitions.
17285
17286 // Local Variables:
17287 // mode: c++
17288 // End: