1 //
2 // Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all float registers
698 reg_class float_reg(
699 V0,
700 V1,
701 V2,
702 V3,
703 V4,
704 V5,
705 V6,
706 V7,
707 V8,
708 V9,
709 V10,
710 V11,
711 V12,
712 V13,
713 V14,
714 V15,
715 V16,
716 V17,
717 V18,
718 V19,
719 V20,
720 V21,
721 V22,
722 V23,
723 V24,
724 V25,
725 V26,
726 V27,
727 V28,
728 V29,
729 V30,
730 V31
731 );
732
733 // Double precision float registers have virtual `high halves' that
734 // are needed by the allocator.
735 // Class for all double registers
736 reg_class double_reg(
737 V0, V0_H,
738 V1, V1_H,
739 V2, V2_H,
740 V3, V3_H,
741 V4, V4_H,
742 V5, V5_H,
743 V6, V6_H,
744 V7, V7_H,
745 V8, V8_H,
746 V9, V9_H,
747 V10, V10_H,
748 V11, V11_H,
749 V12, V12_H,
750 V13, V13_H,
751 V14, V14_H,
752 V15, V15_H,
753 V16, V16_H,
754 V17, V17_H,
755 V18, V18_H,
756 V19, V19_H,
757 V20, V20_H,
758 V21, V21_H,
759 V22, V22_H,
760 V23, V23_H,
761 V24, V24_H,
762 V25, V25_H,
763 V26, V26_H,
764 V27, V27_H,
765 V28, V28_H,
766 V29, V29_H,
767 V30, V30_H,
768 V31, V31_H
769 );
770
771 // Class for all SVE vector registers.
772 reg_class vectora_reg (
773 V0, V0_H, V0_J, V0_K,
774 V1, V1_H, V1_J, V1_K,
775 V2, V2_H, V2_J, V2_K,
776 V3, V3_H, V3_J, V3_K,
777 V4, V4_H, V4_J, V4_K,
778 V5, V5_H, V5_J, V5_K,
779 V6, V6_H, V6_J, V6_K,
780 V7, V7_H, V7_J, V7_K,
781 V8, V8_H, V8_J, V8_K,
782 V9, V9_H, V9_J, V9_K,
783 V10, V10_H, V10_J, V10_K,
784 V11, V11_H, V11_J, V11_K,
785 V12, V12_H, V12_J, V12_K,
786 V13, V13_H, V13_J, V13_K,
787 V14, V14_H, V14_J, V14_K,
788 V15, V15_H, V15_J, V15_K,
789 V16, V16_H, V16_J, V16_K,
790 V17, V17_H, V17_J, V17_K,
791 V18, V18_H, V18_J, V18_K,
792 V19, V19_H, V19_J, V19_K,
793 V20, V20_H, V20_J, V20_K,
794 V21, V21_H, V21_J, V21_K,
795 V22, V22_H, V22_J, V22_K,
796 V23, V23_H, V23_J, V23_K,
797 V24, V24_H, V24_J, V24_K,
798 V25, V25_H, V25_J, V25_K,
799 V26, V26_H, V26_J, V26_K,
800 V27, V27_H, V27_J, V27_K,
801 V28, V28_H, V28_J, V28_K,
802 V29, V29_H, V29_J, V29_K,
803 V30, V30_H, V30_J, V30_K,
804 V31, V31_H, V31_J, V31_K,
805 );
806
807 // Class for all 64bit vector registers
808 reg_class vectord_reg(
809 V0, V0_H,
810 V1, V1_H,
811 V2, V2_H,
812 V3, V3_H,
813 V4, V4_H,
814 V5, V5_H,
815 V6, V6_H,
816 V7, V7_H,
817 V8, V8_H,
818 V9, V9_H,
819 V10, V10_H,
820 V11, V11_H,
821 V12, V12_H,
822 V13, V13_H,
823 V14, V14_H,
824 V15, V15_H,
825 V16, V16_H,
826 V17, V17_H,
827 V18, V18_H,
828 V19, V19_H,
829 V20, V20_H,
830 V21, V21_H,
831 V22, V22_H,
832 V23, V23_H,
833 V24, V24_H,
834 V25, V25_H,
835 V26, V26_H,
836 V27, V27_H,
837 V28, V28_H,
838 V29, V29_H,
839 V30, V30_H,
840 V31, V31_H
841 );
842
843 // Class for all 128bit vector registers
844 reg_class vectorx_reg(
845 V0, V0_H, V0_J, V0_K,
846 V1, V1_H, V1_J, V1_K,
847 V2, V2_H, V2_J, V2_K,
848 V3, V3_H, V3_J, V3_K,
849 V4, V4_H, V4_J, V4_K,
850 V5, V5_H, V5_J, V5_K,
851 V6, V6_H, V6_J, V6_K,
852 V7, V7_H, V7_J, V7_K,
853 V8, V8_H, V8_J, V8_K,
854 V9, V9_H, V9_J, V9_K,
855 V10, V10_H, V10_J, V10_K,
856 V11, V11_H, V11_J, V11_K,
857 V12, V12_H, V12_J, V12_K,
858 V13, V13_H, V13_J, V13_K,
859 V14, V14_H, V14_J, V14_K,
860 V15, V15_H, V15_J, V15_K,
861 V16, V16_H, V16_J, V16_K,
862 V17, V17_H, V17_J, V17_K,
863 V18, V18_H, V18_J, V18_K,
864 V19, V19_H, V19_J, V19_K,
865 V20, V20_H, V20_J, V20_K,
866 V21, V21_H, V21_J, V21_K,
867 V22, V22_H, V22_J, V22_K,
868 V23, V23_H, V23_J, V23_K,
869 V24, V24_H, V24_J, V24_K,
870 V25, V25_H, V25_J, V25_K,
871 V26, V26_H, V26_J, V26_K,
872 V27, V27_H, V27_J, V27_K,
873 V28, V28_H, V28_J, V28_K,
874 V29, V29_H, V29_J, V29_K,
875 V30, V30_H, V30_J, V30_K,
876 V31, V31_H, V31_J, V31_K
877 );
878
879 // Class for 128 bit register v0
880 reg_class v0_reg(
881 V0, V0_H
882 );
883
884 // Class for 128 bit register v1
885 reg_class v1_reg(
886 V1, V1_H
887 );
888
889 // Class for 128 bit register v2
890 reg_class v2_reg(
891 V2, V2_H
892 );
893
894 // Class for 128 bit register v3
895 reg_class v3_reg(
896 V3, V3_H
897 );
898
899 // Class for 128 bit register v4
900 reg_class v4_reg(
901 V4, V4_H
902 );
903
904 // Class for 128 bit register v5
905 reg_class v5_reg(
906 V5, V5_H
907 );
908
909 // Class for 128 bit register v6
910 reg_class v6_reg(
911 V6, V6_H
912 );
913
914 // Class for 128 bit register v7
915 reg_class v7_reg(
916 V7, V7_H
917 );
918
919 // Class for 128 bit register v8
920 reg_class v8_reg(
921 V8, V8_H
922 );
923
924 // Class for 128 bit register v9
925 reg_class v9_reg(
926 V9, V9_H
927 );
928
929 // Class for 128 bit register v10
930 reg_class v10_reg(
931 V10, V10_H
932 );
933
934 // Class for 128 bit register v11
935 reg_class v11_reg(
936 V11, V11_H
937 );
938
939 // Class for 128 bit register v12
940 reg_class v12_reg(
941 V12, V12_H
942 );
943
944 // Class for 128 bit register v13
945 reg_class v13_reg(
946 V13, V13_H
947 );
948
949 // Class for 128 bit register v14
950 reg_class v14_reg(
951 V14, V14_H
952 );
953
954 // Class for 128 bit register v15
955 reg_class v15_reg(
956 V15, V15_H
957 );
958
959 // Class for 128 bit register v16
960 reg_class v16_reg(
961 V16, V16_H
962 );
963
964 // Class for 128 bit register v17
965 reg_class v17_reg(
966 V17, V17_H
967 );
968
969 // Class for 128 bit register v18
970 reg_class v18_reg(
971 V18, V18_H
972 );
973
974 // Class for 128 bit register v19
975 reg_class v19_reg(
976 V19, V19_H
977 );
978
979 // Class for 128 bit register v20
980 reg_class v20_reg(
981 V20, V20_H
982 );
983
984 // Class for 128 bit register v21
985 reg_class v21_reg(
986 V21, V21_H
987 );
988
989 // Class for 128 bit register v22
990 reg_class v22_reg(
991 V22, V22_H
992 );
993
994 // Class for 128 bit register v23
995 reg_class v23_reg(
996 V23, V23_H
997 );
998
999 // Class for 128 bit register v24
1000 reg_class v24_reg(
1001 V24, V24_H
1002 );
1003
1004 // Class for 128 bit register v25
1005 reg_class v25_reg(
1006 V25, V25_H
1007 );
1008
1009 // Class for 128 bit register v26
1010 reg_class v26_reg(
1011 V26, V26_H
1012 );
1013
1014 // Class for 128 bit register v27
1015 reg_class v27_reg(
1016 V27, V27_H
1017 );
1018
1019 // Class for 128 bit register v28
1020 reg_class v28_reg(
1021 V28, V28_H
1022 );
1023
1024 // Class for 128 bit register v29
1025 reg_class v29_reg(
1026 V29, V29_H
1027 );
1028
1029 // Class for 128 bit register v30
1030 reg_class v30_reg(
1031 V30, V30_H
1032 );
1033
1034 // Class for 128 bit register v31
1035 reg_class v31_reg(
1036 V31, V31_H
1037 );
1038
1039 // Class for all SVE predicate registers.
1040 reg_class pr_reg (
1041 P0,
1042 P1,
1043 P2,
1044 P3,
1045 P4,
1046 P5,
1047 P6,
1048 // P7, non-allocatable, preserved with all elements preset to TRUE.
1049 P8,
1050 P9,
1051 P10,
1052 P11,
1053 P12,
1054 P13,
1055 P14,
1056 P15
1057 );
1058
1059 // Class for SVE governing predicate registers, which are used
1060 // to determine the active elements of a predicated instruction.
1061 reg_class gov_pr (
1062 P0,
1063 P1,
1064 P2,
1065 P3,
1066 P4,
1067 P5,
1068 P6,
1069 // P7, non-allocatable, preserved with all elements preset to TRUE.
1070 );
1071
1072 reg_class p0_reg(P0);
1073 reg_class p1_reg(P1);
1074
1075 // Singleton class for condition codes
1076 reg_class int_flags(RFLAGS);
1077
1078 %}
1079
1080 //----------DEFINITION BLOCK---------------------------------------------------
1081 // Define name --> value mappings to inform the ADLC of an integer valued name
1082 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1083 // Format:
1084 // int_def <name> ( <int_value>, <expression>);
1085 // Generated Code in ad_<arch>.hpp
1086 // #define <name> (<expression>)
1087 // // value == <int_value>
1088 // Generated code in ad_<arch>.cpp adlc_verification()
1089 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1090 //
1091
1092 // we follow the ppc-aix port in using a simple cost model which ranks
1093 // register operations as cheap, memory ops as more expensive and
1094 // branches as most expensive. the first two have a low as well as a
1095 // normal cost. huge cost appears to be a way of saying don't do
1096 // something
1097
1098 definitions %{
1099 // The default cost (of a register move instruction).
1100 int_def INSN_COST ( 100, 100);
1101 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1102 int_def CALL_COST ( 200, 2 * INSN_COST);
1103 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1104 %}
1105
1106
1107 //----------SOURCE BLOCK-------------------------------------------------------
1108 // This is a block of C++ code which provides values, functions, and
1109 // definitions necessary in the rest of the architecture description
1110
1111 source_hpp %{
1112
1113 #include "asm/macroAssembler.hpp"
1114 #include "gc/shared/barrierSetAssembler.hpp"
1115 #include "gc/shared/cardTable.hpp"
1116 #include "gc/shared/cardTableBarrierSet.hpp"
1117 #include "gc/shared/collectedHeap.hpp"
1118 #include "opto/addnode.hpp"
1119 #include "opto/convertnode.hpp"
1120 #include "runtime/objectMonitor.hpp"
1121
1122 extern RegMask _ANY_REG32_mask;
1123 extern RegMask _ANY_REG_mask;
1124 extern RegMask _PTR_REG_mask;
1125 extern RegMask _NO_SPECIAL_REG32_mask;
1126 extern RegMask _NO_SPECIAL_REG_mask;
1127 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1128
1129 class CallStubImpl {
1130
1131 //--------------------------------------------------------------
1132 //---< Used for optimization in Compile::shorten_branches >---
1133 //--------------------------------------------------------------
1134
1135 public:
1136 // Size of call trampoline stub.
1137 static uint size_call_trampoline() {
1138 return 0; // no call trampolines on this platform
1139 }
1140
1141 // number of relocations needed by a call trampoline stub
1142 static uint reloc_call_trampoline() {
1143 return 0; // no call trampolines on this platform
1144 }
1145 };
1146
1147 class HandlerImpl {
1148
1149 public:
1150
1151 static int emit_exception_handler(CodeBuffer &cbuf);
1152 static int emit_deopt_handler(CodeBuffer& cbuf);
1153
1154 static uint size_exception_handler() {
1155 return MacroAssembler::far_codestub_branch_size();
1156 }
1157
1158 static uint size_deopt_handler() {
1159 // count one adr and one far branch instruction
1160 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1161 }
1162 };
1163
1164 class Node::PD {
1165 public:
1166 enum NodeFlags {
1167 _last_flag = Node::_last_flag
1168 };
1169 };
1170
1171 bool is_CAS(int opcode, bool maybe_volatile);
1172
1173 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1174
1175 bool unnecessary_acquire(const Node *barrier);
1176 bool needs_acquiring_load(const Node *load);
1177
1178 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1179
1180 bool unnecessary_release(const Node *barrier);
1181 bool unnecessary_volatile(const Node *barrier);
1182 bool needs_releasing_store(const Node *store);
1183
1184 // predicate controlling translation of CompareAndSwapX
1185 bool needs_acquiring_load_exclusive(const Node *load);
1186
1187 // predicate controlling addressing modes
1188 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1189
1190 // Convert BootTest condition to Assembler condition.
1191 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1192 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1193 %}
1194
1195 source %{
1196
1197 // Derived RegMask with conditionally allocatable registers
1198
1199 void PhaseOutput::pd_perform_mach_node_analysis() {
1200 }
1201
1202 int MachNode::pd_alignment_required() const {
1203 return 1;
1204 }
1205
1206 int MachNode::compute_padding(int current_offset) const {
1207 return 0;
1208 }
1209
1210 RegMask _ANY_REG32_mask;
1211 RegMask _ANY_REG_mask;
1212 RegMask _PTR_REG_mask;
1213 RegMask _NO_SPECIAL_REG32_mask;
1214 RegMask _NO_SPECIAL_REG_mask;
1215 RegMask _NO_SPECIAL_PTR_REG_mask;
1216
1217 void reg_mask_init() {
1218 // We derive below RegMask(s) from the ones which are auto-generated from
1219 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1220 // registers conditionally reserved.
1221
1222 _ANY_REG32_mask = _ALL_REG32_mask;
1223 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1224
1225 _ANY_REG_mask = _ALL_REG_mask;
1226
1227 _PTR_REG_mask = _ALL_REG_mask;
1228
1229 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1230 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1231
1232 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1233 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1234
1235 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1236 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1237
1238 // r27 is not allocatable when compressed oops is on and heapbase is not
1239 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1240 if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
1241 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1242 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1243 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1244 }
1245
1246 // r29 is not allocatable when PreserveFramePointer is on
1247 if (PreserveFramePointer) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1251 }
1252 }
1253
1254 // Optimizaton of volatile gets and puts
1255 // -------------------------------------
1256 //
1257 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1258 // use to implement volatile reads and writes. For a volatile read
1259 // we simply need
1260 //
1261 // ldar<x>
1262 //
1263 // and for a volatile write we need
1264 //
1265 // stlr<x>
1266 //
1267 // Alternatively, we can implement them by pairing a normal
1268 // load/store with a memory barrier. For a volatile read we need
1269 //
1270 // ldr<x>
1271 // dmb ishld
1272 //
1273 // for a volatile write
1274 //
1275 // dmb ish
1276 // str<x>
1277 // dmb ish
1278 //
1279 // We can also use ldaxr and stlxr to implement compare and swap CAS
1280 // sequences. These are normally translated to an instruction
1281 // sequence like the following
1282 //
1283 // dmb ish
1284 // retry:
1285 // ldxr<x> rval raddr
1286 // cmp rval rold
1287 // b.ne done
1288 // stlxr<x> rval, rnew, rold
1289 // cbnz rval retry
1290 // done:
1291 // cset r0, eq
1292 // dmb ishld
1293 //
1294 // Note that the exclusive store is already using an stlxr
1295 // instruction. That is required to ensure visibility to other
1296 // threads of the exclusive write (assuming it succeeds) before that
1297 // of any subsequent writes.
1298 //
1299 // The following instruction sequence is an improvement on the above
1300 //
1301 // retry:
1302 // ldaxr<x> rval raddr
1303 // cmp rval rold
1304 // b.ne done
1305 // stlxr<x> rval, rnew, rold
1306 // cbnz rval retry
1307 // done:
1308 // cset r0, eq
1309 //
1310 // We don't need the leading dmb ish since the stlxr guarantees
1311 // visibility of prior writes in the case that the swap is
1312 // successful. Crucially we don't have to worry about the case where
1313 // the swap is not successful since no valid program should be
1314 // relying on visibility of prior changes by the attempting thread
1315 // in the case where the CAS fails.
1316 //
1317 // Similarly, we don't need the trailing dmb ishld if we substitute
1318 // an ldaxr instruction since that will provide all the guarantees we
1319 // require regarding observation of changes made by other threads
1320 // before any change to the CAS address observed by the load.
1321 //
1322 // In order to generate the desired instruction sequence we need to
1323 // be able to identify specific 'signature' ideal graph node
1324 // sequences which i) occur as a translation of a volatile reads or
1325 // writes or CAS operations and ii) do not occur through any other
1326 // translation or graph transformation. We can then provide
1327 // alternative aldc matching rules which translate these node
1328 // sequences to the desired machine code sequences. Selection of the
1329 // alternative rules can be implemented by predicates which identify
1330 // the relevant node sequences.
1331 //
1332 // The ideal graph generator translates a volatile read to the node
1333 // sequence
1334 //
1335 // LoadX[mo_acquire]
1336 // MemBarAcquire
1337 //
1338 // As a special case when using the compressed oops optimization we
1339 // may also see this variant
1340 //
1341 // LoadN[mo_acquire]
1342 // DecodeN
1343 // MemBarAcquire
1344 //
1345 // A volatile write is translated to the node sequence
1346 //
1347 // MemBarRelease
1348 // StoreX[mo_release] {CardMark}-optional
1349 // MemBarVolatile
1350 //
1351 // n.b. the above node patterns are generated with a strict
1352 // 'signature' configuration of input and output dependencies (see
1353 // the predicates below for exact details). The card mark may be as
1354 // simple as a few extra nodes or, in a few GC configurations, may
1355 // include more complex control flow between the leading and
1356 // trailing memory barriers. However, whatever the card mark
1357 // configuration these signatures are unique to translated volatile
1358 // reads/stores -- they will not appear as a result of any other
1359 // bytecode translation or inlining nor as a consequence of
1360 // optimizing transforms.
1361 //
1362 // We also want to catch inlined unsafe volatile gets and puts and
1363 // be able to implement them using either ldar<x>/stlr<x> or some
1364 // combination of ldr<x>/stlr<x> and dmb instructions.
1365 //
1366 // Inlined unsafe volatiles puts manifest as a minor variant of the
1367 // normal volatile put node sequence containing an extra cpuorder
1368 // membar
1369 //
1370 // MemBarRelease
1371 // MemBarCPUOrder
1372 // StoreX[mo_release] {CardMark}-optional
1373 // MemBarCPUOrder
1374 // MemBarVolatile
1375 //
1376 // n.b. as an aside, a cpuorder membar is not itself subject to
1377 // matching and translation by adlc rules. However, the rule
1378 // predicates need to detect its presence in order to correctly
1379 // select the desired adlc rules.
1380 //
1381 // Inlined unsafe volatile gets manifest as a slightly different
1382 // node sequence to a normal volatile get because of the
1383 // introduction of some CPUOrder memory barriers to bracket the
1384 // Load. However, but the same basic skeleton of a LoadX feeding a
1385 // MemBarAcquire, possibly through an optional DecodeN, is still
1386 // present
1387 //
1388 // MemBarCPUOrder
1389 // || \\
1390 // MemBarCPUOrder LoadX[mo_acquire]
1391 // || |
1392 // || {DecodeN} optional
1393 // || /
1394 // MemBarAcquire
1395 //
1396 // In this case the acquire membar does not directly depend on the
1397 // load. However, we can be sure that the load is generated from an
1398 // inlined unsafe volatile get if we see it dependent on this unique
1399 // sequence of membar nodes. Similarly, given an acquire membar we
1400 // can know that it was added because of an inlined unsafe volatile
1401 // get if it is fed and feeds a cpuorder membar and if its feed
1402 // membar also feeds an acquiring load.
1403 //
1404 // Finally an inlined (Unsafe) CAS operation is translated to the
1405 // following ideal graph
1406 //
1407 // MemBarRelease
1408 // MemBarCPUOrder
1409 // CompareAndSwapX {CardMark}-optional
1410 // MemBarCPUOrder
1411 // MemBarAcquire
1412 //
1413 // So, where we can identify these volatile read and write
1414 // signatures we can choose to plant either of the above two code
1415 // sequences. For a volatile read we can simply plant a normal
1416 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1417 // also choose to inhibit translation of the MemBarAcquire and
1418 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1419 //
1420 // When we recognise a volatile store signature we can choose to
1421 // plant at a dmb ish as a translation for the MemBarRelease, a
1422 // normal str<x> and then a dmb ish for the MemBarVolatile.
1423 // Alternatively, we can inhibit translation of the MemBarRelease
1424 // and MemBarVolatile and instead plant a simple stlr<x>
1425 // instruction.
1426 //
1427 // when we recognise a CAS signature we can choose to plant a dmb
1428 // ish as a translation for the MemBarRelease, the conventional
1429 // macro-instruction sequence for the CompareAndSwap node (which
1430 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1431 // Alternatively, we can elide generation of the dmb instructions
1432 // and plant the alternative CompareAndSwap macro-instruction
1433 // sequence (which uses ldaxr<x>).
1434 //
1435 // Of course, the above only applies when we see these signature
1436 // configurations. We still want to plant dmb instructions in any
1437 // other cases where we may see a MemBarAcquire, MemBarRelease or
1438 // MemBarVolatile. For example, at the end of a constructor which
1439 // writes final/volatile fields we will see a MemBarRelease
1440 // instruction and this needs a 'dmb ish' lest we risk the
1441 // constructed object being visible without making the
1442 // final/volatile field writes visible.
1443 //
1444 // n.b. the translation rules below which rely on detection of the
1445 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1446 // If we see anything other than the signature configurations we
1447 // always just translate the loads and stores to ldr<x> and str<x>
1448 // and translate acquire, release and volatile membars to the
1449 // relevant dmb instructions.
1450 //
1451
1452 // is_CAS(int opcode, bool maybe_volatile)
1453 //
1454 // return true if opcode is one of the possible CompareAndSwapX
1455 // values otherwise false.
1456
1457 bool is_CAS(int opcode, bool maybe_volatile)
1458 {
1459 switch(opcode) {
1460 // We handle these
1461 case Op_CompareAndSwapI:
1462 case Op_CompareAndSwapL:
1463 case Op_CompareAndSwapP:
1464 case Op_CompareAndSwapN:
1465 case Op_ShenandoahCompareAndSwapP:
1466 case Op_ShenandoahCompareAndSwapN:
1467 case Op_CompareAndSwapB:
1468 case Op_CompareAndSwapS:
1469 case Op_GetAndSetI:
1470 case Op_GetAndSetL:
1471 case Op_GetAndSetP:
1472 case Op_GetAndSetN:
1473 case Op_GetAndAddI:
1474 case Op_GetAndAddL:
1475 return true;
1476 case Op_CompareAndExchangeI:
1477 case Op_CompareAndExchangeN:
1478 case Op_CompareAndExchangeB:
1479 case Op_CompareAndExchangeS:
1480 case Op_CompareAndExchangeL:
1481 case Op_CompareAndExchangeP:
1482 case Op_WeakCompareAndSwapB:
1483 case Op_WeakCompareAndSwapS:
1484 case Op_WeakCompareAndSwapI:
1485 case Op_WeakCompareAndSwapL:
1486 case Op_WeakCompareAndSwapP:
1487 case Op_WeakCompareAndSwapN:
1488 case Op_ShenandoahWeakCompareAndSwapP:
1489 case Op_ShenandoahWeakCompareAndSwapN:
1490 case Op_ShenandoahCompareAndExchangeP:
1491 case Op_ShenandoahCompareAndExchangeN:
1492 return maybe_volatile;
1493 default:
1494 return false;
1495 }
1496 }
1497
1498 // helper to determine the maximum number of Phi nodes we may need to
1499 // traverse when searching from a card mark membar for the merge mem
1500 // feeding a trailing membar or vice versa
1501
1502 // predicates controlling emit of ldr<x>/ldar<x>
1503
1504 bool unnecessary_acquire(const Node *barrier)
1505 {
1506 assert(barrier->is_MemBar(), "expecting a membar");
1507
1508 MemBarNode* mb = barrier->as_MemBar();
1509
1510 if (mb->trailing_load()) {
1511 return true;
1512 }
1513
1514 if (mb->trailing_load_store()) {
1515 Node* load_store = mb->in(MemBarNode::Precedent);
1516 assert(load_store->is_LoadStore(), "unexpected graph shape");
1517 return is_CAS(load_store->Opcode(), true);
1518 }
1519
1520 return false;
1521 }
1522
1523 bool needs_acquiring_load(const Node *n)
1524 {
1525 assert(n->is_Load(), "expecting a load");
1526 LoadNode *ld = n->as_Load();
1527 return ld->is_acquire();
1528 }
1529
1530 bool unnecessary_release(const Node *n)
1531 {
1532 assert((n->is_MemBar() &&
1533 n->Opcode() == Op_MemBarRelease),
1534 "expecting a release membar");
1535
1536 MemBarNode *barrier = n->as_MemBar();
1537 if (!barrier->leading()) {
1538 return false;
1539 } else {
1540 Node* trailing = barrier->trailing_membar();
1541 MemBarNode* trailing_mb = trailing->as_MemBar();
1542 assert(trailing_mb->trailing(), "Not a trailing membar?");
1543 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1544
1545 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1546 if (mem->is_Store()) {
1547 assert(mem->as_Store()->is_release(), "");
1548 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1549 return true;
1550 } else {
1551 assert(mem->is_LoadStore(), "");
1552 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1553 return is_CAS(mem->Opcode(), true);
1554 }
1555 }
1556 return false;
1557 }
1558
1559 bool unnecessary_volatile(const Node *n)
1560 {
1561 // assert n->is_MemBar();
1562 MemBarNode *mbvol = n->as_MemBar();
1563
1564 bool release = mbvol->trailing_store();
1565 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1566 #ifdef ASSERT
1567 if (release) {
1568 Node* leading = mbvol->leading_membar();
1569 assert(leading->Opcode() == Op_MemBarRelease, "");
1570 assert(leading->as_MemBar()->leading_store(), "");
1571 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1572 }
1573 #endif
1574
1575 return release;
1576 }
1577
1578 // predicates controlling emit of str<x>/stlr<x>
1579
1580 bool needs_releasing_store(const Node *n)
1581 {
1582 // assert n->is_Store();
1583 StoreNode *st = n->as_Store();
1584 return st->trailing_membar() != NULL;
1585 }
1586
1587 // predicate controlling translation of CAS
1588 //
1589 // returns true if CAS needs to use an acquiring load otherwise false
1590
1591 bool needs_acquiring_load_exclusive(const Node *n)
1592 {
1593 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1594 LoadStoreNode* ldst = n->as_LoadStore();
1595 if (is_CAS(n->Opcode(), false)) {
1596 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1597 } else {
1598 return ldst->trailing_membar() != NULL;
1599 }
1600
1601 // so we can just return true here
1602 return true;
1603 }
1604
1605 #define __ _masm.
1606
1607 // advance declarations for helper functions to convert register
1608 // indices to register objects
1609
1610 // the ad file has to provide implementations of certain methods
1611 // expected by the generic code
1612 //
1613 // REQUIRED FUNCTIONALITY
1614
1615 //=============================================================================
1616
1617 // !!!!! Special hack to get all types of calls to specify the byte offset
1618 // from the start of the call to the point where the return address
1619 // will point.
1620
1621 int MachCallStaticJavaNode::ret_addr_offset()
1622 {
1623 // call should be a simple bl
1624 int off = 4;
1625 return off;
1626 }
1627
1628 int MachCallDynamicJavaNode::ret_addr_offset()
1629 {
1630 return 16; // movz, movk, movk, bl
1631 }
1632
1633 int MachCallRuntimeNode::ret_addr_offset() {
1634 // for generated stubs the call will be
1635 // bl(addr)
1636 // or with far branches
1637 // bl(trampoline_stub)
1638 // for real runtime callouts it will be six instructions
1639 // see aarch64_enc_java_to_runtime
1640 // adr(rscratch2, retaddr)
1641 // lea(rscratch1, RuntimeAddress(addr)
1642 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1643 // blr(rscratch1)
1644 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1645 if (cb) {
1646 return 1 * NativeInstruction::instruction_size;
1647 } else {
1648 return 6 * NativeInstruction::instruction_size;
1649 }
1650 }
1651
1652 //=============================================================================
1653
1654 #ifndef PRODUCT
1655 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1656 st->print("BREAKPOINT");
1657 }
1658 #endif
1659
1660 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1661 C2_MacroAssembler _masm(&cbuf);
1662 __ brk(0);
1663 }
1664
1665 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1666 return MachNode::size(ra_);
1667 }
1668
1669 //=============================================================================
1670
1671 #ifndef PRODUCT
1672 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1673 st->print("nop \t# %d bytes pad for loops and calls", _count);
1674 }
1675 #endif
1676
1677 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1678 C2_MacroAssembler _masm(&cbuf);
1679 for (int i = 0; i < _count; i++) {
1680 __ nop();
1681 }
1682 }
1683
1684 uint MachNopNode::size(PhaseRegAlloc*) const {
1685 return _count * NativeInstruction::instruction_size;
1686 }
1687
1688 //=============================================================================
1689 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1690
1691 int ConstantTable::calculate_table_base_offset() const {
1692 return 0; // absolute addressing, no offset
1693 }
1694
1695 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1696 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1697 ShouldNotReachHere();
1698 }
1699
1700 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1701 // Empty encoding
1702 }
1703
1704 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1705 return 0;
1706 }
1707
1708 #ifndef PRODUCT
1709 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1710 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1711 }
1712 #endif
1713
1714 #ifndef PRODUCT
1715 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1716 Compile* C = ra_->C;
1717
1718 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1719
1720 if (C->output()->need_stack_bang(framesize))
1721 st->print("# stack bang size=%d\n\t", framesize);
1722
1723 if (VM_Version::use_rop_protection()) {
1724 st->print("ldr zr, [lr]\n\t");
1725 st->print("pacia lr, rfp\n\t");
1726 }
1727 if (framesize < ((1 << 9) + 2 * wordSize)) {
1728 st->print("sub sp, sp, #%d\n\t", framesize);
1729 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1730 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1731 } else {
1732 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1733 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1734 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1735 st->print("sub sp, sp, rscratch1");
1736 }
1737 if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1738 st->print("\n\t");
1739 st->print("ldr rscratch1, [guard]\n\t");
1740 st->print("dmb ishld\n\t");
1741 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1742 st->print("cmp rscratch1, rscratch2\n\t");
1743 st->print("b.eq skip");
1744 st->print("\n\t");
1745 st->print("blr #nmethod_entry_barrier_stub\n\t");
1746 st->print("b skip\n\t");
1747 st->print("guard: int\n\t");
1748 st->print("\n\t");
1749 st->print("skip:\n\t");
1750 }
1751 }
1752 #endif
1753
1754 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1755 Compile* C = ra_->C;
1756 C2_MacroAssembler _masm(&cbuf);
1757
1758 // n.b. frame size includes space for return pc and rfp
1759 const int framesize = C->output()->frame_size_in_bytes();
1760
1761 // insert a nop at the start of the prolog so we can patch in a
1762 // branch if we need to invalidate the method later
1763 __ nop();
1764
1765 if (C->clinit_barrier_on_entry()) {
1766 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1767
1768 Label L_skip_barrier;
1769
1770 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1771 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1772 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1773 __ bind(L_skip_barrier);
1774 }
1775
1776 if (C->max_vector_size() > 0) {
1777 __ reinitialize_ptrue();
1778 }
1779
1780 int bangsize = C->output()->bang_size_in_bytes();
1781 if (C->output()->need_stack_bang(bangsize))
1782 __ generate_stack_overflow_check(bangsize);
1783
1784 __ build_frame(framesize);
1785
1786 if (C->stub_function() == NULL) {
1787 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1788 if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1789 // Dummy labels for just measuring the code size
1790 Label dummy_slow_path;
1791 Label dummy_continuation;
1792 Label dummy_guard;
1793 Label* slow_path = &dummy_slow_path;
1794 Label* continuation = &dummy_continuation;
1795 Label* guard = &dummy_guard;
1796 if (!Compile::current()->output()->in_scratch_emit_size()) {
1797 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1798 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1799 Compile::current()->output()->add_stub(stub);
1800 slow_path = &stub->entry();
1801 continuation = &stub->continuation();
1802 guard = &stub->guard();
1803 }
1804 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1805 bs->nmethod_entry_barrier(&_masm, slow_path, continuation, guard);
1806 }
1807 }
1808
1809 if (VerifyStackAtCalls) {
1810 Unimplemented();
1811 }
1812
1813 C->output()->set_frame_complete(cbuf.insts_size());
1814
1815 if (C->has_mach_constant_base_node()) {
1816 // NOTE: We set the table base offset here because users might be
1817 // emitted before MachConstantBaseNode.
1818 ConstantTable& constant_table = C->output()->constant_table();
1819 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1820 }
1821 }
1822
1823 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1824 {
1825 return MachNode::size(ra_); // too many variables; just compute it
1826 // the hard way
1827 }
1828
1829 int MachPrologNode::reloc() const
1830 {
1831 return 0;
1832 }
1833
1834 //=============================================================================
1835
1836 #ifndef PRODUCT
1837 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1838 Compile* C = ra_->C;
1839 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1840
1841 st->print("# pop frame %d\n\t",framesize);
1842
1843 if (framesize == 0) {
1844 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1845 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1846 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1847 st->print("add sp, sp, #%d\n\t", framesize);
1848 } else {
1849 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1850 st->print("add sp, sp, rscratch1\n\t");
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 }
1853 if (VM_Version::use_rop_protection()) {
1854 st->print("autia lr, rfp\n\t");
1855 st->print("ldr zr, [lr]\n\t");
1856 }
1857
1858 if (do_polling() && C->is_method_compilation()) {
1859 st->print("# test polling word\n\t");
1860 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1861 st->print("cmp sp, rscratch1\n\t");
1862 st->print("bhi #slow_path");
1863 }
1864 }
1865 #endif
1866
1867 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1868 Compile* C = ra_->C;
1869 C2_MacroAssembler _masm(&cbuf);
1870 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1871
1872 __ remove_frame(framesize);
1873
1874 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1875 __ reserved_stack_check();
1876 }
1877
1878 if (do_polling() && C->is_method_compilation()) {
1879 Label dummy_label;
1880 Label* code_stub = &dummy_label;
1881 if (!C->output()->in_scratch_emit_size()) {
1882 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1883 C->output()->add_stub(stub);
1884 code_stub = &stub->entry();
1885 }
1886 __ relocate(relocInfo::poll_return_type);
1887 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1888 }
1889 }
1890
1891 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1892 // Variable size. Determine dynamically.
1893 return MachNode::size(ra_);
1894 }
1895
1896 int MachEpilogNode::reloc() const {
1897 // Return number of relocatable values contained in this instruction.
1898 return 1; // 1 for polling page.
1899 }
1900
1901 const Pipeline * MachEpilogNode::pipeline() const {
1902 return MachNode::pipeline_class();
1903 }
1904
1905 //=============================================================================
1906
1907 // Figure out which register class each belongs in: rc_int, rc_float or
1908 // rc_stack.
1909 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
1910
1911 static enum RC rc_class(OptoReg::Name reg) {
1912
1913 if (reg == OptoReg::Bad) {
1914 return rc_bad;
1915 }
1916
1917 // we have 32 int registers * 2 halves
1918 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1919
1920 if (reg < slots_of_int_registers) {
1921 return rc_int;
1922 }
1923
1924 // we have 32 float register * 8 halves
1925 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1926 if (reg < slots_of_int_registers + slots_of_float_registers) {
1927 return rc_float;
1928 }
1929
1930 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1931 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1932 return rc_predicate;
1933 }
1934
1935 // Between predicate regs & stack is the flags.
1936 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1937
1938 return rc_stack;
1939 }
1940
1941 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1942 Compile* C = ra_->C;
1943
1944 // Get registers to move.
1945 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1946 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1947 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1948 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1949
1950 enum RC src_hi_rc = rc_class(src_hi);
1951 enum RC src_lo_rc = rc_class(src_lo);
1952 enum RC dst_hi_rc = rc_class(dst_hi);
1953 enum RC dst_lo_rc = rc_class(dst_lo);
1954
1955 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1956
1957 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1958 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1959 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1960 "expected aligned-adjacent pairs");
1961 }
1962
1963 if (src_lo == dst_lo && src_hi == dst_hi) {
1964 return 0; // Self copy, no move.
1965 }
1966
1967 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1968 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1969 int src_offset = ra_->reg2offset(src_lo);
1970 int dst_offset = ra_->reg2offset(dst_lo);
1971
1972 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1973 uint ireg = ideal_reg();
1974 if (ireg == Op_VecA && cbuf) {
1975 C2_MacroAssembler _masm(cbuf);
1976 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1977 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1978 // stack->stack
1979 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1980 sve_vector_reg_size_in_bytes);
1981 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1982 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1983 sve_vector_reg_size_in_bytes);
1984 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1985 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1986 sve_vector_reg_size_in_bytes);
1987 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1988 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1989 as_FloatRegister(Matcher::_regEncode[src_lo]),
1990 as_FloatRegister(Matcher::_regEncode[src_lo]));
1991 } else {
1992 ShouldNotReachHere();
1993 }
1994 } else if (cbuf) {
1995 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1996 C2_MacroAssembler _masm(cbuf);
1997 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1998 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1999 // stack->stack
2000 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2001 if (ireg == Op_VecD) {
2002 __ unspill(rscratch1, true, src_offset);
2003 __ spill(rscratch1, true, dst_offset);
2004 } else {
2005 __ spill_copy128(src_offset, dst_offset);
2006 }
2007 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2008 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2009 ireg == Op_VecD ? __ T8B : __ T16B,
2010 as_FloatRegister(Matcher::_regEncode[src_lo]));
2011 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2012 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2013 ireg == Op_VecD ? __ D : __ Q,
2014 ra_->reg2offset(dst_lo));
2015 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2016 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 ireg == Op_VecD ? __ D : __ Q,
2018 ra_->reg2offset(src_lo));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 }
2023 } else if (cbuf) {
2024 C2_MacroAssembler _masm(cbuf);
2025 switch (src_lo_rc) {
2026 case rc_int:
2027 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2028 if (is64) {
2029 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2030 as_Register(Matcher::_regEncode[src_lo]));
2031 } else {
2032 C2_MacroAssembler _masm(cbuf);
2033 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2034 as_Register(Matcher::_regEncode[src_lo]));
2035 }
2036 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2037 if (is64) {
2038 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2039 as_Register(Matcher::_regEncode[src_lo]));
2040 } else {
2041 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2042 as_Register(Matcher::_regEncode[src_lo]));
2043 }
2044 } else { // gpr --> stack spill
2045 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2046 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2047 }
2048 break;
2049 case rc_float:
2050 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2051 if (is64) {
2052 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2053 as_FloatRegister(Matcher::_regEncode[src_lo]));
2054 } else {
2055 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2056 as_FloatRegister(Matcher::_regEncode[src_lo]));
2057 }
2058 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2059 if (is64) {
2060 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2061 as_FloatRegister(Matcher::_regEncode[src_lo]));
2062 } else {
2063 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2064 as_FloatRegister(Matcher::_regEncode[src_lo]));
2065 }
2066 } else { // fpr --> stack spill
2067 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2068 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2069 is64 ? __ D : __ S, dst_offset);
2070 }
2071 break;
2072 case rc_stack:
2073 if (dst_lo_rc == rc_int) { // stack --> gpr load
2074 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2075 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2076 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2077 is64 ? __ D : __ S, src_offset);
2078 } else if (dst_lo_rc == rc_predicate) {
2079 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2080 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2081 } else { // stack --> stack copy
2082 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2083 if (ideal_reg() == Op_RegVectMask) {
2084 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2085 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2086 } else {
2087 __ unspill(rscratch1, is64, src_offset);
2088 __ spill(rscratch1, is64, dst_offset);
2089 }
2090 }
2091 break;
2092 case rc_predicate:
2093 if (dst_lo_rc == rc_predicate) {
2094 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2095 } else if (dst_lo_rc == rc_stack) {
2096 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2097 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2098 } else {
2099 assert(false, "bad src and dst rc_class combination.");
2100 ShouldNotReachHere();
2101 }
2102 break;
2103 default:
2104 assert(false, "bad rc_class for spill");
2105 ShouldNotReachHere();
2106 }
2107 }
2108
2109 if (st) {
2110 st->print("spill ");
2111 if (src_lo_rc == rc_stack) {
2112 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2113 } else {
2114 st->print("%s -> ", Matcher::regName[src_lo]);
2115 }
2116 if (dst_lo_rc == rc_stack) {
2117 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2118 } else {
2119 st->print("%s", Matcher::regName[dst_lo]);
2120 }
2121 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2122 int vsize = 0;
2123 switch (ideal_reg()) {
2124 case Op_VecD:
2125 vsize = 64;
2126 break;
2127 case Op_VecX:
2128 vsize = 128;
2129 break;
2130 case Op_VecA:
2131 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2132 break;
2133 default:
2134 assert(false, "bad register type for spill");
2135 ShouldNotReachHere();
2136 }
2137 st->print("\t# vector spill size = %d", vsize);
2138 } else if (ideal_reg() == Op_RegVectMask) {
2139 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2140 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2141 st->print("\t# predicate spill size = %d", vsize);
2142 } else {
2143 st->print("\t# spill size = %d", is64 ? 64 : 32);
2144 }
2145 }
2146
2147 return 0;
2148
2149 }
2150
2151 #ifndef PRODUCT
2152 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2153 if (!ra_)
2154 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2155 else
2156 implementation(NULL, ra_, false, st);
2157 }
2158 #endif
2159
2160 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2161 implementation(&cbuf, ra_, false, NULL);
2162 }
2163
2164 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2165 return MachNode::size(ra_);
2166 }
2167
2168 //=============================================================================
2169
2170 #ifndef PRODUCT
2171 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2172 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2173 int reg = ra_->get_reg_first(this);
2174 st->print("add %s, rsp, #%d]\t# box lock",
2175 Matcher::regName[reg], offset);
2176 }
2177 #endif
2178
2179 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2180 C2_MacroAssembler _masm(&cbuf);
2181
2182 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2183 int reg = ra_->get_encode(this);
2184
2185 // This add will handle any 24-bit signed offset. 24 bits allows an
2186 // 8 megabyte stack frame.
2187 __ add(as_Register(reg), sp, offset);
2188 }
2189
2190 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2191 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2192 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2193
2194 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2195 return NativeInstruction::instruction_size;
2196 } else {
2197 return 2 * NativeInstruction::instruction_size;
2198 }
2199 }
2200
2201 //=============================================================================
2202
2203 #ifndef PRODUCT
2204 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2205 {
2206 st->print_cr("# MachUEPNode");
2207 if (UseCompressedClassPointers) {
2208 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2209 if (CompressedKlassPointers::shift() != 0) {
2210 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2211 }
2212 } else {
2213 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2214 }
2215 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2216 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2217 }
2218 #endif
2219
2220 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2221 {
2222 // This is the unverified entry point.
2223 C2_MacroAssembler _masm(&cbuf);
2224
2225 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2226 Label skip;
2227 // TODO
2228 // can we avoid this skip and still use a reloc?
2229 __ br(Assembler::EQ, skip);
2230 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2231 __ bind(skip);
2232 }
2233
2234 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2235 {
2236 return MachNode::size(ra_);
2237 }
2238
2239 // REQUIRED EMIT CODE
2240
2241 //=============================================================================
2242
2243 // Emit exception handler code.
2244 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2245 {
2246 // mov rscratch1 #exception_blob_entry_point
2247 // br rscratch1
2248 // Note that the code buffer's insts_mark is always relative to insts.
2249 // That's why we must use the macroassembler to generate a handler.
2250 C2_MacroAssembler _masm(&cbuf);
2251 address base = __ start_a_stub(size_exception_handler());
2252 if (base == NULL) {
2253 ciEnv::current()->record_failure("CodeCache is full");
2254 return 0; // CodeBuffer::expand failed
2255 }
2256 int offset = __ offset();
2257 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2258 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2259 __ end_a_stub();
2260 return offset;
2261 }
2262
2263 // Emit deopt handler code.
2264 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2265 {
2266 // Note that the code buffer's insts_mark is always relative to insts.
2267 // That's why we must use the macroassembler to generate a handler.
2268 C2_MacroAssembler _masm(&cbuf);
2269 address base = __ start_a_stub(size_deopt_handler());
2270 if (base == NULL) {
2271 ciEnv::current()->record_failure("CodeCache is full");
2272 return 0; // CodeBuffer::expand failed
2273 }
2274 int offset = __ offset();
2275
2276 __ adr(lr, __ pc());
2277 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2278
2279 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2280 __ end_a_stub();
2281 return offset;
2282 }
2283
2284 // REQUIRED MATCHER CODE
2285
2286 //=============================================================================
2287
2288 const bool Matcher::match_rule_supported(int opcode) {
2289 if (!has_match_rule(opcode))
2290 return false;
2291
2292 bool ret_value = true;
2293 switch (opcode) {
2294 case Op_OnSpinWait:
2295 return VM_Version::supports_on_spin_wait();
2296 case Op_CacheWB:
2297 case Op_CacheWBPreSync:
2298 case Op_CacheWBPostSync:
2299 if (!VM_Version::supports_data_cache_line_flush()) {
2300 ret_value = false;
2301 }
2302 break;
2303 case Op_ExpandBits:
2304 case Op_CompressBits:
2305 if (!(UseSVE > 1 && VM_Version::supports_svebitperm())) {
2306 ret_value = false;
2307 }
2308 break;
2309 }
2310
2311 return ret_value; // Per default match rules are supported.
2312 }
2313
2314 const RegMask* Matcher::predicate_reg_mask(void) {
2315 return &_PR_REG_mask;
2316 }
2317
2318 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2319 return new TypeVectMask(elemTy, length);
2320 }
2321
2322 // Vector calling convention not yet implemented.
2323 const bool Matcher::supports_vector_calling_convention(void) {
2324 return false;
2325 }
2326
2327 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2328 Unimplemented();
2329 return OptoRegPair(0, 0);
2330 }
2331
2332 // Is this branch offset short enough that a short branch can be used?
2333 //
2334 // NOTE: If the platform does not provide any short branch variants, then
2335 // this method should return false for offset 0.
2336 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2337 // The passed offset is relative to address of the branch.
2338
2339 return (-32768 <= offset && offset < 32768);
2340 }
2341
2342 // Vector width in bytes.
2343 const int Matcher::vector_width_in_bytes(BasicType bt) {
2344 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2345 int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
2346 // Minimum 2 values in vector
2347 if (size < 2*type2aelembytes(bt)) size = 0;
2348 // But never < 4
2349 if (size < 4) size = 0;
2350 return size;
2351 }
2352
2353 // Limits on vector size (number of elements) loaded into vector.
2354 const int Matcher::max_vector_size(const BasicType bt) {
2355 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2356 }
2357
2358 const int Matcher::min_vector_size(const BasicType bt) {
2359 int max_size = max_vector_size(bt);
2360 // Limit the min vector size to 8 bytes.
2361 int size = 8 / type2aelembytes(bt);
2362 if (bt == T_BYTE) {
2363 // To support vector api shuffle/rearrange.
2364 size = 4;
2365 } else if (bt == T_BOOLEAN) {
2366 // To support vector api load/store mask.
2367 size = 2;
2368 }
2369 if (size < 2) size = 2;
2370 return MIN2(size, max_size);
2371 }
2372
2373 const int Matcher::superword_max_vector_size(const BasicType bt) {
2374 return Matcher::max_vector_size(bt);
2375 }
2376
2377 // Actual max scalable vector register length.
2378 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
2379 return Matcher::max_vector_size(bt);
2380 }
2381
2382 // Vector ideal reg.
2383 const uint Matcher::vector_ideal_reg(int len) {
2384 if (UseSVE > 0 && 16 < len && len <= 256) {
2385 return Op_VecA;
2386 }
2387 switch(len) {
2388 // For 16-bit/32-bit mask vector, reuse VecD.
2389 case 2:
2390 case 4:
2391 case 8: return Op_VecD;
2392 case 16: return Op_VecX;
2393 }
2394 ShouldNotReachHere();
2395 return 0;
2396 }
2397
2398 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2399 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2400 switch (ideal_reg) {
2401 case Op_VecA: return new vecAOper();
2402 case Op_VecD: return new vecDOper();
2403 case Op_VecX: return new vecXOper();
2404 }
2405 ShouldNotReachHere();
2406 return NULL;
2407 }
2408
2409 bool Matcher::is_reg2reg_move(MachNode* m) {
2410 return false;
2411 }
2412
2413 bool Matcher::is_generic_vector(MachOper* opnd) {
2414 return opnd->opcode() == VREG;
2415 }
2416
2417 // Return whether or not this register is ever used as an argument.
2418 // This function is used on startup to build the trampoline stubs in
2419 // generateOptoStub. Registers not mentioned will be killed by the VM
2420 // call in the trampoline, and arguments in those registers not be
2421 // available to the callee.
2422 bool Matcher::can_be_java_arg(int reg)
2423 {
2424 return
2425 reg == R0_num || reg == R0_H_num ||
2426 reg == R1_num || reg == R1_H_num ||
2427 reg == R2_num || reg == R2_H_num ||
2428 reg == R3_num || reg == R3_H_num ||
2429 reg == R4_num || reg == R4_H_num ||
2430 reg == R5_num || reg == R5_H_num ||
2431 reg == R6_num || reg == R6_H_num ||
2432 reg == R7_num || reg == R7_H_num ||
2433 reg == V0_num || reg == V0_H_num ||
2434 reg == V1_num || reg == V1_H_num ||
2435 reg == V2_num || reg == V2_H_num ||
2436 reg == V3_num || reg == V3_H_num ||
2437 reg == V4_num || reg == V4_H_num ||
2438 reg == V5_num || reg == V5_H_num ||
2439 reg == V6_num || reg == V6_H_num ||
2440 reg == V7_num || reg == V7_H_num;
2441 }
2442
2443 bool Matcher::is_spillable_arg(int reg)
2444 {
2445 return can_be_java_arg(reg);
2446 }
2447
2448 uint Matcher::int_pressure_limit()
2449 {
2450 // JDK-8183543: When taking the number of available registers as int
2451 // register pressure threshold, the jtreg test:
2452 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2453 // failed due to C2 compilation failure with
2454 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2455 //
2456 // A derived pointer is live at CallNode and then is flagged by RA
2457 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2458 // derived pointers and lastly fail to spill after reaching maximum
2459 // number of iterations. Lowering the default pressure threshold to
2460 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2461 // a high register pressure area of the code so that split_DEF can
2462 // generate DefinitionSpillCopy for the derived pointer.
2463 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2464 if (!PreserveFramePointer) {
2465 // When PreserveFramePointer is off, frame pointer is allocatable,
2466 // but different from other SOC registers, it is excluded from
2467 // fatproj's mask because its save type is No-Save. Decrease 1 to
2468 // ensure high pressure at fatproj when PreserveFramePointer is off.
2469 // See check_pressure_at_fatproj().
2470 default_int_pressure_threshold--;
2471 }
2472 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2473 }
2474
2475 uint Matcher::float_pressure_limit()
2476 {
2477 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2478 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2479 }
2480
2481 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2482 return false;
2483 }
2484
2485 RegMask Matcher::divI_proj_mask() {
2486 ShouldNotReachHere();
2487 return RegMask();
2488 }
2489
2490 // Register for MODI projection of divmodI.
2491 RegMask Matcher::modI_proj_mask() {
2492 ShouldNotReachHere();
2493 return RegMask();
2494 }
2495
2496 // Register for DIVL projection of divmodL.
2497 RegMask Matcher::divL_proj_mask() {
2498 ShouldNotReachHere();
2499 return RegMask();
2500 }
2501
2502 // Register for MODL projection of divmodL.
2503 RegMask Matcher::modL_proj_mask() {
2504 ShouldNotReachHere();
2505 return RegMask();
2506 }
2507
2508 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2509 return FP_REG_mask();
2510 }
2511
2512 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2513 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2514 Node* u = addp->fast_out(i);
2515 if (u->is_LoadStore()) {
2516 // On AArch64, LoadStoreNodes (i.e. compare and swap
2517 // instructions) only take register indirect as an operand, so
2518 // any attempt to use an AddPNode as an input to a LoadStoreNode
2519 // must fail.
2520 return false;
2521 }
2522 if (u->is_Mem()) {
2523 int opsize = u->as_Mem()->memory_size();
2524 assert(opsize > 0, "unexpected memory operand size");
2525 if (u->as_Mem()->memory_size() != (1<<shift)) {
2526 return false;
2527 }
2528 }
2529 }
2530 return true;
2531 }
2532
2533 // Convert BootTest condition to Assembler condition.
2534 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2535 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2536 Assembler::Condition result;
2537 switch(cond) {
2538 case BoolTest::eq:
2539 result = Assembler::EQ; break;
2540 case BoolTest::ne:
2541 result = Assembler::NE; break;
2542 case BoolTest::le:
2543 result = Assembler::LE; break;
2544 case BoolTest::ge:
2545 result = Assembler::GE; break;
2546 case BoolTest::lt:
2547 result = Assembler::LT; break;
2548 case BoolTest::gt:
2549 result = Assembler::GT; break;
2550 case BoolTest::ule:
2551 result = Assembler::LS; break;
2552 case BoolTest::uge:
2553 result = Assembler::HS; break;
2554 case BoolTest::ult:
2555 result = Assembler::LO; break;
2556 case BoolTest::ugt:
2557 result = Assembler::HI; break;
2558 case BoolTest::overflow:
2559 result = Assembler::VS; break;
2560 case BoolTest::no_overflow:
2561 result = Assembler::VC; break;
2562 default:
2563 ShouldNotReachHere();
2564 return Assembler::Condition(-1);
2565 }
2566
2567 // Check conversion
2568 if (cond & BoolTest::unsigned_compare) {
2569 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2570 } else {
2571 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2572 }
2573
2574 return result;
2575 }
2576
2577 // Binary src (Replicate con)
2578 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2579 if (n == NULL || m == NULL) {
2580 return false;
2581 }
2582
2583 if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
2584 return false;
2585 }
2586
2587 Node* imm_node = m->in(1);
2588 if (!imm_node->is_Con()) {
2589 return false;
2590 }
2591
2592 const Type* t = imm_node->bottom_type();
2593 if (!(t->isa_int() || t->isa_long())) {
2594 return false;
2595 }
2596
2597 switch (n->Opcode()) {
2598 case Op_AndV:
2599 case Op_OrV:
2600 case Op_XorV: {
2601 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2602 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2603 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2604 }
2605 case Op_AddVB:
2606 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2607 case Op_AddVS:
2608 case Op_AddVI:
2609 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2610 case Op_AddVL:
2611 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2612 default:
2613 return false;
2614 }
2615 }
2616
2617 // (XorV src (Replicate m1))
2618 // (XorVMask src (MaskAll m1))
2619 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2620 if (n != NULL && m != NULL) {
2621 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2622 VectorNode::is_all_ones_vector(m);
2623 }
2624 return false;
2625 }
2626
2627 // Should the matcher clone input 'm' of node 'n'?
2628 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2629 if (is_vshift_con_pattern(n, m) ||
2630 is_vector_bitwise_not_pattern(n, m) ||
2631 is_valid_sve_arith_imm_pattern(n, m)) {
2632 mstack.push(m, Visit);
2633 return true;
2634 }
2635 return false;
2636 }
2637
2638 // Should the Matcher clone shifts on addressing modes, expecting them
2639 // to be subsumed into complex addressing expressions or compute them
2640 // into registers?
2641 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2642 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2643 return true;
2644 }
2645
2646 Node *off = m->in(AddPNode::Offset);
2647 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2648 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2649 // Are there other uses besides address expressions?
2650 !is_visited(off)) {
2651 address_visited.set(off->_idx); // Flag as address_visited
2652 mstack.push(off->in(2), Visit);
2653 Node *conv = off->in(1);
2654 if (conv->Opcode() == Op_ConvI2L &&
2655 // Are there other uses besides address expressions?
2656 !is_visited(conv)) {
2657 address_visited.set(conv->_idx); // Flag as address_visited
2658 mstack.push(conv->in(1), Pre_Visit);
2659 } else {
2660 mstack.push(conv, Pre_Visit);
2661 }
2662 address_visited.test_set(m->_idx); // Flag as address_visited
2663 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2664 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2665 return true;
2666 } else if (off->Opcode() == Op_ConvI2L &&
2667 // Are there other uses besides address expressions?
2668 !is_visited(off)) {
2669 address_visited.test_set(m->_idx); // Flag as address_visited
2670 address_visited.set(off->_idx); // Flag as address_visited
2671 mstack.push(off->in(1), Pre_Visit);
2672 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2673 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2674 return true;
2675 }
2676 return false;
2677 }
2678
2679 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2680 C2_MacroAssembler _masm(&cbuf); \
2681 { \
2682 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2683 guarantee(DISP == 0, "mode not permitted for volatile"); \
2684 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2685 __ INSN(REG, as_Register(BASE)); \
2686 }
2687
2688
2689 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2690 {
2691 Address::extend scale;
2692
2693 // Hooboy, this is fugly. We need a way to communicate to the
2694 // encoder that the index needs to be sign extended, so we have to
2695 // enumerate all the cases.
2696 switch (opcode) {
2697 case INDINDEXSCALEDI2L:
2698 case INDINDEXSCALEDI2LN:
2699 case INDINDEXI2L:
2700 case INDINDEXI2LN:
2701 scale = Address::sxtw(size);
2702 break;
2703 default:
2704 scale = Address::lsl(size);
2705 }
2706
2707 if (index == -1) {
2708 return Address(base, disp);
2709 } else {
2710 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2711 return Address(base, as_Register(index), scale);
2712 }
2713 }
2714
2715
2716 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2717 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2718 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2719 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2720 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2721
2722 // Used for all non-volatile memory accesses. The use of
2723 // $mem->opcode() to discover whether this pattern uses sign-extended
2724 // offsets is something of a kludge.
2725 static void loadStore(C2_MacroAssembler masm, mem_insn insn,
2726 Register reg, int opcode,
2727 Register base, int index, int scale, int disp,
2728 int size_in_memory)
2729 {
2730 Address addr = mem2address(opcode, base, index, scale, disp);
2731 if (addr.getMode() == Address::base_plus_offset) {
2732 /* If we get an out-of-range offset it is a bug in the compiler,
2733 so we assert here. */
2734 assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
2735 "c2 compiler bug");
2736 /* Fix up any out-of-range offsets. */
2737 assert_different_registers(rscratch1, base);
2738 assert_different_registers(rscratch1, reg);
2739 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2740 }
2741 (masm.*insn)(reg, addr);
2742 }
2743
2744 static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
2745 FloatRegister reg, int opcode,
2746 Register base, int index, int size, int disp,
2747 int size_in_memory)
2748 {
2749 Address::extend scale;
2750
2751 switch (opcode) {
2752 case INDINDEXSCALEDI2L:
2753 case INDINDEXSCALEDI2LN:
2754 scale = Address::sxtw(size);
2755 break;
2756 default:
2757 scale = Address::lsl(size);
2758 }
2759
2760 if (index == -1) {
2761 /* If we get an out-of-range offset it is a bug in the compiler,
2762 so we assert here. */
2763 assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2764 /* Fix up any out-of-range offsets. */
2765 assert_different_registers(rscratch1, base);
2766 Address addr = Address(base, disp);
2767 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2768 (masm.*insn)(reg, addr);
2769 } else {
2770 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2771 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2772 }
2773 }
2774
2775 static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
2776 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2777 int opcode, Register base, int index, int size, int disp)
2778 {
2779 if (index == -1) {
2780 (masm.*insn)(reg, T, Address(base, disp));
2781 } else {
2782 assert(disp == 0, "unsupported address mode");
2783 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2784 }
2785 }
2786
2787 %}
2788
2789
2790
2791 //----------ENCODING BLOCK-----------------------------------------------------
2792 // This block specifies the encoding classes used by the compiler to
2793 // output byte streams. Encoding classes are parameterized macros
2794 // used by Machine Instruction Nodes in order to generate the bit
2795 // encoding of the instruction. Operands specify their base encoding
2796 // interface with the interface keyword. There are currently
2797 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2798 // COND_INTER. REG_INTER causes an operand to generate a function
2799 // which returns its register number when queried. CONST_INTER causes
2800 // an operand to generate a function which returns the value of the
2801 // constant when queried. MEMORY_INTER causes an operand to generate
2802 // four functions which return the Base Register, the Index Register,
2803 // the Scale Value, and the Offset Value of the operand when queried.
2804 // COND_INTER causes an operand to generate six functions which return
2805 // the encoding code (ie - encoding bits for the instruction)
2806 // associated with each basic boolean condition for a conditional
2807 // instruction.
2808 //
2809 // Instructions specify two basic values for encoding. Again, a
2810 // function is available to check if the constant displacement is an
2811 // oop. They use the ins_encode keyword to specify their encoding
2812 // classes (which must be a sequence of enc_class names, and their
2813 // parameters, specified in the encoding block), and they use the
2814 // opcode keyword to specify, in order, their primary, secondary, and
2815 // tertiary opcode. Only the opcode sections which a particular
2816 // instruction needs for encoding need to be specified.
2817 encode %{
2818 // Build emit functions for each basic byte or larger field in the
2819 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2820 // from C++ code in the enc_class source block. Emit functions will
2821 // live in the main source block for now. In future, we can
2822 // generalize this by adding a syntax that specifies the sizes of
2823 // fields in an order, so that the adlc can build the emit functions
2824 // automagically
2825
2826 // catch all for unimplemented encodings
2827 enc_class enc_unimplemented %{
2828 C2_MacroAssembler _masm(&cbuf);
2829 __ unimplemented("C2 catch all");
2830 %}
2831
2832 // BEGIN Non-volatile memory access
2833
2834 // This encoding class is generated automatically from ad_encode.m4.
2835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2836 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2837 Register dst_reg = as_Register($dst$$reg);
2838 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2839 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2840 %}
2841
2842 // This encoding class is generated automatically from ad_encode.m4.
2843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2844 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2845 Register dst_reg = as_Register($dst$$reg);
2846 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2847 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2848 %}
2849
2850 // This encoding class is generated automatically from ad_encode.m4.
2851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2852 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2853 Register dst_reg = as_Register($dst$$reg);
2854 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2855 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2856 %}
2857
2858 // This encoding class is generated automatically from ad_encode.m4.
2859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2860 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2861 Register dst_reg = as_Register($dst$$reg);
2862 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2863 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2864 %}
2865
2866 // This encoding class is generated automatically from ad_encode.m4.
2867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2868 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2869 Register dst_reg = as_Register($dst$$reg);
2870 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2871 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2872 %}
2873
2874 // This encoding class is generated automatically from ad_encode.m4.
2875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2876 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2877 Register dst_reg = as_Register($dst$$reg);
2878 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2879 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2880 %}
2881
2882 // This encoding class is generated automatically from ad_encode.m4.
2883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2884 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2885 Register dst_reg = as_Register($dst$$reg);
2886 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2887 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2888 %}
2889
2890 // This encoding class is generated automatically from ad_encode.m4.
2891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2892 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2893 Register dst_reg = as_Register($dst$$reg);
2894 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2895 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2896 %}
2897
2898 // This encoding class is generated automatically from ad_encode.m4.
2899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2900 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2901 Register dst_reg = as_Register($dst$$reg);
2902 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2903 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2904 %}
2905
2906 // This encoding class is generated automatically from ad_encode.m4.
2907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2908 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2909 Register dst_reg = as_Register($dst$$reg);
2910 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2911 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2912 %}
2913
2914 // This encoding class is generated automatically from ad_encode.m4.
2915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2916 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2917 Register dst_reg = as_Register($dst$$reg);
2918 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2919 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2920 %}
2921
2922 // This encoding class is generated automatically from ad_encode.m4.
2923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2924 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2925 Register dst_reg = as_Register($dst$$reg);
2926 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2927 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2928 %}
2929
2930 // This encoding class is generated automatically from ad_encode.m4.
2931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2932 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2933 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2934 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2935 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2936 %}
2937
2938 // This encoding class is generated automatically from ad_encode.m4.
2939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2940 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2941 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2942 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2943 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2944 %}
2945
2946 // This encoding class is generated automatically from ad_encode.m4.
2947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2948 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2949 Register src_reg = as_Register($src$$reg);
2950 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2951 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2952 %}
2953
2954 // This encoding class is generated automatically from ad_encode.m4.
2955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2956 enc_class aarch64_enc_strb0(memory1 mem) %{
2957 C2_MacroAssembler _masm(&cbuf);
2958 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2959 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2960 %}
2961
2962 // This encoding class is generated automatically from ad_encode.m4.
2963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2964 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2965 Register src_reg = as_Register($src$$reg);
2966 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2967 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2968 %}
2969
2970 // This encoding class is generated automatically from ad_encode.m4.
2971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2972 enc_class aarch64_enc_strh0(memory2 mem) %{
2973 C2_MacroAssembler _masm(&cbuf);
2974 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2975 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2976 %}
2977
2978 // This encoding class is generated automatically from ad_encode.m4.
2979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2980 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2981 Register src_reg = as_Register($src$$reg);
2982 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2983 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2984 %}
2985
2986 // This encoding class is generated automatically from ad_encode.m4.
2987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2988 enc_class aarch64_enc_strw0(memory4 mem) %{
2989 C2_MacroAssembler _masm(&cbuf);
2990 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2991 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2992 %}
2993
2994 // This encoding class is generated automatically from ad_encode.m4.
2995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2996 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2997 Register src_reg = as_Register($src$$reg);
2998 // we sometimes get asked to store the stack pointer into the
2999 // current thread -- we cannot do that directly on AArch64
3000 if (src_reg == r31_sp) {
3001 C2_MacroAssembler _masm(&cbuf);
3002 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3003 __ mov(rscratch2, sp);
3004 src_reg = rscratch2;
3005 }
3006 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
3007 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3008 %}
3009
3010 // This encoding class is generated automatically from ad_encode.m4.
3011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3012 enc_class aarch64_enc_str0(memory8 mem) %{
3013 C2_MacroAssembler _masm(&cbuf);
3014 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
3015 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3016 %}
3017
3018 // This encoding class is generated automatically from ad_encode.m4.
3019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3020 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3021 FloatRegister src_reg = as_FloatRegister($src$$reg);
3022 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
3023 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3024 %}
3025
3026 // This encoding class is generated automatically from ad_encode.m4.
3027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3028 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3029 FloatRegister src_reg = as_FloatRegister($src$$reg);
3030 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
3031 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3032 %}
3033
3034 // This encoding class is generated automatically from ad_encode.m4.
3035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3036 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3037 C2_MacroAssembler _masm(&cbuf);
3038 __ membar(Assembler::StoreStore);
3039 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
3040 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3041 %}
3042
3043 // END Non-volatile memory access
3044
3045 // Vector loads and stores
3046 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3047 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3048 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3049 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3050 %}
3051
3052 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3053 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3054 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3055 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3056 %}
3057
3058 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3059 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3060 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3061 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3062 %}
3063
3064 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3065 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3066 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3067 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3068 %}
3069
3070 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3071 FloatRegister src_reg = as_FloatRegister($src$$reg);
3072 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
3073 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3074 %}
3075
3076 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3077 FloatRegister src_reg = as_FloatRegister($src$$reg);
3078 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
3079 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3080 %}
3081
3082 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3083 FloatRegister src_reg = as_FloatRegister($src$$reg);
3084 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
3085 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3086 %}
3087
3088 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3089 FloatRegister src_reg = as_FloatRegister($src$$reg);
3090 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
3091 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3092 %}
3093
3094 // volatile loads and stores
3095
3096 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3097 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3098 rscratch1, stlrb);
3099 %}
3100
3101 enc_class aarch64_enc_stlrb0(memory mem) %{
3102 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3103 rscratch1, stlrb);
3104 %}
3105
3106 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3107 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3108 rscratch1, stlrh);
3109 %}
3110
3111 enc_class aarch64_enc_stlrh0(memory mem) %{
3112 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3113 rscratch1, stlrh);
3114 %}
3115
3116 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3117 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3118 rscratch1, stlrw);
3119 %}
3120
3121 enc_class aarch64_enc_stlrw0(memory mem) %{
3122 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3123 rscratch1, stlrw);
3124 %}
3125
3126 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3127 Register dst_reg = as_Register($dst$$reg);
3128 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3129 rscratch1, ldarb);
3130 __ sxtbw(dst_reg, dst_reg);
3131 %}
3132
3133 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3134 Register dst_reg = as_Register($dst$$reg);
3135 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3136 rscratch1, ldarb);
3137 __ sxtb(dst_reg, dst_reg);
3138 %}
3139
3140 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3141 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3142 rscratch1, ldarb);
3143 %}
3144
3145 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3146 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3147 rscratch1, ldarb);
3148 %}
3149
3150 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3151 Register dst_reg = as_Register($dst$$reg);
3152 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3153 rscratch1, ldarh);
3154 __ sxthw(dst_reg, dst_reg);
3155 %}
3156
3157 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3158 Register dst_reg = as_Register($dst$$reg);
3159 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, ldarh);
3161 __ sxth(dst_reg, dst_reg);
3162 %}
3163
3164 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3165 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3166 rscratch1, ldarh);
3167 %}
3168
3169 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3170 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, ldarh);
3172 %}
3173
3174 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3175 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3176 rscratch1, ldarw);
3177 %}
3178
3179 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3180 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3181 rscratch1, ldarw);
3182 %}
3183
3184 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3185 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3186 rscratch1, ldar);
3187 %}
3188
3189 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3190 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3191 rscratch1, ldarw);
3192 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3193 %}
3194
3195 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3196 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3197 rscratch1, ldar);
3198 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3199 %}
3200
3201 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3202 Register src_reg = as_Register($src$$reg);
3203 // we sometimes get asked to store the stack pointer into the
3204 // current thread -- we cannot do that directly on AArch64
3205 if (src_reg == r31_sp) {
3206 C2_MacroAssembler _masm(&cbuf);
3207 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3208 __ mov(rscratch2, sp);
3209 src_reg = rscratch2;
3210 }
3211 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3212 rscratch1, stlr);
3213 %}
3214
3215 enc_class aarch64_enc_stlr0(memory mem) %{
3216 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3217 rscratch1, stlr);
3218 %}
3219
3220 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3221 {
3222 C2_MacroAssembler _masm(&cbuf);
3223 FloatRegister src_reg = as_FloatRegister($src$$reg);
3224 __ fmovs(rscratch2, src_reg);
3225 }
3226 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3227 rscratch1, stlrw);
3228 %}
3229
3230 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3231 {
3232 C2_MacroAssembler _masm(&cbuf);
3233 FloatRegister src_reg = as_FloatRegister($src$$reg);
3234 __ fmovd(rscratch2, src_reg);
3235 }
3236 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3237 rscratch1, stlr);
3238 %}
3239
3240 // synchronized read/update encodings
3241
3242 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3243 C2_MacroAssembler _masm(&cbuf);
3244 Register dst_reg = as_Register($dst$$reg);
3245 Register base = as_Register($mem$$base);
3246 int index = $mem$$index;
3247 int scale = $mem$$scale;
3248 int disp = $mem$$disp;
3249 if (index == -1) {
3250 if (disp != 0) {
3251 __ lea(rscratch1, Address(base, disp));
3252 __ ldaxr(dst_reg, rscratch1);
3253 } else {
3254 // TODO
3255 // should we ever get anything other than this case?
3256 __ ldaxr(dst_reg, base);
3257 }
3258 } else {
3259 Register index_reg = as_Register(index);
3260 if (disp == 0) {
3261 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3262 __ ldaxr(dst_reg, rscratch1);
3263 } else {
3264 __ lea(rscratch1, Address(base, disp));
3265 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3266 __ ldaxr(dst_reg, rscratch1);
3267 }
3268 }
3269 %}
3270
3271 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3272 C2_MacroAssembler _masm(&cbuf);
3273 Register src_reg = as_Register($src$$reg);
3274 Register base = as_Register($mem$$base);
3275 int index = $mem$$index;
3276 int scale = $mem$$scale;
3277 int disp = $mem$$disp;
3278 if (index == -1) {
3279 if (disp != 0) {
3280 __ lea(rscratch2, Address(base, disp));
3281 __ stlxr(rscratch1, src_reg, rscratch2);
3282 } else {
3283 // TODO
3284 // should we ever get anything other than this case?
3285 __ stlxr(rscratch1, src_reg, base);
3286 }
3287 } else {
3288 Register index_reg = as_Register(index);
3289 if (disp == 0) {
3290 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3291 __ stlxr(rscratch1, src_reg, rscratch2);
3292 } else {
3293 __ lea(rscratch2, Address(base, disp));
3294 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3295 __ stlxr(rscratch1, src_reg, rscratch2);
3296 }
3297 }
3298 __ cmpw(rscratch1, zr);
3299 %}
3300
3301 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3302 C2_MacroAssembler _masm(&cbuf);
3303 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3304 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3305 Assembler::xword, /*acquire*/ false, /*release*/ true,
3306 /*weak*/ false, noreg);
3307 %}
3308
3309 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3310 C2_MacroAssembler _masm(&cbuf);
3311 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3312 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3313 Assembler::word, /*acquire*/ false, /*release*/ true,
3314 /*weak*/ false, noreg);
3315 %}
3316
3317 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3318 C2_MacroAssembler _masm(&cbuf);
3319 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3320 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3321 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3322 /*weak*/ false, noreg);
3323 %}
3324
3325 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3326 C2_MacroAssembler _masm(&cbuf);
3327 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3328 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3329 Assembler::byte, /*acquire*/ false, /*release*/ true,
3330 /*weak*/ false, noreg);
3331 %}
3332
3333
3334 // The only difference between aarch64_enc_cmpxchg and
3335 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3336 // CompareAndSwap sequence to serve as a barrier on acquiring a
3337 // lock.
3338 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3339 C2_MacroAssembler _masm(&cbuf);
3340 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3341 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3342 Assembler::xword, /*acquire*/ true, /*release*/ true,
3343 /*weak*/ false, noreg);
3344 %}
3345
3346 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3347 C2_MacroAssembler _masm(&cbuf);
3348 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3349 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3350 Assembler::word, /*acquire*/ true, /*release*/ true,
3351 /*weak*/ false, noreg);
3352 %}
3353
3354 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3355 C2_MacroAssembler _masm(&cbuf);
3356 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3357 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3358 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3359 /*weak*/ false, noreg);
3360 %}
3361
3362 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3363 C2_MacroAssembler _masm(&cbuf);
3364 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3365 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3366 Assembler::byte, /*acquire*/ true, /*release*/ true,
3367 /*weak*/ false, noreg);
3368 %}
3369
3370 // auxiliary used for CompareAndSwapX to set result register
3371 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3372 C2_MacroAssembler _masm(&cbuf);
3373 Register res_reg = as_Register($res$$reg);
3374 __ cset(res_reg, Assembler::EQ);
3375 %}
3376
3377 // prefetch encodings
3378
3379 enc_class aarch64_enc_prefetchw(memory mem) %{
3380 C2_MacroAssembler _masm(&cbuf);
3381 Register base = as_Register($mem$$base);
3382 int index = $mem$$index;
3383 int scale = $mem$$scale;
3384 int disp = $mem$$disp;
3385 if (index == -1) {
3386 __ prfm(Address(base, disp), PSTL1KEEP);
3387 } else {
3388 Register index_reg = as_Register(index);
3389 if (disp == 0) {
3390 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3391 } else {
3392 __ lea(rscratch1, Address(base, disp));
3393 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3394 }
3395 }
3396 %}
3397
3398 /// mov envcodings
3399
3400 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3401 C2_MacroAssembler _masm(&cbuf);
3402 uint32_t con = (uint32_t)$src$$constant;
3403 Register dst_reg = as_Register($dst$$reg);
3404 if (con == 0) {
3405 __ movw(dst_reg, zr);
3406 } else {
3407 __ movw(dst_reg, con);
3408 }
3409 %}
3410
3411 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3412 C2_MacroAssembler _masm(&cbuf);
3413 Register dst_reg = as_Register($dst$$reg);
3414 uint64_t con = (uint64_t)$src$$constant;
3415 if (con == 0) {
3416 __ mov(dst_reg, zr);
3417 } else {
3418 __ mov(dst_reg, con);
3419 }
3420 %}
3421
3422 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3423 C2_MacroAssembler _masm(&cbuf);
3424 Register dst_reg = as_Register($dst$$reg);
3425 address con = (address)$src$$constant;
3426 if (con == NULL || con == (address)1) {
3427 ShouldNotReachHere();
3428 } else {
3429 relocInfo::relocType rtype = $src->constant_reloc();
3430 if (rtype == relocInfo::oop_type) {
3431 __ movoop(dst_reg, (jobject)con);
3432 } else if (rtype == relocInfo::metadata_type) {
3433 __ mov_metadata(dst_reg, (Metadata*)con);
3434 } else {
3435 assert(rtype == relocInfo::none, "unexpected reloc type");
3436 if (! __ is_valid_AArch64_address(con) ||
3437 con < (address)(uintptr_t)os::vm_page_size()) {
3438 __ mov(dst_reg, con);
3439 } else {
3440 uint64_t offset;
3441 __ adrp(dst_reg, con, offset);
3442 __ add(dst_reg, dst_reg, offset);
3443 }
3444 }
3445 }
3446 %}
3447
3448 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3449 C2_MacroAssembler _masm(&cbuf);
3450 Register dst_reg = as_Register($dst$$reg);
3451 __ mov(dst_reg, zr);
3452 %}
3453
3454 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3455 C2_MacroAssembler _masm(&cbuf);
3456 Register dst_reg = as_Register($dst$$reg);
3457 __ mov(dst_reg, (uint64_t)1);
3458 %}
3459
3460 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3461 C2_MacroAssembler _masm(&cbuf);
3462 __ load_byte_map_base($dst$$Register);
3463 %}
3464
3465 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3466 C2_MacroAssembler _masm(&cbuf);
3467 Register dst_reg = as_Register($dst$$reg);
3468 address con = (address)$src$$constant;
3469 if (con == NULL) {
3470 ShouldNotReachHere();
3471 } else {
3472 relocInfo::relocType rtype = $src->constant_reloc();
3473 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3474 __ set_narrow_oop(dst_reg, (jobject)con);
3475 }
3476 %}
3477
3478 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3479 C2_MacroAssembler _masm(&cbuf);
3480 Register dst_reg = as_Register($dst$$reg);
3481 __ mov(dst_reg, zr);
3482 %}
3483
3484 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3485 C2_MacroAssembler _masm(&cbuf);
3486 Register dst_reg = as_Register($dst$$reg);
3487 address con = (address)$src$$constant;
3488 if (con == NULL) {
3489 ShouldNotReachHere();
3490 } else {
3491 relocInfo::relocType rtype = $src->constant_reloc();
3492 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3493 __ set_narrow_klass(dst_reg, (Klass *)con);
3494 }
3495 %}
3496
3497 // arithmetic encodings
3498
3499 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3500 C2_MacroAssembler _masm(&cbuf);
3501 Register dst_reg = as_Register($dst$$reg);
3502 Register src_reg = as_Register($src1$$reg);
3503 int32_t con = (int32_t)$src2$$constant;
3504 // add has primary == 0, subtract has primary == 1
3505 if ($primary) { con = -con; }
3506 if (con < 0) {
3507 __ subw(dst_reg, src_reg, -con);
3508 } else {
3509 __ addw(dst_reg, src_reg, con);
3510 }
3511 %}
3512
3513 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3514 C2_MacroAssembler _masm(&cbuf);
3515 Register dst_reg = as_Register($dst$$reg);
3516 Register src_reg = as_Register($src1$$reg);
3517 int32_t con = (int32_t)$src2$$constant;
3518 // add has primary == 0, subtract has primary == 1
3519 if ($primary) { con = -con; }
3520 if (con < 0) {
3521 __ sub(dst_reg, src_reg, -con);
3522 } else {
3523 __ add(dst_reg, src_reg, con);
3524 }
3525 %}
3526
3527 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3528 C2_MacroAssembler _masm(&cbuf);
3529 Register dst_reg = as_Register($dst$$reg);
3530 Register src1_reg = as_Register($src1$$reg);
3531 Register src2_reg = as_Register($src2$$reg);
3532 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3533 %}
3534
3535 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3536 C2_MacroAssembler _masm(&cbuf);
3537 Register dst_reg = as_Register($dst$$reg);
3538 Register src1_reg = as_Register($src1$$reg);
3539 Register src2_reg = as_Register($src2$$reg);
3540 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3541 %}
3542
3543 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3544 C2_MacroAssembler _masm(&cbuf);
3545 Register dst_reg = as_Register($dst$$reg);
3546 Register src1_reg = as_Register($src1$$reg);
3547 Register src2_reg = as_Register($src2$$reg);
3548 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3549 %}
3550
3551 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3552 C2_MacroAssembler _masm(&cbuf);
3553 Register dst_reg = as_Register($dst$$reg);
3554 Register src1_reg = as_Register($src1$$reg);
3555 Register src2_reg = as_Register($src2$$reg);
3556 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3557 %}
3558
3559 // compare instruction encodings
3560
3561 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3562 C2_MacroAssembler _masm(&cbuf);
3563 Register reg1 = as_Register($src1$$reg);
3564 Register reg2 = as_Register($src2$$reg);
3565 __ cmpw(reg1, reg2);
3566 %}
3567
3568 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3569 C2_MacroAssembler _masm(&cbuf);
3570 Register reg = as_Register($src1$$reg);
3571 int32_t val = $src2$$constant;
3572 if (val >= 0) {
3573 __ subsw(zr, reg, val);
3574 } else {
3575 __ addsw(zr, reg, -val);
3576 }
3577 %}
3578
3579 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3580 C2_MacroAssembler _masm(&cbuf);
3581 Register reg1 = as_Register($src1$$reg);
3582 uint32_t val = (uint32_t)$src2$$constant;
3583 __ movw(rscratch1, val);
3584 __ cmpw(reg1, rscratch1);
3585 %}
3586
3587 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3588 C2_MacroAssembler _masm(&cbuf);
3589 Register reg1 = as_Register($src1$$reg);
3590 Register reg2 = as_Register($src2$$reg);
3591 __ cmp(reg1, reg2);
3592 %}
3593
3594 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3595 C2_MacroAssembler _masm(&cbuf);
3596 Register reg = as_Register($src1$$reg);
3597 int64_t val = $src2$$constant;
3598 if (val >= 0) {
3599 __ subs(zr, reg, val);
3600 } else if (val != -val) {
3601 __ adds(zr, reg, -val);
3602 } else {
3603 // aargh, Long.MIN_VALUE is a special case
3604 __ orr(rscratch1, zr, (uint64_t)val);
3605 __ subs(zr, reg, rscratch1);
3606 }
3607 %}
3608
3609 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3610 C2_MacroAssembler _masm(&cbuf);
3611 Register reg1 = as_Register($src1$$reg);
3612 uint64_t val = (uint64_t)$src2$$constant;
3613 __ mov(rscratch1, val);
3614 __ cmp(reg1, rscratch1);
3615 %}
3616
3617 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3618 C2_MacroAssembler _masm(&cbuf);
3619 Register reg1 = as_Register($src1$$reg);
3620 Register reg2 = as_Register($src2$$reg);
3621 __ cmp(reg1, reg2);
3622 %}
3623
3624 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3625 C2_MacroAssembler _masm(&cbuf);
3626 Register reg1 = as_Register($src1$$reg);
3627 Register reg2 = as_Register($src2$$reg);
3628 __ cmpw(reg1, reg2);
3629 %}
3630
3631 enc_class aarch64_enc_testp(iRegP src) %{
3632 C2_MacroAssembler _masm(&cbuf);
3633 Register reg = as_Register($src$$reg);
3634 __ cmp(reg, zr);
3635 %}
3636
3637 enc_class aarch64_enc_testn(iRegN src) %{
3638 C2_MacroAssembler _masm(&cbuf);
3639 Register reg = as_Register($src$$reg);
3640 __ cmpw(reg, zr);
3641 %}
3642
3643 enc_class aarch64_enc_b(label lbl) %{
3644 C2_MacroAssembler _masm(&cbuf);
3645 Label *L = $lbl$$label;
3646 __ b(*L);
3647 %}
3648
3649 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3650 C2_MacroAssembler _masm(&cbuf);
3651 Label *L = $lbl$$label;
3652 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3653 %}
3654
3655 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3656 C2_MacroAssembler _masm(&cbuf);
3657 Label *L = $lbl$$label;
3658 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3659 %}
3660
3661 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3662 %{
3663 Register sub_reg = as_Register($sub$$reg);
3664 Register super_reg = as_Register($super$$reg);
3665 Register temp_reg = as_Register($temp$$reg);
3666 Register result_reg = as_Register($result$$reg);
3667
3668 Label miss;
3669 C2_MacroAssembler _masm(&cbuf);
3670 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3671 NULL, &miss,
3672 /*set_cond_codes:*/ true);
3673 if ($primary) {
3674 __ mov(result_reg, zr);
3675 }
3676 __ bind(miss);
3677 %}
3678
3679 enc_class aarch64_enc_java_static_call(method meth) %{
3680 C2_MacroAssembler _masm(&cbuf);
3681
3682 address addr = (address)$meth$$method;
3683 address call;
3684 if (!_method) {
3685 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3686 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3687 if (call == NULL) {
3688 ciEnv::current()->record_failure("CodeCache is full");
3689 return;
3690 }
3691 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3692 // The NOP here is purely to ensure that eliding a call to
3693 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3694 __ nop();
3695 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3696 } else {
3697 int method_index = resolved_method_index(cbuf);
3698 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3699 : static_call_Relocation::spec(method_index);
3700 call = __ trampoline_call(Address(addr, rspec));
3701 if (call == NULL) {
3702 ciEnv::current()->record_failure("CodeCache is full");
3703 return;
3704 }
3705 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3706 // Calls of the same statically bound method can share
3707 // a stub to the interpreter.
3708 cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
3709 } else {
3710 // Emit stub for static call
3711 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
3712 if (stub == NULL) {
3713 ciEnv::current()->record_failure("CodeCache is full");
3714 return;
3715 }
3716 }
3717 }
3718
3719 __ post_call_nop();
3720
3721 // Only non uncommon_trap calls need to reinitialize ptrue.
3722 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3723 __ reinitialize_ptrue();
3724 }
3725 %}
3726
3727 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3728 C2_MacroAssembler _masm(&cbuf);
3729 int method_index = resolved_method_index(cbuf);
3730 address call = __ ic_call((address)$meth$$method, method_index);
3731 if (call == NULL) {
3732 ciEnv::current()->record_failure("CodeCache is full");
3733 return;
3734 }
3735 __ post_call_nop();
3736 if (Compile::current()->max_vector_size() > 0) {
3737 __ reinitialize_ptrue();
3738 }
3739 %}
3740
3741 enc_class aarch64_enc_call_epilog() %{
3742 C2_MacroAssembler _masm(&cbuf);
3743 if (VerifyStackAtCalls) {
3744 // Check that stack depth is unchanged: find majik cookie on stack
3745 __ call_Unimplemented();
3746 }
3747 %}
3748
3749 enc_class aarch64_enc_java_to_runtime(method meth) %{
3750 C2_MacroAssembler _masm(&cbuf);
3751
3752 // some calls to generated routines (arraycopy code) are scheduled
3753 // by C2 as runtime calls. if so we can call them using a br (they
3754 // will be in a reachable segment) otherwise we have to use a blr
3755 // which loads the absolute address into a register.
3756 address entry = (address)$meth$$method;
3757 CodeBlob *cb = CodeCache::find_blob(entry);
3758 if (cb) {
3759 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3760 if (call == NULL) {
3761 ciEnv::current()->record_failure("CodeCache is full");
3762 return;
3763 }
3764 __ post_call_nop();
3765 } else {
3766 Label retaddr;
3767 __ adr(rscratch2, retaddr);
3768 __ lea(rscratch1, RuntimeAddress(entry));
3769 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3770 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3771 __ blr(rscratch1);
3772 __ bind(retaddr);
3773 __ post_call_nop();
3774 __ add(sp, sp, 2 * wordSize);
3775 }
3776 if (Compile::current()->max_vector_size() > 0) {
3777 __ reinitialize_ptrue();
3778 }
3779 %}
3780
3781 enc_class aarch64_enc_rethrow() %{
3782 C2_MacroAssembler _masm(&cbuf);
3783 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3784 %}
3785
3786 enc_class aarch64_enc_ret() %{
3787 C2_MacroAssembler _masm(&cbuf);
3788 #ifdef ASSERT
3789 if (Compile::current()->max_vector_size() > 0) {
3790 __ verify_ptrue();
3791 }
3792 #endif
3793 __ ret(lr);
3794 %}
3795
3796 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3797 C2_MacroAssembler _masm(&cbuf);
3798 Register target_reg = as_Register($jump_target$$reg);
3799 __ br(target_reg);
3800 %}
3801
3802 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3803 C2_MacroAssembler _masm(&cbuf);
3804 Register target_reg = as_Register($jump_target$$reg);
3805 // exception oop should be in r0
3806 // ret addr has been popped into lr
3807 // callee expects it in r3
3808 __ mov(r3, lr);
3809 __ br(target_reg);
3810 %}
3811
3812 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3813 C2_MacroAssembler _masm(&cbuf);
3814 Register oop = as_Register($object$$reg);
3815 Register box = as_Register($box$$reg);
3816 Register disp_hdr = as_Register($tmp$$reg);
3817 Register tmp = as_Register($tmp2$$reg);
3818 Label cont;
3819 Label object_has_monitor;
3820 Label count, no_count;
3821
3822 assert_different_registers(oop, box, tmp, disp_hdr);
3823
3824 // Load markWord from object into displaced_header.
3825 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3826
3827 if (DiagnoseSyncOnValueBasedClasses != 0) {
3828 __ load_klass(tmp, oop);
3829 __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
3830 __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
3831 __ br(Assembler::NE, cont);
3832 }
3833
3834 // Check for existing monitor
3835 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3836
3837 if (LockingMode == LM_MONITOR) {
3838 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3839 __ b(cont);
3840 } else if (LockingMode == LM_LEGACY) {
3841 // Set tmp to be (markWord of object | UNLOCK_VALUE).
3842 __ orr(tmp, disp_hdr, markWord::unlocked_value);
3843
3844 // Initialize the box. (Must happen before we update the object mark!)
3845 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3846
3847 // Compare object markWord with an unlocked value (tmp) and if
3848 // equal exchange the stack address of our box with object markWord.
3849 // On failure disp_hdr contains the possibly locked markWord.
3850 __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3851 /*release*/ true, /*weak*/ false, disp_hdr);
3852 __ br(Assembler::EQ, cont);
3853
3854 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3855
3856 // If the compare-and-exchange succeeded, then we found an unlocked
3857 // object, will have now locked it will continue at label cont
3858
3859 // Check if the owner is self by comparing the value in the
3860 // markWord of object (disp_hdr) with the stack pointer.
3861 __ mov(rscratch1, sp);
3862 __ sub(disp_hdr, disp_hdr, rscratch1);
3863 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
3864 // If condition is true we are cont and hence we can store 0 as the
3865 // displaced header in the box, which indicates that it is a recursive lock.
3866 __ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
3867 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3868 __ b(cont);
3869 } else {
3870 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
3871 __ fast_lock(oop, disp_hdr, tmp, rscratch1, no_count);
3872 __ b(count);
3873 }
3874
3875 // Handle existing monitor.
3876 __ bind(object_has_monitor);
3877
3878 // The object's monitor m is unlocked iff m->owner == NULL,
3879 // otherwise m->owner may contain a thread or a stack address.
3880 //
3881 // Try to CAS m->owner from NULL to current thread.
3882 __ add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value));
3883 __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3884 /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
3885
3886 if (LockingMode != LM_LIGHTWEIGHT) {
3887 // Store a non-null value into the box to avoid looking like a re-entrant
3888 // lock. The fast-path monitor unlock code checks for
3889 // markWord::monitor_value so use markWord::unused_mark which has the
3890 // relevant bit set, and also matches ObjectSynchronizer::enter.
3891 __ mov(tmp, (address)markWord::unused_mark().value());
3892 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3893 }
3894 __ br(Assembler::EQ, cont); // CAS success means locking succeeded
3895
3896 __ cmp(rscratch1, rthread);
3897 __ br(Assembler::NE, cont); // Check for recursive locking
3898
3899 // Recursive lock case
3900 __ increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1);
3901 // flag == EQ still from the cmp above, checking if this is a reentrant lock
3902
3903 __ bind(cont);
3904 // flag == EQ indicates success
3905 // flag == NE indicates failure
3906 __ br(Assembler::NE, no_count);
3907
3908 __ bind(count);
3909 __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
3910
3911 __ bind(no_count);
3912 %}
3913
3914 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3915 C2_MacroAssembler _masm(&cbuf);
3916 Register oop = as_Register($object$$reg);
3917 Register box = as_Register($box$$reg);
3918 Register disp_hdr = as_Register($tmp$$reg);
3919 Register tmp = as_Register($tmp2$$reg);
3920 Label cont;
3921 Label object_has_monitor;
3922 Label count, no_count;
3923
3924 assert_different_registers(oop, box, tmp, disp_hdr);
3925
3926 if (LockingMode == LM_LEGACY) {
3927 // Find the lock address and load the displaced header from the stack.
3928 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3929
3930 // If the displaced header is 0, we have a recursive unlock.
3931 __ cmp(disp_hdr, zr);
3932 __ br(Assembler::EQ, cont);
3933 }
3934
3935 // Handle existing monitor.
3936 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3937 __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
3938
3939 if (LockingMode == LM_MONITOR) {
3940 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3941 __ b(cont);
3942 } else if (LockingMode == LM_LEGACY) {
3943 // Check if it is still a light weight lock, this is is true if we
3944 // see the stack address of the basicLock in the markWord of the
3945 // object.
3946
3947 __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
3948 /*release*/ true, /*weak*/ false, tmp);
3949 __ b(cont);
3950 } else {
3951 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
3952 __ fast_unlock(oop, tmp, box, disp_hdr, no_count);
3953 __ b(count);
3954 }
3955
3956 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3957
3958 // Handle existing monitor.
3959 __ bind(object_has_monitor);
3960 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
3961 __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
3962
3963 if (LockingMode == LM_LIGHTWEIGHT) {
3964 // If the owner is anonymous, we need to fix it -- in an outline stub.
3965 Register tmp2 = disp_hdr;
3966 __ ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset()));
3967 // We cannot use tbnz here, the target might be too far away and cannot
3968 // be encoded.
3969 __ tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER);
3970 C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
3971 Compile::current()->output()->add_stub(stub);
3972 __ br(Assembler::NE, stub->entry());
3973 __ bind(stub->continuation());
3974 }
3975
3976 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
3977
3978 Label notRecursive;
3979 __ cbz(disp_hdr, notRecursive);
3980
3981 // Recursive lock
3982 __ sub(disp_hdr, disp_hdr, 1u);
3983 __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
3984 __ cmp(disp_hdr, disp_hdr); // Sets flags for result
3985 __ b(cont);
3986
3987 __ bind(notRecursive);
3988 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset()));
3989 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
3990 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3991 __ cmp(rscratch1, zr); // Sets flags for result
3992 __ cbnz(rscratch1, cont);
3993 // need a release store here
3994 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset()));
3995 __ stlr(zr, tmp); // set unowned
3996
3997 __ bind(cont);
3998 // flag == EQ indicates success
3999 // flag == NE indicates failure
4000 __ br(Assembler::NE, no_count);
4001
4002 __ bind(count);
4003 __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
4004
4005 __ bind(no_count);
4006 %}
4007
4008 %}
4009
4010 //----------FRAME--------------------------------------------------------------
4011 // Definition of frame structure and management information.
4012 //
4013 // S T A C K L A Y O U T Allocators stack-slot number
4014 // | (to get allocators register number
4015 // G Owned by | | v add OptoReg::stack0())
4016 // r CALLER | |
4017 // o | +--------+ pad to even-align allocators stack-slot
4018 // w V | pad0 | numbers; owned by CALLER
4019 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
4020 // h ^ | in | 5
4021 // | | args | 4 Holes in incoming args owned by SELF
4022 // | | | | 3
4023 // | | +--------+
4024 // V | | old out| Empty on Intel, window on Sparc
4025 // | old |preserve| Must be even aligned.
4026 // | SP-+--------+----> Matcher::_old_SP, even aligned
4027 // | | in | 3 area for Intel ret address
4028 // Owned by |preserve| Empty on Sparc.
4029 // SELF +--------+
4030 // | | pad2 | 2 pad to align old SP
4031 // | +--------+ 1
4032 // | | locks | 0
4033 // | +--------+----> OptoReg::stack0(), even aligned
4034 // | | pad1 | 11 pad to align new SP
4035 // | +--------+
4036 // | | | 10
4037 // | | spills | 9 spills
4038 // V | | 8 (pad0 slot for callee)
4039 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
4040 // ^ | out | 7
4041 // | | args | 6 Holes in outgoing args owned by CALLEE
4042 // Owned by +--------+
4043 // CALLEE | new out| 6 Empty on Intel, window on Sparc
4044 // | new |preserve| Must be even-aligned.
4045 // | SP-+--------+----> Matcher::_new_SP, even aligned
4046 // | | |
4047 //
4048 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
4049 // known from SELF's arguments and the Java calling convention.
4050 // Region 6-7 is determined per call site.
4051 // Note 2: If the calling convention leaves holes in the incoming argument
4052 // area, those holes are owned by SELF. Holes in the outgoing area
4053 // are owned by the CALLEE. Holes should not be necessary in the
4054 // incoming area, as the Java calling convention is completely under
4055 // the control of the AD file. Doubles can be sorted and packed to
4056 // avoid holes. Holes in the outgoing arguments may be necessary for
4057 // varargs C calling conventions.
4058 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
4059 // even aligned with pad0 as needed.
4060 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
4061 // (the latter is true on Intel but is it false on AArch64?)
4062 // region 6-11 is even aligned; it may be padded out more so that
4063 // the region from SP to FP meets the minimum stack alignment.
4064 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
4065 // alignment. Region 11, pad1, may be dynamically extended so that
4066 // SP meets the minimum alignment.
4067
4068 frame %{
4069 // These three registers define part of the calling convention
4070 // between compiled code and the interpreter.
4071
4072 // Inline Cache Register or Method for I2C.
4073 inline_cache_reg(R12);
4074
4075 // Number of stack slots consumed by locking an object
4076 sync_stack_slots(2);
4077
4078 // Compiled code's Frame Pointer
4079 frame_pointer(R31);
4080
4081 // Interpreter stores its frame pointer in a register which is
4082 // stored to the stack by I2CAdaptors.
4083 // I2CAdaptors convert from interpreted java to compiled java.
4084 interpreter_frame_pointer(R29);
4085
4086 // Stack alignment requirement
4087 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4088
4089 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4090 // for calls to C. Supports the var-args backing area for register parms.
4091 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4092
4093 // The after-PROLOG location of the return address. Location of
4094 // return address specifies a type (REG or STACK) and a number
4095 // representing the register number (i.e. - use a register name) or
4096 // stack slot.
4097 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4098 // Otherwise, it is above the locks and verification slot and alignment word
4099 // TODO this may well be correct but need to check why that - 2 is there
4100 // ppc port uses 0 but we definitely need to allow for fixed_slots
4101 // which folds in the space used for monitors
4102 return_addr(STACK - 2 +
4103 align_up((Compile::current()->in_preserve_stack_slots() +
4104 Compile::current()->fixed_slots()),
4105 stack_alignment_in_slots()));
4106
4107 // Location of compiled Java return values. Same as C for now.
4108 return_value
4109 %{
4110 // TODO do we allow ideal_reg == Op_RegN???
4111 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4112 "only return normal values");
4113
4114 static const int lo[Op_RegL + 1] = { // enum name
4115 0, // Op_Node
4116 0, // Op_Set
4117 R0_num, // Op_RegN
4118 R0_num, // Op_RegI
4119 R0_num, // Op_RegP
4120 V0_num, // Op_RegF
4121 V0_num, // Op_RegD
4122 R0_num // Op_RegL
4123 };
4124
4125 static const int hi[Op_RegL + 1] = { // enum name
4126 0, // Op_Node
4127 0, // Op_Set
4128 OptoReg::Bad, // Op_RegN
4129 OptoReg::Bad, // Op_RegI
4130 R0_H_num, // Op_RegP
4131 OptoReg::Bad, // Op_RegF
4132 V0_H_num, // Op_RegD
4133 R0_H_num // Op_RegL
4134 };
4135
4136 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4137 %}
4138 %}
4139
4140 //----------ATTRIBUTES---------------------------------------------------------
4141 //----------Operand Attributes-------------------------------------------------
4142 op_attrib op_cost(1); // Required cost attribute
4143
4144 //----------Instruction Attributes---------------------------------------------
4145 ins_attrib ins_cost(INSN_COST); // Required cost attribute
4146 ins_attrib ins_size(32); // Required size attribute (in bits)
4147 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4148 // a non-matching short branch variant
4149 // of some long branch?
4150 ins_attrib ins_alignment(4); // Required alignment attribute (must
4151 // be a power of 2) specifies the
4152 // alignment that some part of the
4153 // instruction (not necessarily the
4154 // start) requires. If > 1, a
4155 // compute_padding() function must be
4156 // provided for the instruction
4157
4158 //----------OPERANDS-----------------------------------------------------------
4159 // Operand definitions must precede instruction definitions for correct parsing
4160 // in the ADLC because operands constitute user defined types which are used in
4161 // instruction definitions.
4162
4163 //----------Simple Operands----------------------------------------------------
4164
4165 // Integer operands 32 bit
4166 // 32 bit immediate
4167 operand immI()
4168 %{
4169 match(ConI);
4170
4171 op_cost(0);
4172 format %{ %}
4173 interface(CONST_INTER);
4174 %}
4175
4176 // 32 bit zero
4177 operand immI0()
4178 %{
4179 predicate(n->get_int() == 0);
4180 match(ConI);
4181
4182 op_cost(0);
4183 format %{ %}
4184 interface(CONST_INTER);
4185 %}
4186
4187 // 32 bit unit increment
4188 operand immI_1()
4189 %{
4190 predicate(n->get_int() == 1);
4191 match(ConI);
4192
4193 op_cost(0);
4194 format %{ %}
4195 interface(CONST_INTER);
4196 %}
4197
4198 // 32 bit unit decrement
4199 operand immI_M1()
4200 %{
4201 predicate(n->get_int() == -1);
4202 match(ConI);
4203
4204 op_cost(0);
4205 format %{ %}
4206 interface(CONST_INTER);
4207 %}
4208
4209 // Shift values for add/sub extension shift
4210 operand immIExt()
4211 %{
4212 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4213 match(ConI);
4214
4215 op_cost(0);
4216 format %{ %}
4217 interface(CONST_INTER);
4218 %}
4219
4220 operand immI_gt_1()
4221 %{
4222 predicate(n->get_int() > 1);
4223 match(ConI);
4224
4225 op_cost(0);
4226 format %{ %}
4227 interface(CONST_INTER);
4228 %}
4229
4230 operand immI_le_4()
4231 %{
4232 predicate(n->get_int() <= 4);
4233 match(ConI);
4234
4235 op_cost(0);
4236 format %{ %}
4237 interface(CONST_INTER);
4238 %}
4239
4240 operand immI_16()
4241 %{
4242 predicate(n->get_int() == 16);
4243 match(ConI);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immI_24()
4251 %{
4252 predicate(n->get_int() == 24);
4253 match(ConI);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immI_32()
4261 %{
4262 predicate(n->get_int() == 32);
4263 match(ConI);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immI_48()
4271 %{
4272 predicate(n->get_int() == 48);
4273 match(ConI);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 operand immI_56()
4281 %{
4282 predicate(n->get_int() == 56);
4283 match(ConI);
4284
4285 op_cost(0);
4286 format %{ %}
4287 interface(CONST_INTER);
4288 %}
4289
4290 operand immI_63()
4291 %{
4292 predicate(n->get_int() == 63);
4293 match(ConI);
4294
4295 op_cost(0);
4296 format %{ %}
4297 interface(CONST_INTER);
4298 %}
4299
4300 operand immI_64()
4301 %{
4302 predicate(n->get_int() == 64);
4303 match(ConI);
4304
4305 op_cost(0);
4306 format %{ %}
4307 interface(CONST_INTER);
4308 %}
4309
4310 operand immI_255()
4311 %{
4312 predicate(n->get_int() == 255);
4313 match(ConI);
4314
4315 op_cost(0);
4316 format %{ %}
4317 interface(CONST_INTER);
4318 %}
4319
4320 operand immI_65535()
4321 %{
4322 predicate(n->get_int() == 65535);
4323 match(ConI);
4324
4325 op_cost(0);
4326 format %{ %}
4327 interface(CONST_INTER);
4328 %}
4329
4330 operand immI_positive()
4331 %{
4332 predicate(n->get_int() > 0);
4333 match(ConI);
4334
4335 op_cost(0);
4336 format %{ %}
4337 interface(CONST_INTER);
4338 %}
4339
4340 // BoolTest condition for signed compare
4341 operand immI_cmp_cond()
4342 %{
4343 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4344 match(ConI);
4345
4346 op_cost(0);
4347 format %{ %}
4348 interface(CONST_INTER);
4349 %}
4350
4351 // BoolTest condition for unsigned compare
4352 operand immI_cmpU_cond()
4353 %{
4354 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4355 match(ConI);
4356
4357 op_cost(0);
4358 format %{ %}
4359 interface(CONST_INTER);
4360 %}
4361
4362 operand immL_255()
4363 %{
4364 predicate(n->get_long() == 255L);
4365 match(ConL);
4366
4367 op_cost(0);
4368 format %{ %}
4369 interface(CONST_INTER);
4370 %}
4371
4372 operand immL_65535()
4373 %{
4374 predicate(n->get_long() == 65535L);
4375 match(ConL);
4376
4377 op_cost(0);
4378 format %{ %}
4379 interface(CONST_INTER);
4380 %}
4381
4382 operand immL_4294967295()
4383 %{
4384 predicate(n->get_long() == 4294967295L);
4385 match(ConL);
4386
4387 op_cost(0);
4388 format %{ %}
4389 interface(CONST_INTER);
4390 %}
4391
4392 operand immL_bitmask()
4393 %{
4394 predicate((n->get_long() != 0)
4395 && ((n->get_long() & 0xc000000000000000l) == 0)
4396 && is_power_of_2(n->get_long() + 1));
4397 match(ConL);
4398
4399 op_cost(0);
4400 format %{ %}
4401 interface(CONST_INTER);
4402 %}
4403
4404 operand immI_bitmask()
4405 %{
4406 predicate((n->get_int() != 0)
4407 && ((n->get_int() & 0xc0000000) == 0)
4408 && is_power_of_2(n->get_int() + 1));
4409 match(ConI);
4410
4411 op_cost(0);
4412 format %{ %}
4413 interface(CONST_INTER);
4414 %}
4415
4416 operand immL_positive_bitmaskI()
4417 %{
4418 predicate((n->get_long() != 0)
4419 && ((julong)n->get_long() < 0x80000000ULL)
4420 && is_power_of_2(n->get_long() + 1));
4421 match(ConL);
4422
4423 op_cost(0);
4424 format %{ %}
4425 interface(CONST_INTER);
4426 %}
4427
4428 // Scale values for scaled offset addressing modes (up to long but not quad)
4429 operand immIScale()
4430 %{
4431 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4432 match(ConI);
4433
4434 op_cost(0);
4435 format %{ %}
4436 interface(CONST_INTER);
4437 %}
4438
4439 // 26 bit signed offset -- for pc-relative branches
4440 operand immI26()
4441 %{
4442 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4443 match(ConI);
4444
4445 op_cost(0);
4446 format %{ %}
4447 interface(CONST_INTER);
4448 %}
4449
4450 // 19 bit signed offset -- for pc-relative loads
4451 operand immI19()
4452 %{
4453 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4454 match(ConI);
4455
4456 op_cost(0);
4457 format %{ %}
4458 interface(CONST_INTER);
4459 %}
4460
4461 // 5 bit signed integer
4462 operand immI5()
4463 %{
4464 predicate(Assembler::is_simm(n->get_int(), 5));
4465 match(ConI);
4466
4467 op_cost(0);
4468 format %{ %}
4469 interface(CONST_INTER);
4470 %}
4471
4472 // 7 bit unsigned integer
4473 operand immIU7()
4474 %{
4475 predicate(Assembler::is_uimm(n->get_int(), 7));
4476 match(ConI);
4477
4478 op_cost(0);
4479 format %{ %}
4480 interface(CONST_INTER);
4481 %}
4482
4483 // 12 bit unsigned offset -- for base plus immediate loads
4484 operand immIU12()
4485 %{
4486 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4487 match(ConI);
4488
4489 op_cost(0);
4490 format %{ %}
4491 interface(CONST_INTER);
4492 %}
4493
4494 operand immLU12()
4495 %{
4496 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4497 match(ConL);
4498
4499 op_cost(0);
4500 format %{ %}
4501 interface(CONST_INTER);
4502 %}
4503
4504 // Offset for scaled or unscaled immediate loads and stores
4505 operand immIOffset()
4506 %{
4507 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4508 match(ConI);
4509
4510 op_cost(0);
4511 format %{ %}
4512 interface(CONST_INTER);
4513 %}
4514
4515 operand immIOffset1()
4516 %{
4517 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4518 match(ConI);
4519
4520 op_cost(0);
4521 format %{ %}
4522 interface(CONST_INTER);
4523 %}
4524
4525 operand immIOffset2()
4526 %{
4527 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4528 match(ConI);
4529
4530 op_cost(0);
4531 format %{ %}
4532 interface(CONST_INTER);
4533 %}
4534
4535 operand immIOffset4()
4536 %{
4537 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4538 match(ConI);
4539
4540 op_cost(0);
4541 format %{ %}
4542 interface(CONST_INTER);
4543 %}
4544
4545 operand immIOffset8()
4546 %{
4547 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4548 match(ConI);
4549
4550 op_cost(0);
4551 format %{ %}
4552 interface(CONST_INTER);
4553 %}
4554
4555 operand immIOffset16()
4556 %{
4557 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4558 match(ConI);
4559
4560 op_cost(0);
4561 format %{ %}
4562 interface(CONST_INTER);
4563 %}
4564
4565 operand immLoffset()
4566 %{
4567 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4568 match(ConL);
4569
4570 op_cost(0);
4571 format %{ %}
4572 interface(CONST_INTER);
4573 %}
4574
4575 operand immLoffset1()
4576 %{
4577 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4578 match(ConL);
4579
4580 op_cost(0);
4581 format %{ %}
4582 interface(CONST_INTER);
4583 %}
4584
4585 operand immLoffset2()
4586 %{
4587 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4588 match(ConL);
4589
4590 op_cost(0);
4591 format %{ %}
4592 interface(CONST_INTER);
4593 %}
4594
4595 operand immLoffset4()
4596 %{
4597 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4598 match(ConL);
4599
4600 op_cost(0);
4601 format %{ %}
4602 interface(CONST_INTER);
4603 %}
4604
4605 operand immLoffset8()
4606 %{
4607 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4608 match(ConL);
4609
4610 op_cost(0);
4611 format %{ %}
4612 interface(CONST_INTER);
4613 %}
4614
4615 operand immLoffset16()
4616 %{
4617 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4618 match(ConL);
4619
4620 op_cost(0);
4621 format %{ %}
4622 interface(CONST_INTER);
4623 %}
4624
4625 // 5 bit signed long integer
4626 operand immL5()
4627 %{
4628 predicate(Assembler::is_simm(n->get_long(), 5));
4629 match(ConL);
4630
4631 op_cost(0);
4632 format %{ %}
4633 interface(CONST_INTER);
4634 %}
4635
4636 // 7 bit unsigned long integer
4637 operand immLU7()
4638 %{
4639 predicate(Assembler::is_uimm(n->get_long(), 7));
4640 match(ConL);
4641
4642 op_cost(0);
4643 format %{ %}
4644 interface(CONST_INTER);
4645 %}
4646
4647 // 8 bit signed value.
4648 operand immI8()
4649 %{
4650 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4651 match(ConI);
4652
4653 op_cost(0);
4654 format %{ %}
4655 interface(CONST_INTER);
4656 %}
4657
4658 // 8 bit signed value (simm8), or #simm8 LSL 8.
4659 operand immI8_shift8()
4660 %{
4661 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4662 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4663 match(ConI);
4664
4665 op_cost(0);
4666 format %{ %}
4667 interface(CONST_INTER);
4668 %}
4669
4670 // 8 bit signed value (simm8), or #simm8 LSL 8.
4671 operand immL8_shift8()
4672 %{
4673 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4674 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4675 match(ConL);
4676
4677 op_cost(0);
4678 format %{ %}
4679 interface(CONST_INTER);
4680 %}
4681
4682 // 8 bit integer valid for vector add sub immediate
4683 operand immBAddSubV()
4684 %{
4685 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4686 match(ConI);
4687
4688 op_cost(0);
4689 format %{ %}
4690 interface(CONST_INTER);
4691 %}
4692
4693 // 32 bit integer valid for add sub immediate
4694 operand immIAddSub()
4695 %{
4696 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4697 match(ConI);
4698 op_cost(0);
4699 format %{ %}
4700 interface(CONST_INTER);
4701 %}
4702
4703 // 32 bit integer valid for vector add sub immediate
4704 operand immIAddSubV()
4705 %{
4706 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4707 match(ConI);
4708
4709 op_cost(0);
4710 format %{ %}
4711 interface(CONST_INTER);
4712 %}
4713
4714 // 32 bit unsigned integer valid for logical immediate
4715
4716 operand immBLog()
4717 %{
4718 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4719 match(ConI);
4720
4721 op_cost(0);
4722 format %{ %}
4723 interface(CONST_INTER);
4724 %}
4725
4726 operand immSLog()
4727 %{
4728 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4729 match(ConI);
4730
4731 op_cost(0);
4732 format %{ %}
4733 interface(CONST_INTER);
4734 %}
4735
4736 operand immILog()
4737 %{
4738 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4739 match(ConI);
4740
4741 op_cost(0);
4742 format %{ %}
4743 interface(CONST_INTER);
4744 %}
4745
4746 // Integer operands 64 bit
4747 // 64 bit immediate
4748 operand immL()
4749 %{
4750 match(ConL);
4751
4752 op_cost(0);
4753 format %{ %}
4754 interface(CONST_INTER);
4755 %}
4756
4757 // 64 bit zero
4758 operand immL0()
4759 %{
4760 predicate(n->get_long() == 0);
4761 match(ConL);
4762
4763 op_cost(0);
4764 format %{ %}
4765 interface(CONST_INTER);
4766 %}
4767
4768 // 64 bit unit increment
4769 operand immL_1()
4770 %{
4771 predicate(n->get_long() == 1);
4772 match(ConL);
4773
4774 op_cost(0);
4775 format %{ %}
4776 interface(CONST_INTER);
4777 %}
4778
4779 // 64 bit unit decrement
4780 operand immL_M1()
4781 %{
4782 predicate(n->get_long() == -1);
4783 match(ConL);
4784
4785 op_cost(0);
4786 format %{ %}
4787 interface(CONST_INTER);
4788 %}
4789
4790 // 32 bit offset of pc in thread anchor
4791
4792 operand immL_pc_off()
4793 %{
4794 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4795 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4796 match(ConL);
4797
4798 op_cost(0);
4799 format %{ %}
4800 interface(CONST_INTER);
4801 %}
4802
4803 // 64 bit integer valid for add sub immediate
4804 operand immLAddSub()
4805 %{
4806 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4807 match(ConL);
4808 op_cost(0);
4809 format %{ %}
4810 interface(CONST_INTER);
4811 %}
4812
4813 // 64 bit integer valid for addv subv immediate
4814 operand immLAddSubV()
4815 %{
4816 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4817 match(ConL);
4818
4819 op_cost(0);
4820 format %{ %}
4821 interface(CONST_INTER);
4822 %}
4823
4824 // 64 bit integer valid for logical immediate
4825 operand immLLog()
4826 %{
4827 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4828 match(ConL);
4829 op_cost(0);
4830 format %{ %}
4831 interface(CONST_INTER);
4832 %}
4833
4834 // Long Immediate: low 32-bit mask
4835 operand immL_32bits()
4836 %{
4837 predicate(n->get_long() == 0xFFFFFFFFL);
4838 match(ConL);
4839 op_cost(0);
4840 format %{ %}
4841 interface(CONST_INTER);
4842 %}
4843
4844 // Pointer operands
4845 // Pointer Immediate
4846 operand immP()
4847 %{
4848 match(ConP);
4849
4850 op_cost(0);
4851 format %{ %}
4852 interface(CONST_INTER);
4853 %}
4854
4855 // NULL Pointer Immediate
4856 operand immP0()
4857 %{
4858 predicate(n->get_ptr() == 0);
4859 match(ConP);
4860
4861 op_cost(0);
4862 format %{ %}
4863 interface(CONST_INTER);
4864 %}
4865
4866 // Pointer Immediate One
4867 // this is used in object initialization (initial object header)
4868 operand immP_1()
4869 %{
4870 predicate(n->get_ptr() == 1);
4871 match(ConP);
4872
4873 op_cost(0);
4874 format %{ %}
4875 interface(CONST_INTER);
4876 %}
4877
4878 // Card Table Byte Map Base
4879 operand immByteMapBase()
4880 %{
4881 // Get base of card map
4882 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4883 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4884 match(ConP);
4885
4886 op_cost(0);
4887 format %{ %}
4888 interface(CONST_INTER);
4889 %}
4890
4891 // Pointer Immediate Minus One
4892 // this is used when we want to write the current PC to the thread anchor
4893 operand immP_M1()
4894 %{
4895 predicate(n->get_ptr() == -1);
4896 match(ConP);
4897
4898 op_cost(0);
4899 format %{ %}
4900 interface(CONST_INTER);
4901 %}
4902
4903 // Pointer Immediate Minus Two
4904 // this is used when we want to write the current PC to the thread anchor
4905 operand immP_M2()
4906 %{
4907 predicate(n->get_ptr() == -2);
4908 match(ConP);
4909
4910 op_cost(0);
4911 format %{ %}
4912 interface(CONST_INTER);
4913 %}
4914
4915 // Float and Double operands
4916 // Double Immediate
4917 operand immD()
4918 %{
4919 match(ConD);
4920 op_cost(0);
4921 format %{ %}
4922 interface(CONST_INTER);
4923 %}
4924
4925 // Double Immediate: +0.0d
4926 operand immD0()
4927 %{
4928 predicate(jlong_cast(n->getd()) == 0);
4929 match(ConD);
4930
4931 op_cost(0);
4932 format %{ %}
4933 interface(CONST_INTER);
4934 %}
4935
4936 // constant 'double +0.0'.
4937 operand immDPacked()
4938 %{
4939 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4940 match(ConD);
4941 op_cost(0);
4942 format %{ %}
4943 interface(CONST_INTER);
4944 %}
4945
4946 // Float Immediate
4947 operand immF()
4948 %{
4949 match(ConF);
4950 op_cost(0);
4951 format %{ %}
4952 interface(CONST_INTER);
4953 %}
4954
4955 // Float Immediate: +0.0f.
4956 operand immF0()
4957 %{
4958 predicate(jint_cast(n->getf()) == 0);
4959 match(ConF);
4960
4961 op_cost(0);
4962 format %{ %}
4963 interface(CONST_INTER);
4964 %}
4965
4966 //
4967 operand immFPacked()
4968 %{
4969 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4970 match(ConF);
4971 op_cost(0);
4972 format %{ %}
4973 interface(CONST_INTER);
4974 %}
4975
4976 // Narrow pointer operands
4977 // Narrow Pointer Immediate
4978 operand immN()
4979 %{
4980 match(ConN);
4981
4982 op_cost(0);
4983 format %{ %}
4984 interface(CONST_INTER);
4985 %}
4986
4987 // Narrow NULL Pointer Immediate
4988 operand immN0()
4989 %{
4990 predicate(n->get_narrowcon() == 0);
4991 match(ConN);
4992
4993 op_cost(0);
4994 format %{ %}
4995 interface(CONST_INTER);
4996 %}
4997
4998 operand immNKlass()
4999 %{
5000 match(ConNKlass);
5001
5002 op_cost(0);
5003 format %{ %}
5004 interface(CONST_INTER);
5005 %}
5006
5007 // Integer 32 bit Register Operands
5008 // Integer 32 bitRegister (excludes SP)
5009 operand iRegI()
5010 %{
5011 constraint(ALLOC_IN_RC(any_reg32));
5012 match(RegI);
5013 match(iRegINoSp);
5014 op_cost(0);
5015 format %{ %}
5016 interface(REG_INTER);
5017 %}
5018
5019 // Integer 32 bit Register not Special
5020 operand iRegINoSp()
5021 %{
5022 constraint(ALLOC_IN_RC(no_special_reg32));
5023 match(RegI);
5024 op_cost(0);
5025 format %{ %}
5026 interface(REG_INTER);
5027 %}
5028
5029 // Integer 64 bit Register Operands
5030 // Integer 64 bit Register (includes SP)
5031 operand iRegL()
5032 %{
5033 constraint(ALLOC_IN_RC(any_reg));
5034 match(RegL);
5035 match(iRegLNoSp);
5036 op_cost(0);
5037 format %{ %}
5038 interface(REG_INTER);
5039 %}
5040
5041 // Integer 64 bit Register not Special
5042 operand iRegLNoSp()
5043 %{
5044 constraint(ALLOC_IN_RC(no_special_reg));
5045 match(RegL);
5046 match(iRegL_R0);
5047 format %{ %}
5048 interface(REG_INTER);
5049 %}
5050
5051 // Pointer Register Operands
5052 // Pointer Register
5053 operand iRegP()
5054 %{
5055 constraint(ALLOC_IN_RC(ptr_reg));
5056 match(RegP);
5057 match(iRegPNoSp);
5058 match(iRegP_R0);
5059 //match(iRegP_R2);
5060 //match(iRegP_R4);
5061 match(iRegP_R5);
5062 match(thread_RegP);
5063 op_cost(0);
5064 format %{ %}
5065 interface(REG_INTER);
5066 %}
5067
5068 // Pointer 64 bit Register not Special
5069 operand iRegPNoSp()
5070 %{
5071 constraint(ALLOC_IN_RC(no_special_ptr_reg));
5072 match(RegP);
5073 // match(iRegP);
5074 // match(iRegP_R0);
5075 // match(iRegP_R2);
5076 // match(iRegP_R4);
5077 // match(iRegP_R5);
5078 // match(thread_RegP);
5079 op_cost(0);
5080 format %{ %}
5081 interface(REG_INTER);
5082 %}
5083
5084 // Pointer 64 bit Register R0 only
5085 operand iRegP_R0()
5086 %{
5087 constraint(ALLOC_IN_RC(r0_reg));
5088 match(RegP);
5089 // match(iRegP);
5090 match(iRegPNoSp);
5091 op_cost(0);
5092 format %{ %}
5093 interface(REG_INTER);
5094 %}
5095
5096 // Pointer 64 bit Register R1 only
5097 operand iRegP_R1()
5098 %{
5099 constraint(ALLOC_IN_RC(r1_reg));
5100 match(RegP);
5101 // match(iRegP);
5102 match(iRegPNoSp);
5103 op_cost(0);
5104 format %{ %}
5105 interface(REG_INTER);
5106 %}
5107
5108 // Pointer 64 bit Register R2 only
5109 operand iRegP_R2()
5110 %{
5111 constraint(ALLOC_IN_RC(r2_reg));
5112 match(RegP);
5113 // match(iRegP);
5114 match(iRegPNoSp);
5115 op_cost(0);
5116 format %{ %}
5117 interface(REG_INTER);
5118 %}
5119
5120 // Pointer 64 bit Register R3 only
5121 operand iRegP_R3()
5122 %{
5123 constraint(ALLOC_IN_RC(r3_reg));
5124 match(RegP);
5125 // match(iRegP);
5126 match(iRegPNoSp);
5127 op_cost(0);
5128 format %{ %}
5129 interface(REG_INTER);
5130 %}
5131
5132 // Pointer 64 bit Register R4 only
5133 operand iRegP_R4()
5134 %{
5135 constraint(ALLOC_IN_RC(r4_reg));
5136 match(RegP);
5137 // match(iRegP);
5138 match(iRegPNoSp);
5139 op_cost(0);
5140 format %{ %}
5141 interface(REG_INTER);
5142 %}
5143
5144 // Pointer 64 bit Register R5 only
5145 operand iRegP_R5()
5146 %{
5147 constraint(ALLOC_IN_RC(r5_reg));
5148 match(RegP);
5149 // match(iRegP);
5150 match(iRegPNoSp);
5151 op_cost(0);
5152 format %{ %}
5153 interface(REG_INTER);
5154 %}
5155
5156 // Pointer 64 bit Register R10 only
5157 operand iRegP_R10()
5158 %{
5159 constraint(ALLOC_IN_RC(r10_reg));
5160 match(RegP);
5161 // match(iRegP);
5162 match(iRegPNoSp);
5163 op_cost(0);
5164 format %{ %}
5165 interface(REG_INTER);
5166 %}
5167
5168 // Long 64 bit Register R0 only
5169 operand iRegL_R0()
5170 %{
5171 constraint(ALLOC_IN_RC(r0_reg));
5172 match(RegL);
5173 match(iRegLNoSp);
5174 op_cost(0);
5175 format %{ %}
5176 interface(REG_INTER);
5177 %}
5178
5179 // Long 64 bit Register R2 only
5180 operand iRegL_R2()
5181 %{
5182 constraint(ALLOC_IN_RC(r2_reg));
5183 match(RegL);
5184 match(iRegLNoSp);
5185 op_cost(0);
5186 format %{ %}
5187 interface(REG_INTER);
5188 %}
5189
5190 // Long 64 bit Register R3 only
5191 operand iRegL_R3()
5192 %{
5193 constraint(ALLOC_IN_RC(r3_reg));
5194 match(RegL);
5195 match(iRegLNoSp);
5196 op_cost(0);
5197 format %{ %}
5198 interface(REG_INTER);
5199 %}
5200
5201 // Long 64 bit Register R11 only
5202 operand iRegL_R11()
5203 %{
5204 constraint(ALLOC_IN_RC(r11_reg));
5205 match(RegL);
5206 match(iRegLNoSp);
5207 op_cost(0);
5208 format %{ %}
5209 interface(REG_INTER);
5210 %}
5211
5212 // Pointer 64 bit Register FP only
5213 operand iRegP_FP()
5214 %{
5215 constraint(ALLOC_IN_RC(fp_reg));
5216 match(RegP);
5217 // match(iRegP);
5218 op_cost(0);
5219 format %{ %}
5220 interface(REG_INTER);
5221 %}
5222
5223 // Register R0 only
5224 operand iRegI_R0()
5225 %{
5226 constraint(ALLOC_IN_RC(int_r0_reg));
5227 match(RegI);
5228 match(iRegINoSp);
5229 op_cost(0);
5230 format %{ %}
5231 interface(REG_INTER);
5232 %}
5233
5234 // Register R2 only
5235 operand iRegI_R2()
5236 %{
5237 constraint(ALLOC_IN_RC(int_r2_reg));
5238 match(RegI);
5239 match(iRegINoSp);
5240 op_cost(0);
5241 format %{ %}
5242 interface(REG_INTER);
5243 %}
5244
5245 // Register R3 only
5246 operand iRegI_R3()
5247 %{
5248 constraint(ALLOC_IN_RC(int_r3_reg));
5249 match(RegI);
5250 match(iRegINoSp);
5251 op_cost(0);
5252 format %{ %}
5253 interface(REG_INTER);
5254 %}
5255
5256
5257 // Register R4 only
5258 operand iRegI_R4()
5259 %{
5260 constraint(ALLOC_IN_RC(int_r4_reg));
5261 match(RegI);
5262 match(iRegINoSp);
5263 op_cost(0);
5264 format %{ %}
5265 interface(REG_INTER);
5266 %}
5267
5268
5269 // Pointer Register Operands
5270 // Narrow Pointer Register
5271 operand iRegN()
5272 %{
5273 constraint(ALLOC_IN_RC(any_reg32));
5274 match(RegN);
5275 match(iRegNNoSp);
5276 op_cost(0);
5277 format %{ %}
5278 interface(REG_INTER);
5279 %}
5280
5281 operand iRegN_R0()
5282 %{
5283 constraint(ALLOC_IN_RC(r0_reg));
5284 match(iRegN);
5285 op_cost(0);
5286 format %{ %}
5287 interface(REG_INTER);
5288 %}
5289
5290 operand iRegN_R2()
5291 %{
5292 constraint(ALLOC_IN_RC(r2_reg));
5293 match(iRegN);
5294 op_cost(0);
5295 format %{ %}
5296 interface(REG_INTER);
5297 %}
5298
5299 operand iRegN_R3()
5300 %{
5301 constraint(ALLOC_IN_RC(r3_reg));
5302 match(iRegN);
5303 op_cost(0);
5304 format %{ %}
5305 interface(REG_INTER);
5306 %}
5307
5308 // Integer 64 bit Register not Special
5309 operand iRegNNoSp()
5310 %{
5311 constraint(ALLOC_IN_RC(no_special_reg32));
5312 match(RegN);
5313 op_cost(0);
5314 format %{ %}
5315 interface(REG_INTER);
5316 %}
5317
5318 // Float Register
5319 // Float register operands
5320 operand vRegF()
5321 %{
5322 constraint(ALLOC_IN_RC(float_reg));
5323 match(RegF);
5324
5325 op_cost(0);
5326 format %{ %}
5327 interface(REG_INTER);
5328 %}
5329
5330 // Double Register
5331 // Double register operands
5332 operand vRegD()
5333 %{
5334 constraint(ALLOC_IN_RC(double_reg));
5335 match(RegD);
5336
5337 op_cost(0);
5338 format %{ %}
5339 interface(REG_INTER);
5340 %}
5341
5342 // Generic vector class. This will be used for
5343 // all vector operands, including NEON and SVE.
5344 operand vReg()
5345 %{
5346 constraint(ALLOC_IN_RC(dynamic));
5347 match(VecA);
5348 match(VecD);
5349 match(VecX);
5350
5351 op_cost(0);
5352 format %{ %}
5353 interface(REG_INTER);
5354 %}
5355
5356 operand vecA()
5357 %{
5358 constraint(ALLOC_IN_RC(vectora_reg));
5359 match(VecA);
5360
5361 op_cost(0);
5362 format %{ %}
5363 interface(REG_INTER);
5364 %}
5365
5366 operand vecD()
5367 %{
5368 constraint(ALLOC_IN_RC(vectord_reg));
5369 match(VecD);
5370
5371 op_cost(0);
5372 format %{ %}
5373 interface(REG_INTER);
5374 %}
5375
5376 operand vecX()
5377 %{
5378 constraint(ALLOC_IN_RC(vectorx_reg));
5379 match(VecX);
5380
5381 op_cost(0);
5382 format %{ %}
5383 interface(REG_INTER);
5384 %}
5385
5386 operand vRegD_V0()
5387 %{
5388 constraint(ALLOC_IN_RC(v0_reg));
5389 match(RegD);
5390 op_cost(0);
5391 format %{ %}
5392 interface(REG_INTER);
5393 %}
5394
5395 operand vRegD_V1()
5396 %{
5397 constraint(ALLOC_IN_RC(v1_reg));
5398 match(RegD);
5399 op_cost(0);
5400 format %{ %}
5401 interface(REG_INTER);
5402 %}
5403
5404 operand vRegD_V2()
5405 %{
5406 constraint(ALLOC_IN_RC(v2_reg));
5407 match(RegD);
5408 op_cost(0);
5409 format %{ %}
5410 interface(REG_INTER);
5411 %}
5412
5413 operand vRegD_V3()
5414 %{
5415 constraint(ALLOC_IN_RC(v3_reg));
5416 match(RegD);
5417 op_cost(0);
5418 format %{ %}
5419 interface(REG_INTER);
5420 %}
5421
5422 operand vRegD_V4()
5423 %{
5424 constraint(ALLOC_IN_RC(v4_reg));
5425 match(RegD);
5426 op_cost(0);
5427 format %{ %}
5428 interface(REG_INTER);
5429 %}
5430
5431 operand vRegD_V5()
5432 %{
5433 constraint(ALLOC_IN_RC(v5_reg));
5434 match(RegD);
5435 op_cost(0);
5436 format %{ %}
5437 interface(REG_INTER);
5438 %}
5439
5440 operand vRegD_V6()
5441 %{
5442 constraint(ALLOC_IN_RC(v6_reg));
5443 match(RegD);
5444 op_cost(0);
5445 format %{ %}
5446 interface(REG_INTER);
5447 %}
5448
5449 operand vRegD_V7()
5450 %{
5451 constraint(ALLOC_IN_RC(v7_reg));
5452 match(RegD);
5453 op_cost(0);
5454 format %{ %}
5455 interface(REG_INTER);
5456 %}
5457
5458 operand vRegD_V8()
5459 %{
5460 constraint(ALLOC_IN_RC(v8_reg));
5461 match(RegD);
5462 op_cost(0);
5463 format %{ %}
5464 interface(REG_INTER);
5465 %}
5466
5467 operand vRegD_V9()
5468 %{
5469 constraint(ALLOC_IN_RC(v9_reg));
5470 match(RegD);
5471 op_cost(0);
5472 format %{ %}
5473 interface(REG_INTER);
5474 %}
5475
5476 operand vRegD_V10()
5477 %{
5478 constraint(ALLOC_IN_RC(v10_reg));
5479 match(RegD);
5480 op_cost(0);
5481 format %{ %}
5482 interface(REG_INTER);
5483 %}
5484
5485 operand vRegD_V11()
5486 %{
5487 constraint(ALLOC_IN_RC(v11_reg));
5488 match(RegD);
5489 op_cost(0);
5490 format %{ %}
5491 interface(REG_INTER);
5492 %}
5493
5494 operand vRegD_V12()
5495 %{
5496 constraint(ALLOC_IN_RC(v12_reg));
5497 match(RegD);
5498 op_cost(0);
5499 format %{ %}
5500 interface(REG_INTER);
5501 %}
5502
5503 operand vRegD_V13()
5504 %{
5505 constraint(ALLOC_IN_RC(v13_reg));
5506 match(RegD);
5507 op_cost(0);
5508 format %{ %}
5509 interface(REG_INTER);
5510 %}
5511
5512 operand vRegD_V14()
5513 %{
5514 constraint(ALLOC_IN_RC(v14_reg));
5515 match(RegD);
5516 op_cost(0);
5517 format %{ %}
5518 interface(REG_INTER);
5519 %}
5520
5521 operand vRegD_V15()
5522 %{
5523 constraint(ALLOC_IN_RC(v15_reg));
5524 match(RegD);
5525 op_cost(0);
5526 format %{ %}
5527 interface(REG_INTER);
5528 %}
5529
5530 operand vRegD_V16()
5531 %{
5532 constraint(ALLOC_IN_RC(v16_reg));
5533 match(RegD);
5534 op_cost(0);
5535 format %{ %}
5536 interface(REG_INTER);
5537 %}
5538
5539 operand vRegD_V17()
5540 %{
5541 constraint(ALLOC_IN_RC(v17_reg));
5542 match(RegD);
5543 op_cost(0);
5544 format %{ %}
5545 interface(REG_INTER);
5546 %}
5547
5548 operand vRegD_V18()
5549 %{
5550 constraint(ALLOC_IN_RC(v18_reg));
5551 match(RegD);
5552 op_cost(0);
5553 format %{ %}
5554 interface(REG_INTER);
5555 %}
5556
5557 operand vRegD_V19()
5558 %{
5559 constraint(ALLOC_IN_RC(v19_reg));
5560 match(RegD);
5561 op_cost(0);
5562 format %{ %}
5563 interface(REG_INTER);
5564 %}
5565
5566 operand vRegD_V20()
5567 %{
5568 constraint(ALLOC_IN_RC(v20_reg));
5569 match(RegD);
5570 op_cost(0);
5571 format %{ %}
5572 interface(REG_INTER);
5573 %}
5574
5575 operand vRegD_V21()
5576 %{
5577 constraint(ALLOC_IN_RC(v21_reg));
5578 match(RegD);
5579 op_cost(0);
5580 format %{ %}
5581 interface(REG_INTER);
5582 %}
5583
5584 operand vRegD_V22()
5585 %{
5586 constraint(ALLOC_IN_RC(v22_reg));
5587 match(RegD);
5588 op_cost(0);
5589 format %{ %}
5590 interface(REG_INTER);
5591 %}
5592
5593 operand vRegD_V23()
5594 %{
5595 constraint(ALLOC_IN_RC(v23_reg));
5596 match(RegD);
5597 op_cost(0);
5598 format %{ %}
5599 interface(REG_INTER);
5600 %}
5601
5602 operand vRegD_V24()
5603 %{
5604 constraint(ALLOC_IN_RC(v24_reg));
5605 match(RegD);
5606 op_cost(0);
5607 format %{ %}
5608 interface(REG_INTER);
5609 %}
5610
5611 operand vRegD_V25()
5612 %{
5613 constraint(ALLOC_IN_RC(v25_reg));
5614 match(RegD);
5615 op_cost(0);
5616 format %{ %}
5617 interface(REG_INTER);
5618 %}
5619
5620 operand vRegD_V26()
5621 %{
5622 constraint(ALLOC_IN_RC(v26_reg));
5623 match(RegD);
5624 op_cost(0);
5625 format %{ %}
5626 interface(REG_INTER);
5627 %}
5628
5629 operand vRegD_V27()
5630 %{
5631 constraint(ALLOC_IN_RC(v27_reg));
5632 match(RegD);
5633 op_cost(0);
5634 format %{ %}
5635 interface(REG_INTER);
5636 %}
5637
5638 operand vRegD_V28()
5639 %{
5640 constraint(ALLOC_IN_RC(v28_reg));
5641 match(RegD);
5642 op_cost(0);
5643 format %{ %}
5644 interface(REG_INTER);
5645 %}
5646
5647 operand vRegD_V29()
5648 %{
5649 constraint(ALLOC_IN_RC(v29_reg));
5650 match(RegD);
5651 op_cost(0);
5652 format %{ %}
5653 interface(REG_INTER);
5654 %}
5655
5656 operand vRegD_V30()
5657 %{
5658 constraint(ALLOC_IN_RC(v30_reg));
5659 match(RegD);
5660 op_cost(0);
5661 format %{ %}
5662 interface(REG_INTER);
5663 %}
5664
5665 operand vRegD_V31()
5666 %{
5667 constraint(ALLOC_IN_RC(v31_reg));
5668 match(RegD);
5669 op_cost(0);
5670 format %{ %}
5671 interface(REG_INTER);
5672 %}
5673
5674 operand pReg()
5675 %{
5676 constraint(ALLOC_IN_RC(pr_reg));
5677 match(RegVectMask);
5678 match(pRegGov);
5679 op_cost(0);
5680 format %{ %}
5681 interface(REG_INTER);
5682 %}
5683
5684 operand pRegGov()
5685 %{
5686 constraint(ALLOC_IN_RC(gov_pr));
5687 match(RegVectMask);
5688 match(pReg);
5689 op_cost(0);
5690 format %{ %}
5691 interface(REG_INTER);
5692 %}
5693
5694 operand pRegGov_P0()
5695 %{
5696 constraint(ALLOC_IN_RC(p0_reg));
5697 match(RegVectMask);
5698 op_cost(0);
5699 format %{ %}
5700 interface(REG_INTER);
5701 %}
5702
5703 operand pRegGov_P1()
5704 %{
5705 constraint(ALLOC_IN_RC(p1_reg));
5706 match(RegVectMask);
5707 op_cost(0);
5708 format %{ %}
5709 interface(REG_INTER);
5710 %}
5711
5712 // Flags register, used as output of signed compare instructions
5713
5714 // note that on AArch64 we also use this register as the output for
5715 // for floating point compare instructions (CmpF CmpD). this ensures
5716 // that ordered inequality tests use GT, GE, LT or LE none of which
5717 // pass through cases where the result is unordered i.e. one or both
5718 // inputs to the compare is a NaN. this means that the ideal code can
5719 // replace e.g. a GT with an LE and not end up capturing the NaN case
5720 // (where the comparison should always fail). EQ and NE tests are
5721 // always generated in ideal code so that unordered folds into the NE
5722 // case, matching the behaviour of AArch64 NE.
5723 //
5724 // This differs from x86 where the outputs of FP compares use a
5725 // special FP flags registers and where compares based on this
5726 // register are distinguished into ordered inequalities (cmpOpUCF) and
5727 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5728 // to explicitly handle the unordered case in branches. x86 also has
5729 // to include extra CMoveX rules to accept a cmpOpUCF input.
5730
5731 operand rFlagsReg()
5732 %{
5733 constraint(ALLOC_IN_RC(int_flags));
5734 match(RegFlags);
5735
5736 op_cost(0);
5737 format %{ "RFLAGS" %}
5738 interface(REG_INTER);
5739 %}
5740
5741 // Flags register, used as output of unsigned compare instructions
5742 operand rFlagsRegU()
5743 %{
5744 constraint(ALLOC_IN_RC(int_flags));
5745 match(RegFlags);
5746
5747 op_cost(0);
5748 format %{ "RFLAGSU" %}
5749 interface(REG_INTER);
5750 %}
5751
5752 // Special Registers
5753
5754 // Method Register
5755 operand inline_cache_RegP(iRegP reg)
5756 %{
5757 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5758 match(reg);
5759 match(iRegPNoSp);
5760 op_cost(0);
5761 format %{ %}
5762 interface(REG_INTER);
5763 %}
5764
5765 // Thread Register
5766 operand thread_RegP(iRegP reg)
5767 %{
5768 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5769 match(reg);
5770 op_cost(0);
5771 format %{ %}
5772 interface(REG_INTER);
5773 %}
5774
5775 operand lr_RegP(iRegP reg)
5776 %{
5777 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5778 match(reg);
5779 op_cost(0);
5780 format %{ %}
5781 interface(REG_INTER);
5782 %}
5783
5784 //----------Memory Operands----------------------------------------------------
5785
5786 operand indirect(iRegP reg)
5787 %{
5788 constraint(ALLOC_IN_RC(ptr_reg));
5789 match(reg);
5790 op_cost(0);
5791 format %{ "[$reg]" %}
5792 interface(MEMORY_INTER) %{
5793 base($reg);
5794 index(0xffffffff);
5795 scale(0x0);
5796 disp(0x0);
5797 %}
5798 %}
5799
5800 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5801 %{
5802 constraint(ALLOC_IN_RC(ptr_reg));
5803 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5804 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5805 op_cost(0);
5806 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5807 interface(MEMORY_INTER) %{
5808 base($reg);
5809 index($ireg);
5810 scale($scale);
5811 disp(0x0);
5812 %}
5813 %}
5814
5815 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5816 %{
5817 constraint(ALLOC_IN_RC(ptr_reg));
5818 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5819 match(AddP reg (LShiftL lreg scale));
5820 op_cost(0);
5821 format %{ "$reg, $lreg lsl($scale)" %}
5822 interface(MEMORY_INTER) %{
5823 base($reg);
5824 index($lreg);
5825 scale($scale);
5826 disp(0x0);
5827 %}
5828 %}
5829
5830 operand indIndexI2L(iRegP reg, iRegI ireg)
5831 %{
5832 constraint(ALLOC_IN_RC(ptr_reg));
5833 match(AddP reg (ConvI2L ireg));
5834 op_cost(0);
5835 format %{ "$reg, $ireg, 0, I2L" %}
5836 interface(MEMORY_INTER) %{
5837 base($reg);
5838 index($ireg);
5839 scale(0x0);
5840 disp(0x0);
5841 %}
5842 %}
5843
5844 operand indIndex(iRegP reg, iRegL lreg)
5845 %{
5846 constraint(ALLOC_IN_RC(ptr_reg));
5847 match(AddP reg lreg);
5848 op_cost(0);
5849 format %{ "$reg, $lreg" %}
5850 interface(MEMORY_INTER) %{
5851 base($reg);
5852 index($lreg);
5853 scale(0x0);
5854 disp(0x0);
5855 %}
5856 %}
5857
5858 operand indOffI(iRegP reg, immIOffset off)
5859 %{
5860 constraint(ALLOC_IN_RC(ptr_reg));
5861 match(AddP reg off);
5862 op_cost(0);
5863 format %{ "[$reg, $off]" %}
5864 interface(MEMORY_INTER) %{
5865 base($reg);
5866 index(0xffffffff);
5867 scale(0x0);
5868 disp($off);
5869 %}
5870 %}
5871
5872 operand indOffI1(iRegP reg, immIOffset1 off)
5873 %{
5874 constraint(ALLOC_IN_RC(ptr_reg));
5875 match(AddP reg off);
5876 op_cost(0);
5877 format %{ "[$reg, $off]" %}
5878 interface(MEMORY_INTER) %{
5879 base($reg);
5880 index(0xffffffff);
5881 scale(0x0);
5882 disp($off);
5883 %}
5884 %}
5885
5886 operand indOffI2(iRegP reg, immIOffset2 off)
5887 %{
5888 constraint(ALLOC_IN_RC(ptr_reg));
5889 match(AddP reg off);
5890 op_cost(0);
5891 format %{ "[$reg, $off]" %}
5892 interface(MEMORY_INTER) %{
5893 base($reg);
5894 index(0xffffffff);
5895 scale(0x0);
5896 disp($off);
5897 %}
5898 %}
5899
5900 operand indOffI4(iRegP reg, immIOffset4 off)
5901 %{
5902 constraint(ALLOC_IN_RC(ptr_reg));
5903 match(AddP reg off);
5904 op_cost(0);
5905 format %{ "[$reg, $off]" %}
5906 interface(MEMORY_INTER) %{
5907 base($reg);
5908 index(0xffffffff);
5909 scale(0x0);
5910 disp($off);
5911 %}
5912 %}
5913
5914 operand indOffI8(iRegP reg, immIOffset8 off)
5915 %{
5916 constraint(ALLOC_IN_RC(ptr_reg));
5917 match(AddP reg off);
5918 op_cost(0);
5919 format %{ "[$reg, $off]" %}
5920 interface(MEMORY_INTER) %{
5921 base($reg);
5922 index(0xffffffff);
5923 scale(0x0);
5924 disp($off);
5925 %}
5926 %}
5927
5928 operand indOffI16(iRegP reg, immIOffset16 off)
5929 %{
5930 constraint(ALLOC_IN_RC(ptr_reg));
5931 match(AddP reg off);
5932 op_cost(0);
5933 format %{ "[$reg, $off]" %}
5934 interface(MEMORY_INTER) %{
5935 base($reg);
5936 index(0xffffffff);
5937 scale(0x0);
5938 disp($off);
5939 %}
5940 %}
5941
5942 operand indOffL(iRegP reg, immLoffset off)
5943 %{
5944 constraint(ALLOC_IN_RC(ptr_reg));
5945 match(AddP reg off);
5946 op_cost(0);
5947 format %{ "[$reg, $off]" %}
5948 interface(MEMORY_INTER) %{
5949 base($reg);
5950 index(0xffffffff);
5951 scale(0x0);
5952 disp($off);
5953 %}
5954 %}
5955
5956 operand indOffL1(iRegP reg, immLoffset1 off)
5957 %{
5958 constraint(ALLOC_IN_RC(ptr_reg));
5959 match(AddP reg off);
5960 op_cost(0);
5961 format %{ "[$reg, $off]" %}
5962 interface(MEMORY_INTER) %{
5963 base($reg);
5964 index(0xffffffff);
5965 scale(0x0);
5966 disp($off);
5967 %}
5968 %}
5969
5970 operand indOffL2(iRegP reg, immLoffset2 off)
5971 %{
5972 constraint(ALLOC_IN_RC(ptr_reg));
5973 match(AddP reg off);
5974 op_cost(0);
5975 format %{ "[$reg, $off]" %}
5976 interface(MEMORY_INTER) %{
5977 base($reg);
5978 index(0xffffffff);
5979 scale(0x0);
5980 disp($off);
5981 %}
5982 %}
5983
5984 operand indOffL4(iRegP reg, immLoffset4 off)
5985 %{
5986 constraint(ALLOC_IN_RC(ptr_reg));
5987 match(AddP reg off);
5988 op_cost(0);
5989 format %{ "[$reg, $off]" %}
5990 interface(MEMORY_INTER) %{
5991 base($reg);
5992 index(0xffffffff);
5993 scale(0x0);
5994 disp($off);
5995 %}
5996 %}
5997
5998 operand indOffL8(iRegP reg, immLoffset8 off)
5999 %{
6000 constraint(ALLOC_IN_RC(ptr_reg));
6001 match(AddP reg off);
6002 op_cost(0);
6003 format %{ "[$reg, $off]" %}
6004 interface(MEMORY_INTER) %{
6005 base($reg);
6006 index(0xffffffff);
6007 scale(0x0);
6008 disp($off);
6009 %}
6010 %}
6011
6012 operand indOffL16(iRegP reg, immLoffset16 off)
6013 %{
6014 constraint(ALLOC_IN_RC(ptr_reg));
6015 match(AddP reg off);
6016 op_cost(0);
6017 format %{ "[$reg, $off]" %}
6018 interface(MEMORY_INTER) %{
6019 base($reg);
6020 index(0xffffffff);
6021 scale(0x0);
6022 disp($off);
6023 %}
6024 %}
6025
6026 operand indirectN(iRegN reg)
6027 %{
6028 predicate(CompressedOops::shift() == 0);
6029 constraint(ALLOC_IN_RC(ptr_reg));
6030 match(DecodeN reg);
6031 op_cost(0);
6032 format %{ "[$reg]\t# narrow" %}
6033 interface(MEMORY_INTER) %{
6034 base($reg);
6035 index(0xffffffff);
6036 scale(0x0);
6037 disp(0x0);
6038 %}
6039 %}
6040
6041 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
6042 %{
6043 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
6044 constraint(ALLOC_IN_RC(ptr_reg));
6045 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
6046 op_cost(0);
6047 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
6048 interface(MEMORY_INTER) %{
6049 base($reg);
6050 index($ireg);
6051 scale($scale);
6052 disp(0x0);
6053 %}
6054 %}
6055
6056 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
6057 %{
6058 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
6059 constraint(ALLOC_IN_RC(ptr_reg));
6060 match(AddP (DecodeN reg) (LShiftL lreg scale));
6061 op_cost(0);
6062 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
6063 interface(MEMORY_INTER) %{
6064 base($reg);
6065 index($lreg);
6066 scale($scale);
6067 disp(0x0);
6068 %}
6069 %}
6070
6071 operand indIndexI2LN(iRegN reg, iRegI ireg)
6072 %{
6073 predicate(CompressedOops::shift() == 0);
6074 constraint(ALLOC_IN_RC(ptr_reg));
6075 match(AddP (DecodeN reg) (ConvI2L ireg));
6076 op_cost(0);
6077 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
6078 interface(MEMORY_INTER) %{
6079 base($reg);
6080 index($ireg);
6081 scale(0x0);
6082 disp(0x0);
6083 %}
6084 %}
6085
6086 operand indIndexN(iRegN reg, iRegL lreg)
6087 %{
6088 predicate(CompressedOops::shift() == 0);
6089 constraint(ALLOC_IN_RC(ptr_reg));
6090 match(AddP (DecodeN reg) lreg);
6091 op_cost(0);
6092 format %{ "$reg, $lreg\t# narrow" %}
6093 interface(MEMORY_INTER) %{
6094 base($reg);
6095 index($lreg);
6096 scale(0x0);
6097 disp(0x0);
6098 %}
6099 %}
6100
6101 operand indOffIN(iRegN reg, immIOffset off)
6102 %{
6103 predicate(CompressedOops::shift() == 0);
6104 constraint(ALLOC_IN_RC(ptr_reg));
6105 match(AddP (DecodeN reg) off);
6106 op_cost(0);
6107 format %{ "[$reg, $off]\t# narrow" %}
6108 interface(MEMORY_INTER) %{
6109 base($reg);
6110 index(0xffffffff);
6111 scale(0x0);
6112 disp($off);
6113 %}
6114 %}
6115
6116 operand indOffLN(iRegN reg, immLoffset off)
6117 %{
6118 predicate(CompressedOops::shift() == 0);
6119 constraint(ALLOC_IN_RC(ptr_reg));
6120 match(AddP (DecodeN reg) off);
6121 op_cost(0);
6122 format %{ "[$reg, $off]\t# narrow" %}
6123 interface(MEMORY_INTER) %{
6124 base($reg);
6125 index(0xffffffff);
6126 scale(0x0);
6127 disp($off);
6128 %}
6129 %}
6130
6131
6132
6133 // AArch64 opto stubs need to write to the pc slot in the thread anchor
6134 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
6135 %{
6136 constraint(ALLOC_IN_RC(ptr_reg));
6137 match(AddP reg off);
6138 op_cost(0);
6139 format %{ "[$reg, $off]" %}
6140 interface(MEMORY_INTER) %{
6141 base($reg);
6142 index(0xffffffff);
6143 scale(0x0);
6144 disp($off);
6145 %}
6146 %}
6147
6148 //----------Special Memory Operands--------------------------------------------
6149 // Stack Slot Operand - This operand is used for loading and storing temporary
6150 // values on the stack where a match requires a value to
6151 // flow through memory.
6152 operand stackSlotP(sRegP reg)
6153 %{
6154 constraint(ALLOC_IN_RC(stack_slots));
6155 op_cost(100);
6156 // No match rule because this operand is only generated in matching
6157 // match(RegP);
6158 format %{ "[$reg]" %}
6159 interface(MEMORY_INTER) %{
6160 base(0x1e); // RSP
6161 index(0x0); // No Index
6162 scale(0x0); // No Scale
6163 disp($reg); // Stack Offset
6164 %}
6165 %}
6166
6167 operand stackSlotI(sRegI reg)
6168 %{
6169 constraint(ALLOC_IN_RC(stack_slots));
6170 // No match rule because this operand is only generated in matching
6171 // match(RegI);
6172 format %{ "[$reg]" %}
6173 interface(MEMORY_INTER) %{
6174 base(0x1e); // RSP
6175 index(0x0); // No Index
6176 scale(0x0); // No Scale
6177 disp($reg); // Stack Offset
6178 %}
6179 %}
6180
6181 operand stackSlotF(sRegF reg)
6182 %{
6183 constraint(ALLOC_IN_RC(stack_slots));
6184 // No match rule because this operand is only generated in matching
6185 // match(RegF);
6186 format %{ "[$reg]" %}
6187 interface(MEMORY_INTER) %{
6188 base(0x1e); // RSP
6189 index(0x0); // No Index
6190 scale(0x0); // No Scale
6191 disp($reg); // Stack Offset
6192 %}
6193 %}
6194
6195 operand stackSlotD(sRegD reg)
6196 %{
6197 constraint(ALLOC_IN_RC(stack_slots));
6198 // No match rule because this operand is only generated in matching
6199 // match(RegD);
6200 format %{ "[$reg]" %}
6201 interface(MEMORY_INTER) %{
6202 base(0x1e); // RSP
6203 index(0x0); // No Index
6204 scale(0x0); // No Scale
6205 disp($reg); // Stack Offset
6206 %}
6207 %}
6208
6209 operand stackSlotL(sRegL reg)
6210 %{
6211 constraint(ALLOC_IN_RC(stack_slots));
6212 // No match rule because this operand is only generated in matching
6213 // match(RegL);
6214 format %{ "[$reg]" %}
6215 interface(MEMORY_INTER) %{
6216 base(0x1e); // RSP
6217 index(0x0); // No Index
6218 scale(0x0); // No Scale
6219 disp($reg); // Stack Offset
6220 %}
6221 %}
6222
6223 // Operands for expressing Control Flow
6224 // NOTE: Label is a predefined operand which should not be redefined in
6225 // the AD file. It is generically handled within the ADLC.
6226
6227 //----------Conditional Branch Operands----------------------------------------
6228 // Comparison Op - This is the operation of the comparison, and is limited to
6229 // the following set of codes:
6230 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6231 //
6232 // Other attributes of the comparison, such as unsignedness, are specified
6233 // by the comparison instruction that sets a condition code flags register.
6234 // That result is represented by a flags operand whose subtype is appropriate
6235 // to the unsignedness (etc.) of the comparison.
6236 //
6237 // Later, the instruction which matches both the Comparison Op (a Bool) and
6238 // the flags (produced by the Cmp) specifies the coding of the comparison op
6239 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6240
6241 // used for signed integral comparisons and fp comparisons
6242
6243 operand cmpOp()
6244 %{
6245 match(Bool);
6246
6247 format %{ "" %}
6248 interface(COND_INTER) %{
6249 equal(0x0, "eq");
6250 not_equal(0x1, "ne");
6251 less(0xb, "lt");
6252 greater_equal(0xa, "ge");
6253 less_equal(0xd, "le");
6254 greater(0xc, "gt");
6255 overflow(0x6, "vs");
6256 no_overflow(0x7, "vc");
6257 %}
6258 %}
6259
6260 // used for unsigned integral comparisons
6261
6262 operand cmpOpU()
6263 %{
6264 match(Bool);
6265
6266 format %{ "" %}
6267 interface(COND_INTER) %{
6268 equal(0x0, "eq");
6269 not_equal(0x1, "ne");
6270 less(0x3, "lo");
6271 greater_equal(0x2, "hs");
6272 less_equal(0x9, "ls");
6273 greater(0x8, "hi");
6274 overflow(0x6, "vs");
6275 no_overflow(0x7, "vc");
6276 %}
6277 %}
6278
6279 // used for certain integral comparisons which can be
6280 // converted to cbxx or tbxx instructions
6281
6282 operand cmpOpEqNe()
6283 %{
6284 match(Bool);
6285 op_cost(0);
6286 predicate(n->as_Bool()->_test._test == BoolTest::ne
6287 || n->as_Bool()->_test._test == BoolTest::eq);
6288
6289 format %{ "" %}
6290 interface(COND_INTER) %{
6291 equal(0x0, "eq");
6292 not_equal(0x1, "ne");
6293 less(0xb, "lt");
6294 greater_equal(0xa, "ge");
6295 less_equal(0xd, "le");
6296 greater(0xc, "gt");
6297 overflow(0x6, "vs");
6298 no_overflow(0x7, "vc");
6299 %}
6300 %}
6301
6302 // used for certain integral comparisons which can be
6303 // converted to cbxx or tbxx instructions
6304
6305 operand cmpOpLtGe()
6306 %{
6307 match(Bool);
6308 op_cost(0);
6309
6310 predicate(n->as_Bool()->_test._test == BoolTest::lt
6311 || n->as_Bool()->_test._test == BoolTest::ge);
6312
6313 format %{ "" %}
6314 interface(COND_INTER) %{
6315 equal(0x0, "eq");
6316 not_equal(0x1, "ne");
6317 less(0xb, "lt");
6318 greater_equal(0xa, "ge");
6319 less_equal(0xd, "le");
6320 greater(0xc, "gt");
6321 overflow(0x6, "vs");
6322 no_overflow(0x7, "vc");
6323 %}
6324 %}
6325
6326 // used for certain unsigned integral comparisons which can be
6327 // converted to cbxx or tbxx instructions
6328
6329 operand cmpOpUEqNeLtGe()
6330 %{
6331 match(Bool);
6332 op_cost(0);
6333
6334 predicate(n->as_Bool()->_test._test == BoolTest::eq
6335 || n->as_Bool()->_test._test == BoolTest::ne
6336 || n->as_Bool()->_test._test == BoolTest::lt
6337 || n->as_Bool()->_test._test == BoolTest::ge);
6338
6339 format %{ "" %}
6340 interface(COND_INTER) %{
6341 equal(0x0, "eq");
6342 not_equal(0x1, "ne");
6343 less(0xb, "lt");
6344 greater_equal(0xa, "ge");
6345 less_equal(0xd, "le");
6346 greater(0xc, "gt");
6347 overflow(0x6, "vs");
6348 no_overflow(0x7, "vc");
6349 %}
6350 %}
6351
6352 // Special operand allowing long args to int ops to be truncated for free
6353
6354 operand iRegL2I(iRegL reg) %{
6355
6356 op_cost(0);
6357
6358 match(ConvL2I reg);
6359
6360 format %{ "l2i($reg)" %}
6361
6362 interface(REG_INTER)
6363 %}
6364
6365 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
6366 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
6367 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
6368 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
6369
6370 //----------OPERAND CLASSES----------------------------------------------------
6371 // Operand Classes are groups of operands that are used as to simplify
6372 // instruction definitions by not requiring the AD writer to specify
6373 // separate instructions for every form of operand when the
6374 // instruction accepts multiple operand types with the same basic
6375 // encoding and format. The classic case of this is memory operands.
6376
6377 // memory is used to define read/write location for load/store
6378 // instruction defs. we can turn a memory op into an Address
6379
6380 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
6381 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6382
6383 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
6384 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6385
6386 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
6387 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6388
6389 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
6390 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6391
6392 // All of the memory operands. For the pipeline description.
6393 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
6394 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
6395 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6396
6397
6398 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
6399 // operations. it allows the src to be either an iRegI or a (ConvL2I
6400 // iRegL). in the latter case the l2i normally planted for a ConvL2I
6401 // can be elided because the 32-bit instruction will just employ the
6402 // lower 32 bits anyway.
6403 //
6404 // n.b. this does not elide all L2I conversions. if the truncated
6405 // value is consumed by more than one operation then the ConvL2I
6406 // cannot be bundled into the consuming nodes so an l2i gets planted
6407 // (actually a movw $dst $src) and the downstream instructions consume
6408 // the result of the l2i as an iRegI input. That's a shame since the
6409 // movw is actually redundant but its not too costly.
6410
6411 opclass iRegIorL2I(iRegI, iRegL2I);
6412
6413 //----------PIPELINE-----------------------------------------------------------
6414 // Rules which define the behavior of the target architectures pipeline.
6415
6416 // For specific pipelines, eg A53, define the stages of that pipeline
6417 //pipe_desc(ISS, EX1, EX2, WR);
6418 #define ISS S0
6419 #define EX1 S1
6420 #define EX2 S2
6421 #define WR S3
6422
6423 // Integer ALU reg operation
6424 pipeline %{
6425
6426 attributes %{
6427 // ARM instructions are of fixed length
6428 fixed_size_instructions; // Fixed size instructions TODO does
6429 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
6430 // ARM instructions come in 32-bit word units
6431 instruction_unit_size = 4; // An instruction is 4 bytes long
6432 instruction_fetch_unit_size = 64; // The processor fetches one line
6433 instruction_fetch_units = 1; // of 64 bytes
6434
6435 // List of nop instructions
6436 nops( MachNop );
6437 %}
6438
6439 // We don't use an actual pipeline model so don't care about resources
6440 // or description. we do use pipeline classes to introduce fixed
6441 // latencies
6442
6443 //----------RESOURCES----------------------------------------------------------
6444 // Resources are the functional units available to the machine
6445
6446 resources( INS0, INS1, INS01 = INS0 | INS1,
6447 ALU0, ALU1, ALU = ALU0 | ALU1,
6448 MAC,
6449 DIV,
6450 BRANCH,
6451 LDST,
6452 NEON_FP);
6453
6454 //----------PIPELINE DESCRIPTION-----------------------------------------------
6455 // Pipeline Description specifies the stages in the machine's pipeline
6456
6457 // Define the pipeline as a generic 6 stage pipeline
6458 pipe_desc(S0, S1, S2, S3, S4, S5);
6459
6460 //----------PIPELINE CLASSES---------------------------------------------------
6461 // Pipeline Classes describe the stages in which input and output are
6462 // referenced by the hardware pipeline.
6463
6464 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
6465 %{
6466 single_instruction;
6467 src1 : S1(read);
6468 src2 : S2(read);
6469 dst : S5(write);
6470 INS01 : ISS;
6471 NEON_FP : S5;
6472 %}
6473
6474 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6475 %{
6476 single_instruction;
6477 src1 : S1(read);
6478 src2 : S2(read);
6479 dst : S5(write);
6480 INS01 : ISS;
6481 NEON_FP : S5;
6482 %}
6483
6484 pipe_class fp_uop_s(vRegF dst, vRegF src)
6485 %{
6486 single_instruction;
6487 src : S1(read);
6488 dst : S5(write);
6489 INS01 : ISS;
6490 NEON_FP : S5;
6491 %}
6492
6493 pipe_class fp_uop_d(vRegD dst, vRegD src)
6494 %{
6495 single_instruction;
6496 src : S1(read);
6497 dst : S5(write);
6498 INS01 : ISS;
6499 NEON_FP : S5;
6500 %}
6501
6502 pipe_class fp_d2f(vRegF dst, vRegD src)
6503 %{
6504 single_instruction;
6505 src : S1(read);
6506 dst : S5(write);
6507 INS01 : ISS;
6508 NEON_FP : S5;
6509 %}
6510
6511 pipe_class fp_f2d(vRegD dst, vRegF src)
6512 %{
6513 single_instruction;
6514 src : S1(read);
6515 dst : S5(write);
6516 INS01 : ISS;
6517 NEON_FP : S5;
6518 %}
6519
6520 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6521 %{
6522 single_instruction;
6523 src : S1(read);
6524 dst : S5(write);
6525 INS01 : ISS;
6526 NEON_FP : S5;
6527 %}
6528
6529 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6530 %{
6531 single_instruction;
6532 src : S1(read);
6533 dst : S5(write);
6534 INS01 : ISS;
6535 NEON_FP : S5;
6536 %}
6537
6538 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6539 %{
6540 single_instruction;
6541 src : S1(read);
6542 dst : S5(write);
6543 INS01 : ISS;
6544 NEON_FP : S5;
6545 %}
6546
6547 pipe_class fp_l2f(vRegF dst, iRegL src)
6548 %{
6549 single_instruction;
6550 src : S1(read);
6551 dst : S5(write);
6552 INS01 : ISS;
6553 NEON_FP : S5;
6554 %}
6555
6556 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6557 %{
6558 single_instruction;
6559 src : S1(read);
6560 dst : S5(write);
6561 INS01 : ISS;
6562 NEON_FP : S5;
6563 %}
6564
6565 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6566 %{
6567 single_instruction;
6568 src : S1(read);
6569 dst : S5(write);
6570 INS01 : ISS;
6571 NEON_FP : S5;
6572 %}
6573
6574 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6575 %{
6576 single_instruction;
6577 src : S1(read);
6578 dst : S5(write);
6579 INS01 : ISS;
6580 NEON_FP : S5;
6581 %}
6582
6583 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6584 %{
6585 single_instruction;
6586 src : S1(read);
6587 dst : S5(write);
6588 INS01 : ISS;
6589 NEON_FP : S5;
6590 %}
6591
6592 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6593 %{
6594 single_instruction;
6595 src1 : S1(read);
6596 src2 : S2(read);
6597 dst : S5(write);
6598 INS0 : ISS;
6599 NEON_FP : S5;
6600 %}
6601
6602 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6603 %{
6604 single_instruction;
6605 src1 : S1(read);
6606 src2 : S2(read);
6607 dst : S5(write);
6608 INS0 : ISS;
6609 NEON_FP : S5;
6610 %}
6611
6612 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6613 %{
6614 single_instruction;
6615 cr : S1(read);
6616 src1 : S1(read);
6617 src2 : S1(read);
6618 dst : S3(write);
6619 INS01 : ISS;
6620 NEON_FP : S3;
6621 %}
6622
6623 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6624 %{
6625 single_instruction;
6626 cr : S1(read);
6627 src1 : S1(read);
6628 src2 : S1(read);
6629 dst : S3(write);
6630 INS01 : ISS;
6631 NEON_FP : S3;
6632 %}
6633
6634 pipe_class fp_imm_s(vRegF dst)
6635 %{
6636 single_instruction;
6637 dst : S3(write);
6638 INS01 : ISS;
6639 NEON_FP : S3;
6640 %}
6641
6642 pipe_class fp_imm_d(vRegD dst)
6643 %{
6644 single_instruction;
6645 dst : S3(write);
6646 INS01 : ISS;
6647 NEON_FP : S3;
6648 %}
6649
6650 pipe_class fp_load_constant_s(vRegF dst)
6651 %{
6652 single_instruction;
6653 dst : S4(write);
6654 INS01 : ISS;
6655 NEON_FP : S4;
6656 %}
6657
6658 pipe_class fp_load_constant_d(vRegD dst)
6659 %{
6660 single_instruction;
6661 dst : S4(write);
6662 INS01 : ISS;
6663 NEON_FP : S4;
6664 %}
6665
6666 //------- Integer ALU operations --------------------------
6667
6668 // Integer ALU reg-reg operation
6669 // Operands needed in EX1, result generated in EX2
6670 // Eg. ADD x0, x1, x2
6671 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6672 %{
6673 single_instruction;
6674 dst : EX2(write);
6675 src1 : EX1(read);
6676 src2 : EX1(read);
6677 INS01 : ISS; // Dual issue as instruction 0 or 1
6678 ALU : EX2;
6679 %}
6680
6681 // Integer ALU reg-reg operation with constant shift
6682 // Shifted register must be available in LATE_ISS instead of EX1
6683 // Eg. ADD x0, x1, x2, LSL #2
6684 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6685 %{
6686 single_instruction;
6687 dst : EX2(write);
6688 src1 : EX1(read);
6689 src2 : ISS(read);
6690 INS01 : ISS;
6691 ALU : EX2;
6692 %}
6693
6694 // Integer ALU reg operation with constant shift
6695 // Eg. LSL x0, x1, #shift
6696 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6697 %{
6698 single_instruction;
6699 dst : EX2(write);
6700 src1 : ISS(read);
6701 INS01 : ISS;
6702 ALU : EX2;
6703 %}
6704
6705 // Integer ALU reg-reg operation with variable shift
6706 // Both operands must be available in LATE_ISS instead of EX1
6707 // Result is available in EX1 instead of EX2
6708 // Eg. LSLV x0, x1, x2
6709 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6710 %{
6711 single_instruction;
6712 dst : EX1(write);
6713 src1 : ISS(read);
6714 src2 : ISS(read);
6715 INS01 : ISS;
6716 ALU : EX1;
6717 %}
6718
6719 // Integer ALU reg-reg operation with extract
6720 // As for _vshift above, but result generated in EX2
6721 // Eg. EXTR x0, x1, x2, #N
6722 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6723 %{
6724 single_instruction;
6725 dst : EX2(write);
6726 src1 : ISS(read);
6727 src2 : ISS(read);
6728 INS1 : ISS; // Can only dual issue as Instruction 1
6729 ALU : EX1;
6730 %}
6731
6732 // Integer ALU reg operation
6733 // Eg. NEG x0, x1
6734 pipe_class ialu_reg(iRegI dst, iRegI src)
6735 %{
6736 single_instruction;
6737 dst : EX2(write);
6738 src : EX1(read);
6739 INS01 : ISS;
6740 ALU : EX2;
6741 %}
6742
6743 // Integer ALU reg mmediate operation
6744 // Eg. ADD x0, x1, #N
6745 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6746 %{
6747 single_instruction;
6748 dst : EX2(write);
6749 src1 : EX1(read);
6750 INS01 : ISS;
6751 ALU : EX2;
6752 %}
6753
6754 // Integer ALU immediate operation (no source operands)
6755 // Eg. MOV x0, #N
6756 pipe_class ialu_imm(iRegI dst)
6757 %{
6758 single_instruction;
6759 dst : EX1(write);
6760 INS01 : ISS;
6761 ALU : EX1;
6762 %}
6763
6764 //------- Compare operation -------------------------------
6765
6766 // Compare reg-reg
6767 // Eg. CMP x0, x1
6768 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6769 %{
6770 single_instruction;
6771 // fixed_latency(16);
6772 cr : EX2(write);
6773 op1 : EX1(read);
6774 op2 : EX1(read);
6775 INS01 : ISS;
6776 ALU : EX2;
6777 %}
6778
6779 // Compare reg-reg
6780 // Eg. CMP x0, #N
6781 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6782 %{
6783 single_instruction;
6784 // fixed_latency(16);
6785 cr : EX2(write);
6786 op1 : EX1(read);
6787 INS01 : ISS;
6788 ALU : EX2;
6789 %}
6790
6791 //------- Conditional instructions ------------------------
6792
6793 // Conditional no operands
6794 // Eg. CSINC x0, zr, zr, <cond>
6795 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6796 %{
6797 single_instruction;
6798 cr : EX1(read);
6799 dst : EX2(write);
6800 INS01 : ISS;
6801 ALU : EX2;
6802 %}
6803
6804 // Conditional 2 operand
6805 // EG. CSEL X0, X1, X2, <cond>
6806 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6807 %{
6808 single_instruction;
6809 cr : EX1(read);
6810 src1 : EX1(read);
6811 src2 : EX1(read);
6812 dst : EX2(write);
6813 INS01 : ISS;
6814 ALU : EX2;
6815 %}
6816
6817 // Conditional 2 operand
6818 // EG. CSEL X0, X1, X2, <cond>
6819 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6820 %{
6821 single_instruction;
6822 cr : EX1(read);
6823 src : EX1(read);
6824 dst : EX2(write);
6825 INS01 : ISS;
6826 ALU : EX2;
6827 %}
6828
6829 //------- Multiply pipeline operations --------------------
6830
6831 // Multiply reg-reg
6832 // Eg. MUL w0, w1, w2
6833 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6834 %{
6835 single_instruction;
6836 dst : WR(write);
6837 src1 : ISS(read);
6838 src2 : ISS(read);
6839 INS01 : ISS;
6840 MAC : WR;
6841 %}
6842
6843 // Multiply accumulate
6844 // Eg. MADD w0, w1, w2, w3
6845 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6846 %{
6847 single_instruction;
6848 dst : WR(write);
6849 src1 : ISS(read);
6850 src2 : ISS(read);
6851 src3 : ISS(read);
6852 INS01 : ISS;
6853 MAC : WR;
6854 %}
6855
6856 // Eg. MUL w0, w1, w2
6857 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6858 %{
6859 single_instruction;
6860 fixed_latency(3); // Maximum latency for 64 bit mul
6861 dst : WR(write);
6862 src1 : ISS(read);
6863 src2 : ISS(read);
6864 INS01 : ISS;
6865 MAC : WR;
6866 %}
6867
6868 // Multiply accumulate
6869 // Eg. MADD w0, w1, w2, w3
6870 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6871 %{
6872 single_instruction;
6873 fixed_latency(3); // Maximum latency for 64 bit mul
6874 dst : WR(write);
6875 src1 : ISS(read);
6876 src2 : ISS(read);
6877 src3 : ISS(read);
6878 INS01 : ISS;
6879 MAC : WR;
6880 %}
6881
6882 //------- Divide pipeline operations --------------------
6883
6884 // Eg. SDIV w0, w1, w2
6885 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6886 %{
6887 single_instruction;
6888 fixed_latency(8); // Maximum latency for 32 bit divide
6889 dst : WR(write);
6890 src1 : ISS(read);
6891 src2 : ISS(read);
6892 INS0 : ISS; // Can only dual issue as instruction 0
6893 DIV : WR;
6894 %}
6895
6896 // Eg. SDIV x0, x1, x2
6897 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6898 %{
6899 single_instruction;
6900 fixed_latency(16); // Maximum latency for 64 bit divide
6901 dst : WR(write);
6902 src1 : ISS(read);
6903 src2 : ISS(read);
6904 INS0 : ISS; // Can only dual issue as instruction 0
6905 DIV : WR;
6906 %}
6907
6908 //------- Load pipeline operations ------------------------
6909
6910 // Load - prefetch
6911 // Eg. PFRM <mem>
6912 pipe_class iload_prefetch(memory mem)
6913 %{
6914 single_instruction;
6915 mem : ISS(read);
6916 INS01 : ISS;
6917 LDST : WR;
6918 %}
6919
6920 // Load - reg, mem
6921 // Eg. LDR x0, <mem>
6922 pipe_class iload_reg_mem(iRegI dst, memory mem)
6923 %{
6924 single_instruction;
6925 dst : WR(write);
6926 mem : ISS(read);
6927 INS01 : ISS;
6928 LDST : WR;
6929 %}
6930
6931 // Load - reg, reg
6932 // Eg. LDR x0, [sp, x1]
6933 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6934 %{
6935 single_instruction;
6936 dst : WR(write);
6937 src : ISS(read);
6938 INS01 : ISS;
6939 LDST : WR;
6940 %}
6941
6942 //------- Store pipeline operations -----------------------
6943
6944 // Store - zr, mem
6945 // Eg. STR zr, <mem>
6946 pipe_class istore_mem(memory mem)
6947 %{
6948 single_instruction;
6949 mem : ISS(read);
6950 INS01 : ISS;
6951 LDST : WR;
6952 %}
6953
6954 // Store - reg, mem
6955 // Eg. STR x0, <mem>
6956 pipe_class istore_reg_mem(iRegI src, memory mem)
6957 %{
6958 single_instruction;
6959 mem : ISS(read);
6960 src : EX2(read);
6961 INS01 : ISS;
6962 LDST : WR;
6963 %}
6964
6965 // Store - reg, reg
6966 // Eg. STR x0, [sp, x1]
6967 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6968 %{
6969 single_instruction;
6970 dst : ISS(read);
6971 src : EX2(read);
6972 INS01 : ISS;
6973 LDST : WR;
6974 %}
6975
6976 //------- Store pipeline operations -----------------------
6977
6978 // Branch
6979 pipe_class pipe_branch()
6980 %{
6981 single_instruction;
6982 INS01 : ISS;
6983 BRANCH : EX1;
6984 %}
6985
6986 // Conditional branch
6987 pipe_class pipe_branch_cond(rFlagsReg cr)
6988 %{
6989 single_instruction;
6990 cr : EX1(read);
6991 INS01 : ISS;
6992 BRANCH : EX1;
6993 %}
6994
6995 // Compare & Branch
6996 // EG. CBZ/CBNZ
6997 pipe_class pipe_cmp_branch(iRegI op1)
6998 %{
6999 single_instruction;
7000 op1 : EX1(read);
7001 INS01 : ISS;
7002 BRANCH : EX1;
7003 %}
7004
7005 //------- Synchronisation operations ----------------------
7006
7007 // Any operation requiring serialization.
7008 // EG. DMB/Atomic Ops/Load Acquire/Str Release
7009 pipe_class pipe_serial()
7010 %{
7011 single_instruction;
7012 force_serialization;
7013 fixed_latency(16);
7014 INS01 : ISS(2); // Cannot dual issue with any other instruction
7015 LDST : WR;
7016 %}
7017
7018 // Generic big/slow expanded idiom - also serialized
7019 pipe_class pipe_slow()
7020 %{
7021 instruction_count(10);
7022 multiple_bundles;
7023 force_serialization;
7024 fixed_latency(16);
7025 INS01 : ISS(2); // Cannot dual issue with any other instruction
7026 LDST : WR;
7027 %}
7028
7029 // Empty pipeline class
7030 pipe_class pipe_class_empty()
7031 %{
7032 single_instruction;
7033 fixed_latency(0);
7034 %}
7035
7036 // Default pipeline class.
7037 pipe_class pipe_class_default()
7038 %{
7039 single_instruction;
7040 fixed_latency(2);
7041 %}
7042
7043 // Pipeline class for compares.
7044 pipe_class pipe_class_compare()
7045 %{
7046 single_instruction;
7047 fixed_latency(16);
7048 %}
7049
7050 // Pipeline class for memory operations.
7051 pipe_class pipe_class_memory()
7052 %{
7053 single_instruction;
7054 fixed_latency(16);
7055 %}
7056
7057 // Pipeline class for call.
7058 pipe_class pipe_class_call()
7059 %{
7060 single_instruction;
7061 fixed_latency(100);
7062 %}
7063
7064 // Define the class for the Nop node.
7065 define %{
7066 MachNop = pipe_class_empty;
7067 %}
7068
7069 %}
7070 //----------INSTRUCTIONS-------------------------------------------------------
7071 //
7072 // match -- States which machine-independent subtree may be replaced
7073 // by this instruction.
7074 // ins_cost -- The estimated cost of this instruction is used by instruction
7075 // selection to identify a minimum cost tree of machine
7076 // instructions that matches a tree of machine-independent
7077 // instructions.
7078 // format -- A string providing the disassembly for this instruction.
7079 // The value of an instruction's operand may be inserted
7080 // by referring to it with a '$' prefix.
7081 // opcode -- Three instruction opcodes may be provided. These are referred
7082 // to within an encode class as $primary, $secondary, and $tertiary
7083 // rrspectively. The primary opcode is commonly used to
7084 // indicate the type of machine instruction, while secondary
7085 // and tertiary are often used for prefix options or addressing
7086 // modes.
7087 // ins_encode -- A list of encode classes with parameters. The encode class
7088 // name must have been defined in an 'enc_class' specification
7089 // in the encode section of the architecture description.
7090
7091 // ============================================================================
7092 // Memory (Load/Store) Instructions
7093
7094 // Load Instructions
7095
7096 // Load Byte (8 bit signed)
7097 instruct loadB(iRegINoSp dst, memory1 mem)
7098 %{
7099 match(Set dst (LoadB mem));
7100 predicate(!needs_acquiring_load(n));
7101
7102 ins_cost(4 * INSN_COST);
7103 format %{ "ldrsbw $dst, $mem\t# byte" %}
7104
7105 ins_encode(aarch64_enc_ldrsbw(dst, mem));
7106
7107 ins_pipe(iload_reg_mem);
7108 %}
7109
7110 // Load Byte (8 bit signed) into long
7111 instruct loadB2L(iRegLNoSp dst, memory1 mem)
7112 %{
7113 match(Set dst (ConvI2L (LoadB mem)));
7114 predicate(!needs_acquiring_load(n->in(1)));
7115
7116 ins_cost(4 * INSN_COST);
7117 format %{ "ldrsb $dst, $mem\t# byte" %}
7118
7119 ins_encode(aarch64_enc_ldrsb(dst, mem));
7120
7121 ins_pipe(iload_reg_mem);
7122 %}
7123
7124 // Load Byte (8 bit unsigned)
7125 instruct loadUB(iRegINoSp dst, memory1 mem)
7126 %{
7127 match(Set dst (LoadUB mem));
7128 predicate(!needs_acquiring_load(n));
7129
7130 ins_cost(4 * INSN_COST);
7131 format %{ "ldrbw $dst, $mem\t# byte" %}
7132
7133 ins_encode(aarch64_enc_ldrb(dst, mem));
7134
7135 ins_pipe(iload_reg_mem);
7136 %}
7137
7138 // Load Byte (8 bit unsigned) into long
7139 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
7140 %{
7141 match(Set dst (ConvI2L (LoadUB mem)));
7142 predicate(!needs_acquiring_load(n->in(1)));
7143
7144 ins_cost(4 * INSN_COST);
7145 format %{ "ldrb $dst, $mem\t# byte" %}
7146
7147 ins_encode(aarch64_enc_ldrb(dst, mem));
7148
7149 ins_pipe(iload_reg_mem);
7150 %}
7151
7152 // Load Short (16 bit signed)
7153 instruct loadS(iRegINoSp dst, memory2 mem)
7154 %{
7155 match(Set dst (LoadS mem));
7156 predicate(!needs_acquiring_load(n));
7157
7158 ins_cost(4 * INSN_COST);
7159 format %{ "ldrshw $dst, $mem\t# short" %}
7160
7161 ins_encode(aarch64_enc_ldrshw(dst, mem));
7162
7163 ins_pipe(iload_reg_mem);
7164 %}
7165
7166 // Load Short (16 bit signed) into long
7167 instruct loadS2L(iRegLNoSp dst, memory2 mem)
7168 %{
7169 match(Set dst (ConvI2L (LoadS mem)));
7170 predicate(!needs_acquiring_load(n->in(1)));
7171
7172 ins_cost(4 * INSN_COST);
7173 format %{ "ldrsh $dst, $mem\t# short" %}
7174
7175 ins_encode(aarch64_enc_ldrsh(dst, mem));
7176
7177 ins_pipe(iload_reg_mem);
7178 %}
7179
7180 // Load Char (16 bit unsigned)
7181 instruct loadUS(iRegINoSp dst, memory2 mem)
7182 %{
7183 match(Set dst (LoadUS mem));
7184 predicate(!needs_acquiring_load(n));
7185
7186 ins_cost(4 * INSN_COST);
7187 format %{ "ldrh $dst, $mem\t# short" %}
7188
7189 ins_encode(aarch64_enc_ldrh(dst, mem));
7190
7191 ins_pipe(iload_reg_mem);
7192 %}
7193
7194 // Load Short/Char (16 bit unsigned) into long
7195 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
7196 %{
7197 match(Set dst (ConvI2L (LoadUS mem)));
7198 predicate(!needs_acquiring_load(n->in(1)));
7199
7200 ins_cost(4 * INSN_COST);
7201 format %{ "ldrh $dst, $mem\t# short" %}
7202
7203 ins_encode(aarch64_enc_ldrh(dst, mem));
7204
7205 ins_pipe(iload_reg_mem);
7206 %}
7207
7208 // Load Integer (32 bit signed)
7209 instruct loadI(iRegINoSp dst, memory4 mem)
7210 %{
7211 match(Set dst (LoadI mem));
7212 predicate(!needs_acquiring_load(n));
7213
7214 ins_cost(4 * INSN_COST);
7215 format %{ "ldrw $dst, $mem\t# int" %}
7216
7217 ins_encode(aarch64_enc_ldrw(dst, mem));
7218
7219 ins_pipe(iload_reg_mem);
7220 %}
7221
7222 // Load Integer (32 bit signed) into long
7223 instruct loadI2L(iRegLNoSp dst, memory4 mem)
7224 %{
7225 match(Set dst (ConvI2L (LoadI mem)));
7226 predicate(!needs_acquiring_load(n->in(1)));
7227
7228 ins_cost(4 * INSN_COST);
7229 format %{ "ldrsw $dst, $mem\t# int" %}
7230
7231 ins_encode(aarch64_enc_ldrsw(dst, mem));
7232
7233 ins_pipe(iload_reg_mem);
7234 %}
7235
7236 // Load Integer (32 bit unsigned) into long
7237 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
7238 %{
7239 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7240 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
7241
7242 ins_cost(4 * INSN_COST);
7243 format %{ "ldrw $dst, $mem\t# int" %}
7244
7245 ins_encode(aarch64_enc_ldrw(dst, mem));
7246
7247 ins_pipe(iload_reg_mem);
7248 %}
7249
7250 // Load Long (64 bit signed)
7251 instruct loadL(iRegLNoSp dst, memory8 mem)
7252 %{
7253 match(Set dst (LoadL mem));
7254 predicate(!needs_acquiring_load(n));
7255
7256 ins_cost(4 * INSN_COST);
7257 format %{ "ldr $dst, $mem\t# int" %}
7258
7259 ins_encode(aarch64_enc_ldr(dst, mem));
7260
7261 ins_pipe(iload_reg_mem);
7262 %}
7263
7264 // Load Range
7265 instruct loadRange(iRegINoSp dst, memory4 mem)
7266 %{
7267 match(Set dst (LoadRange mem));
7268
7269 ins_cost(4 * INSN_COST);
7270 format %{ "ldrw $dst, $mem\t# range" %}
7271
7272 ins_encode(aarch64_enc_ldrw(dst, mem));
7273
7274 ins_pipe(iload_reg_mem);
7275 %}
7276
7277 // Load Pointer
7278 instruct loadP(iRegPNoSp dst, memory8 mem)
7279 %{
7280 match(Set dst (LoadP mem));
7281 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
7282
7283 ins_cost(4 * INSN_COST);
7284 format %{ "ldr $dst, $mem\t# ptr" %}
7285
7286 ins_encode(aarch64_enc_ldr(dst, mem));
7287
7288 ins_pipe(iload_reg_mem);
7289 %}
7290
7291 // Load Compressed Pointer
7292 instruct loadN(iRegNNoSp dst, memory4 mem)
7293 %{
7294 match(Set dst (LoadN mem));
7295 predicate(!needs_acquiring_load(n));
7296
7297 ins_cost(4 * INSN_COST);
7298 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
7299
7300 ins_encode(aarch64_enc_ldrw(dst, mem));
7301
7302 ins_pipe(iload_reg_mem);
7303 %}
7304
7305 // Load Klass Pointer
7306 instruct loadKlass(iRegPNoSp dst, memory8 mem)
7307 %{
7308 match(Set dst (LoadKlass mem));
7309 predicate(!needs_acquiring_load(n));
7310
7311 ins_cost(4 * INSN_COST);
7312 format %{ "ldr $dst, $mem\t# class" %}
7313
7314 ins_encode(aarch64_enc_ldr(dst, mem));
7315
7316 ins_pipe(iload_reg_mem);
7317 %}
7318
7319 // Load Narrow Klass Pointer
7320 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
7321 %{
7322 match(Set dst (LoadNKlass mem));
7323 predicate(!needs_acquiring_load(n));
7324
7325 ins_cost(4 * INSN_COST);
7326 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7327
7328 ins_encode(aarch64_enc_ldrw(dst, mem));
7329
7330 ins_pipe(iload_reg_mem);
7331 %}
7332
7333 // Load Float
7334 instruct loadF(vRegF dst, memory4 mem)
7335 %{
7336 match(Set dst (LoadF mem));
7337 predicate(!needs_acquiring_load(n));
7338
7339 ins_cost(4 * INSN_COST);
7340 format %{ "ldrs $dst, $mem\t# float" %}
7341
7342 ins_encode( aarch64_enc_ldrs(dst, mem) );
7343
7344 ins_pipe(pipe_class_memory);
7345 %}
7346
7347 // Load Double
7348 instruct loadD(vRegD dst, memory8 mem)
7349 %{
7350 match(Set dst (LoadD mem));
7351 predicate(!needs_acquiring_load(n));
7352
7353 ins_cost(4 * INSN_COST);
7354 format %{ "ldrd $dst, $mem\t# double" %}
7355
7356 ins_encode( aarch64_enc_ldrd(dst, mem) );
7357
7358 ins_pipe(pipe_class_memory);
7359 %}
7360
7361
7362 // Load Int Constant
7363 instruct loadConI(iRegINoSp dst, immI src)
7364 %{
7365 match(Set dst src);
7366
7367 ins_cost(INSN_COST);
7368 format %{ "mov $dst, $src\t# int" %}
7369
7370 ins_encode( aarch64_enc_movw_imm(dst, src) );
7371
7372 ins_pipe(ialu_imm);
7373 %}
7374
7375 // Load Long Constant
7376 instruct loadConL(iRegLNoSp dst, immL src)
7377 %{
7378 match(Set dst src);
7379
7380 ins_cost(INSN_COST);
7381 format %{ "mov $dst, $src\t# long" %}
7382
7383 ins_encode( aarch64_enc_mov_imm(dst, src) );
7384
7385 ins_pipe(ialu_imm);
7386 %}
7387
7388 // Load Pointer Constant
7389
7390 instruct loadConP(iRegPNoSp dst, immP con)
7391 %{
7392 match(Set dst con);
7393
7394 ins_cost(INSN_COST * 4);
7395 format %{
7396 "mov $dst, $con\t# ptr\n\t"
7397 %}
7398
7399 ins_encode(aarch64_enc_mov_p(dst, con));
7400
7401 ins_pipe(ialu_imm);
7402 %}
7403
7404 // Load Null Pointer Constant
7405
7406 instruct loadConP0(iRegPNoSp dst, immP0 con)
7407 %{
7408 match(Set dst con);
7409
7410 ins_cost(INSN_COST);
7411 format %{ "mov $dst, $con\t# NULL ptr" %}
7412
7413 ins_encode(aarch64_enc_mov_p0(dst, con));
7414
7415 ins_pipe(ialu_imm);
7416 %}
7417
7418 // Load Pointer Constant One
7419
7420 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7421 %{
7422 match(Set dst con);
7423
7424 ins_cost(INSN_COST);
7425 format %{ "mov $dst, $con\t# NULL ptr" %}
7426
7427 ins_encode(aarch64_enc_mov_p1(dst, con));
7428
7429 ins_pipe(ialu_imm);
7430 %}
7431
7432 // Load Byte Map Base Constant
7433
7434 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7435 %{
7436 match(Set dst con);
7437
7438 ins_cost(INSN_COST);
7439 format %{ "adr $dst, $con\t# Byte Map Base" %}
7440
7441 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7442
7443 ins_pipe(ialu_imm);
7444 %}
7445
7446 // Load Narrow Pointer Constant
7447
7448 instruct loadConN(iRegNNoSp dst, immN con)
7449 %{
7450 match(Set dst con);
7451
7452 ins_cost(INSN_COST * 4);
7453 format %{ "mov $dst, $con\t# compressed ptr" %}
7454
7455 ins_encode(aarch64_enc_mov_n(dst, con));
7456
7457 ins_pipe(ialu_imm);
7458 %}
7459
7460 // Load Narrow Null Pointer Constant
7461
7462 instruct loadConN0(iRegNNoSp dst, immN0 con)
7463 %{
7464 match(Set dst con);
7465
7466 ins_cost(INSN_COST);
7467 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
7468
7469 ins_encode(aarch64_enc_mov_n0(dst, con));
7470
7471 ins_pipe(ialu_imm);
7472 %}
7473
7474 // Load Narrow Klass Constant
7475
7476 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7477 %{
7478 match(Set dst con);
7479
7480 ins_cost(INSN_COST);
7481 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7482
7483 ins_encode(aarch64_enc_mov_nk(dst, con));
7484
7485 ins_pipe(ialu_imm);
7486 %}
7487
7488 // Load Packed Float Constant
7489
7490 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7491 match(Set dst con);
7492 ins_cost(INSN_COST * 4);
7493 format %{ "fmovs $dst, $con"%}
7494 ins_encode %{
7495 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7496 %}
7497
7498 ins_pipe(fp_imm_s);
7499 %}
7500
7501 // Load Float Constant
7502
7503 instruct loadConF(vRegF dst, immF con) %{
7504 match(Set dst con);
7505
7506 ins_cost(INSN_COST * 4);
7507
7508 format %{
7509 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7510 %}
7511
7512 ins_encode %{
7513 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7514 %}
7515
7516 ins_pipe(fp_load_constant_s);
7517 %}
7518
7519 // Load Packed Double Constant
7520
7521 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7522 match(Set dst con);
7523 ins_cost(INSN_COST);
7524 format %{ "fmovd $dst, $con"%}
7525 ins_encode %{
7526 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7527 %}
7528
7529 ins_pipe(fp_imm_d);
7530 %}
7531
7532 // Load Double Constant
7533
7534 instruct loadConD(vRegD dst, immD con) %{
7535 match(Set dst con);
7536
7537 ins_cost(INSN_COST * 5);
7538 format %{
7539 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7540 %}
7541
7542 ins_encode %{
7543 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7544 %}
7545
7546 ins_pipe(fp_load_constant_d);
7547 %}
7548
7549 // Store Instructions
7550
7551 // Store CMS card-mark Immediate
7552 instruct storeimmCM0(immI0 zero, memory1 mem)
7553 %{
7554 match(Set mem (StoreCM mem zero));
7555
7556 ins_cost(INSN_COST);
7557 format %{ "storestore (elided)\n\t"
7558 "strb zr, $mem\t# byte" %}
7559
7560 ins_encode(aarch64_enc_strb0(mem));
7561
7562 ins_pipe(istore_mem);
7563 %}
7564
7565 // Store CMS card-mark Immediate with intervening StoreStore
7566 // needed when using CMS with no conditional card marking
7567 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
7568 %{
7569 match(Set mem (StoreCM mem zero));
7570
7571 ins_cost(INSN_COST * 2);
7572 format %{ "storestore\n\t"
7573 "dmb ishst"
7574 "\n\tstrb zr, $mem\t# byte" %}
7575
7576 ins_encode(aarch64_enc_strb0_ordered(mem));
7577
7578 ins_pipe(istore_mem);
7579 %}
7580
7581 // Store Byte
7582 instruct storeB(iRegIorL2I src, memory1 mem)
7583 %{
7584 match(Set mem (StoreB mem src));
7585 predicate(!needs_releasing_store(n));
7586
7587 ins_cost(INSN_COST);
7588 format %{ "strb $src, $mem\t# byte" %}
7589
7590 ins_encode(aarch64_enc_strb(src, mem));
7591
7592 ins_pipe(istore_reg_mem);
7593 %}
7594
7595
7596 instruct storeimmB0(immI0 zero, memory1 mem)
7597 %{
7598 match(Set mem (StoreB mem zero));
7599 predicate(!needs_releasing_store(n));
7600
7601 ins_cost(INSN_COST);
7602 format %{ "strb rscractch2, $mem\t# byte" %}
7603
7604 ins_encode(aarch64_enc_strb0(mem));
7605
7606 ins_pipe(istore_mem);
7607 %}
7608
7609 // Store Char/Short
7610 instruct storeC(iRegIorL2I src, memory2 mem)
7611 %{
7612 match(Set mem (StoreC mem src));
7613 predicate(!needs_releasing_store(n));
7614
7615 ins_cost(INSN_COST);
7616 format %{ "strh $src, $mem\t# short" %}
7617
7618 ins_encode(aarch64_enc_strh(src, mem));
7619
7620 ins_pipe(istore_reg_mem);
7621 %}
7622
7623 instruct storeimmC0(immI0 zero, memory2 mem)
7624 %{
7625 match(Set mem (StoreC mem zero));
7626 predicate(!needs_releasing_store(n));
7627
7628 ins_cost(INSN_COST);
7629 format %{ "strh zr, $mem\t# short" %}
7630
7631 ins_encode(aarch64_enc_strh0(mem));
7632
7633 ins_pipe(istore_mem);
7634 %}
7635
7636 // Store Integer
7637
7638 instruct storeI(iRegIorL2I src, memory4 mem)
7639 %{
7640 match(Set mem(StoreI mem src));
7641 predicate(!needs_releasing_store(n));
7642
7643 ins_cost(INSN_COST);
7644 format %{ "strw $src, $mem\t# int" %}
7645
7646 ins_encode(aarch64_enc_strw(src, mem));
7647
7648 ins_pipe(istore_reg_mem);
7649 %}
7650
7651 instruct storeimmI0(immI0 zero, memory4 mem)
7652 %{
7653 match(Set mem(StoreI mem zero));
7654 predicate(!needs_releasing_store(n));
7655
7656 ins_cost(INSN_COST);
7657 format %{ "strw zr, $mem\t# int" %}
7658
7659 ins_encode(aarch64_enc_strw0(mem));
7660
7661 ins_pipe(istore_mem);
7662 %}
7663
7664 // Store Long (64 bit signed)
7665 instruct storeL(iRegL src, memory8 mem)
7666 %{
7667 match(Set mem (StoreL mem src));
7668 predicate(!needs_releasing_store(n));
7669
7670 ins_cost(INSN_COST);
7671 format %{ "str $src, $mem\t# int" %}
7672
7673 ins_encode(aarch64_enc_str(src, mem));
7674
7675 ins_pipe(istore_reg_mem);
7676 %}
7677
7678 // Store Long (64 bit signed)
7679 instruct storeimmL0(immL0 zero, memory8 mem)
7680 %{
7681 match(Set mem (StoreL mem zero));
7682 predicate(!needs_releasing_store(n));
7683
7684 ins_cost(INSN_COST);
7685 format %{ "str zr, $mem\t# int" %}
7686
7687 ins_encode(aarch64_enc_str0(mem));
7688
7689 ins_pipe(istore_mem);
7690 %}
7691
7692 // Store Pointer
7693 instruct storeP(iRegP src, memory8 mem)
7694 %{
7695 match(Set mem (StoreP mem src));
7696 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7697
7698 ins_cost(INSN_COST);
7699 format %{ "str $src, $mem\t# ptr" %}
7700
7701 ins_encode(aarch64_enc_str(src, mem));
7702
7703 ins_pipe(istore_reg_mem);
7704 %}
7705
7706 // Store Pointer
7707 instruct storeimmP0(immP0 zero, memory8 mem)
7708 %{
7709 match(Set mem (StoreP mem zero));
7710 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7711
7712 ins_cost(INSN_COST);
7713 format %{ "str zr, $mem\t# ptr" %}
7714
7715 ins_encode(aarch64_enc_str0(mem));
7716
7717 ins_pipe(istore_mem);
7718 %}
7719
7720 // Store Compressed Pointer
7721 instruct storeN(iRegN src, memory4 mem)
7722 %{
7723 match(Set mem (StoreN mem src));
7724 predicate(!needs_releasing_store(n));
7725
7726 ins_cost(INSN_COST);
7727 format %{ "strw $src, $mem\t# compressed ptr" %}
7728
7729 ins_encode(aarch64_enc_strw(src, mem));
7730
7731 ins_pipe(istore_reg_mem);
7732 %}
7733
7734 instruct storeImmN0(immN0 zero, memory4 mem)
7735 %{
7736 match(Set mem (StoreN mem zero));
7737 predicate(!needs_releasing_store(n));
7738
7739 ins_cost(INSN_COST);
7740 format %{ "strw zr, $mem\t# compressed ptr" %}
7741
7742 ins_encode(aarch64_enc_strw0(mem));
7743
7744 ins_pipe(istore_mem);
7745 %}
7746
7747 // Store Float
7748 instruct storeF(vRegF src, memory4 mem)
7749 %{
7750 match(Set mem (StoreF mem src));
7751 predicate(!needs_releasing_store(n));
7752
7753 ins_cost(INSN_COST);
7754 format %{ "strs $src, $mem\t# float" %}
7755
7756 ins_encode( aarch64_enc_strs(src, mem) );
7757
7758 ins_pipe(pipe_class_memory);
7759 %}
7760
7761 // TODO
7762 // implement storeImmF0 and storeFImmPacked
7763
7764 // Store Double
7765 instruct storeD(vRegD src, memory8 mem)
7766 %{
7767 match(Set mem (StoreD mem src));
7768 predicate(!needs_releasing_store(n));
7769
7770 ins_cost(INSN_COST);
7771 format %{ "strd $src, $mem\t# double" %}
7772
7773 ins_encode( aarch64_enc_strd(src, mem) );
7774
7775 ins_pipe(pipe_class_memory);
7776 %}
7777
7778 // Store Compressed Klass Pointer
7779 instruct storeNKlass(iRegN src, memory4 mem)
7780 %{
7781 predicate(!needs_releasing_store(n));
7782 match(Set mem (StoreNKlass mem src));
7783
7784 ins_cost(INSN_COST);
7785 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7786
7787 ins_encode(aarch64_enc_strw(src, mem));
7788
7789 ins_pipe(istore_reg_mem);
7790 %}
7791
7792 // TODO
7793 // implement storeImmD0 and storeDImmPacked
7794
7795 // prefetch instructions
7796 // Must be safe to execute with invalid address (cannot fault).
7797
7798 instruct prefetchalloc( memory8 mem ) %{
7799 match(PrefetchAllocation mem);
7800
7801 ins_cost(INSN_COST);
7802 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7803
7804 ins_encode( aarch64_enc_prefetchw(mem) );
7805
7806 ins_pipe(iload_prefetch);
7807 %}
7808
7809 // ---------------- volatile loads and stores ----------------
7810
7811 // Load Byte (8 bit signed)
7812 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7813 %{
7814 match(Set dst (LoadB mem));
7815
7816 ins_cost(VOLATILE_REF_COST);
7817 format %{ "ldarsb $dst, $mem\t# byte" %}
7818
7819 ins_encode(aarch64_enc_ldarsb(dst, mem));
7820
7821 ins_pipe(pipe_serial);
7822 %}
7823
7824 // Load Byte (8 bit signed) into long
7825 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7826 %{
7827 match(Set dst (ConvI2L (LoadB mem)));
7828
7829 ins_cost(VOLATILE_REF_COST);
7830 format %{ "ldarsb $dst, $mem\t# byte" %}
7831
7832 ins_encode(aarch64_enc_ldarsb(dst, mem));
7833
7834 ins_pipe(pipe_serial);
7835 %}
7836
7837 // Load Byte (8 bit unsigned)
7838 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7839 %{
7840 match(Set dst (LoadUB mem));
7841
7842 ins_cost(VOLATILE_REF_COST);
7843 format %{ "ldarb $dst, $mem\t# byte" %}
7844
7845 ins_encode(aarch64_enc_ldarb(dst, mem));
7846
7847 ins_pipe(pipe_serial);
7848 %}
7849
7850 // Load Byte (8 bit unsigned) into long
7851 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7852 %{
7853 match(Set dst (ConvI2L (LoadUB mem)));
7854
7855 ins_cost(VOLATILE_REF_COST);
7856 format %{ "ldarb $dst, $mem\t# byte" %}
7857
7858 ins_encode(aarch64_enc_ldarb(dst, mem));
7859
7860 ins_pipe(pipe_serial);
7861 %}
7862
7863 // Load Short (16 bit signed)
7864 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7865 %{
7866 match(Set dst (LoadS mem));
7867
7868 ins_cost(VOLATILE_REF_COST);
7869 format %{ "ldarshw $dst, $mem\t# short" %}
7870
7871 ins_encode(aarch64_enc_ldarshw(dst, mem));
7872
7873 ins_pipe(pipe_serial);
7874 %}
7875
7876 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7877 %{
7878 match(Set dst (LoadUS mem));
7879
7880 ins_cost(VOLATILE_REF_COST);
7881 format %{ "ldarhw $dst, $mem\t# short" %}
7882
7883 ins_encode(aarch64_enc_ldarhw(dst, mem));
7884
7885 ins_pipe(pipe_serial);
7886 %}
7887
7888 // Load Short/Char (16 bit unsigned) into long
7889 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7890 %{
7891 match(Set dst (ConvI2L (LoadUS mem)));
7892
7893 ins_cost(VOLATILE_REF_COST);
7894 format %{ "ldarh $dst, $mem\t# short" %}
7895
7896 ins_encode(aarch64_enc_ldarh(dst, mem));
7897
7898 ins_pipe(pipe_serial);
7899 %}
7900
7901 // Load Short/Char (16 bit signed) into long
7902 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7903 %{
7904 match(Set dst (ConvI2L (LoadS mem)));
7905
7906 ins_cost(VOLATILE_REF_COST);
7907 format %{ "ldarh $dst, $mem\t# short" %}
7908
7909 ins_encode(aarch64_enc_ldarsh(dst, mem));
7910
7911 ins_pipe(pipe_serial);
7912 %}
7913
7914 // Load Integer (32 bit signed)
7915 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7916 %{
7917 match(Set dst (LoadI mem));
7918
7919 ins_cost(VOLATILE_REF_COST);
7920 format %{ "ldarw $dst, $mem\t# int" %}
7921
7922 ins_encode(aarch64_enc_ldarw(dst, mem));
7923
7924 ins_pipe(pipe_serial);
7925 %}
7926
7927 // Load Integer (32 bit unsigned) into long
7928 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7929 %{
7930 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7931
7932 ins_cost(VOLATILE_REF_COST);
7933 format %{ "ldarw $dst, $mem\t# int" %}
7934
7935 ins_encode(aarch64_enc_ldarw(dst, mem));
7936
7937 ins_pipe(pipe_serial);
7938 %}
7939
7940 // Load Long (64 bit signed)
7941 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7942 %{
7943 match(Set dst (LoadL mem));
7944
7945 ins_cost(VOLATILE_REF_COST);
7946 format %{ "ldar $dst, $mem\t# int" %}
7947
7948 ins_encode(aarch64_enc_ldar(dst, mem));
7949
7950 ins_pipe(pipe_serial);
7951 %}
7952
7953 // Load Pointer
7954 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7955 %{
7956 match(Set dst (LoadP mem));
7957 predicate(n->as_Load()->barrier_data() == 0);
7958
7959 ins_cost(VOLATILE_REF_COST);
7960 format %{ "ldar $dst, $mem\t# ptr" %}
7961
7962 ins_encode(aarch64_enc_ldar(dst, mem));
7963
7964 ins_pipe(pipe_serial);
7965 %}
7966
7967 // Load Compressed Pointer
7968 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7969 %{
7970 match(Set dst (LoadN mem));
7971
7972 ins_cost(VOLATILE_REF_COST);
7973 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7974
7975 ins_encode(aarch64_enc_ldarw(dst, mem));
7976
7977 ins_pipe(pipe_serial);
7978 %}
7979
7980 // Load Float
7981 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7982 %{
7983 match(Set dst (LoadF mem));
7984
7985 ins_cost(VOLATILE_REF_COST);
7986 format %{ "ldars $dst, $mem\t# float" %}
7987
7988 ins_encode( aarch64_enc_fldars(dst, mem) );
7989
7990 ins_pipe(pipe_serial);
7991 %}
7992
7993 // Load Double
7994 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7995 %{
7996 match(Set dst (LoadD mem));
7997
7998 ins_cost(VOLATILE_REF_COST);
7999 format %{ "ldard $dst, $mem\t# double" %}
8000
8001 ins_encode( aarch64_enc_fldard(dst, mem) );
8002
8003 ins_pipe(pipe_serial);
8004 %}
8005
8006 // Store Byte
8007 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8008 %{
8009 match(Set mem (StoreB mem src));
8010
8011 ins_cost(VOLATILE_REF_COST);
8012 format %{ "stlrb $src, $mem\t# byte" %}
8013
8014 ins_encode(aarch64_enc_stlrb(src, mem));
8015
8016 ins_pipe(pipe_class_memory);
8017 %}
8018
8019 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8020 %{
8021 match(Set mem (StoreB mem zero));
8022
8023 ins_cost(VOLATILE_REF_COST);
8024 format %{ "stlrb zr, $mem\t# byte" %}
8025
8026 ins_encode(aarch64_enc_stlrb0(mem));
8027
8028 ins_pipe(pipe_class_memory);
8029 %}
8030
8031 // Store Char/Short
8032 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8033 %{
8034 match(Set mem (StoreC mem src));
8035
8036 ins_cost(VOLATILE_REF_COST);
8037 format %{ "stlrh $src, $mem\t# short" %}
8038
8039 ins_encode(aarch64_enc_stlrh(src, mem));
8040
8041 ins_pipe(pipe_class_memory);
8042 %}
8043
8044 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8045 %{
8046 match(Set mem (StoreC mem zero));
8047
8048 ins_cost(VOLATILE_REF_COST);
8049 format %{ "stlrh zr, $mem\t# short" %}
8050
8051 ins_encode(aarch64_enc_stlrh0(mem));
8052
8053 ins_pipe(pipe_class_memory);
8054 %}
8055
8056 // Store Integer
8057
8058 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8059 %{
8060 match(Set mem(StoreI mem src));
8061
8062 ins_cost(VOLATILE_REF_COST);
8063 format %{ "stlrw $src, $mem\t# int" %}
8064
8065 ins_encode(aarch64_enc_stlrw(src, mem));
8066
8067 ins_pipe(pipe_class_memory);
8068 %}
8069
8070 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8071 %{
8072 match(Set mem(StoreI mem zero));
8073
8074 ins_cost(VOLATILE_REF_COST);
8075 format %{ "stlrw zr, $mem\t# int" %}
8076
8077 ins_encode(aarch64_enc_stlrw0(mem));
8078
8079 ins_pipe(pipe_class_memory);
8080 %}
8081
8082 // Store Long (64 bit signed)
8083 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
8084 %{
8085 match(Set mem (StoreL mem src));
8086
8087 ins_cost(VOLATILE_REF_COST);
8088 format %{ "stlr $src, $mem\t# int" %}
8089
8090 ins_encode(aarch64_enc_stlr(src, mem));
8091
8092 ins_pipe(pipe_class_memory);
8093 %}
8094
8095 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
8096 %{
8097 match(Set mem (StoreL mem zero));
8098
8099 ins_cost(VOLATILE_REF_COST);
8100 format %{ "stlr zr, $mem\t# int" %}
8101
8102 ins_encode(aarch64_enc_stlr0(mem));
8103
8104 ins_pipe(pipe_class_memory);
8105 %}
8106
8107 // Store Pointer
8108 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
8109 %{
8110 match(Set mem (StoreP mem src));
8111 predicate(n->as_Store()->barrier_data() == 0);
8112
8113 ins_cost(VOLATILE_REF_COST);
8114 format %{ "stlr $src, $mem\t# ptr" %}
8115
8116 ins_encode(aarch64_enc_stlr(src, mem));
8117
8118 ins_pipe(pipe_class_memory);
8119 %}
8120
8121 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
8122 %{
8123 match(Set mem (StoreP mem zero));
8124 predicate(n->as_Store()->barrier_data() == 0);
8125
8126 ins_cost(VOLATILE_REF_COST);
8127 format %{ "stlr zr, $mem\t# ptr" %}
8128
8129 ins_encode(aarch64_enc_stlr0(mem));
8130
8131 ins_pipe(pipe_class_memory);
8132 %}
8133
8134 // Store Compressed Pointer
8135 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
8136 %{
8137 match(Set mem (StoreN mem src));
8138
8139 ins_cost(VOLATILE_REF_COST);
8140 format %{ "stlrw $src, $mem\t# compressed ptr" %}
8141
8142 ins_encode(aarch64_enc_stlrw(src, mem));
8143
8144 ins_pipe(pipe_class_memory);
8145 %}
8146
8147 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
8148 %{
8149 match(Set mem (StoreN mem zero));
8150
8151 ins_cost(VOLATILE_REF_COST);
8152 format %{ "stlrw zr, $mem\t# compressed ptr" %}
8153
8154 ins_encode(aarch64_enc_stlrw0(mem));
8155
8156 ins_pipe(pipe_class_memory);
8157 %}
8158
8159 // Store Float
8160 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
8161 %{
8162 match(Set mem (StoreF mem src));
8163
8164 ins_cost(VOLATILE_REF_COST);
8165 format %{ "stlrs $src, $mem\t# float" %}
8166
8167 ins_encode( aarch64_enc_fstlrs(src, mem) );
8168
8169 ins_pipe(pipe_class_memory);
8170 %}
8171
8172 // TODO
8173 // implement storeImmF0 and storeFImmPacked
8174
8175 // Store Double
8176 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
8177 %{
8178 match(Set mem (StoreD mem src));
8179
8180 ins_cost(VOLATILE_REF_COST);
8181 format %{ "stlrd $src, $mem\t# double" %}
8182
8183 ins_encode( aarch64_enc_fstlrd(src, mem) );
8184
8185 ins_pipe(pipe_class_memory);
8186 %}
8187
8188 // ---------------- end of volatile loads and stores ----------------
8189
8190 instruct cacheWB(indirect addr)
8191 %{
8192 predicate(VM_Version::supports_data_cache_line_flush());
8193 match(CacheWB addr);
8194
8195 ins_cost(100);
8196 format %{"cache wb $addr" %}
8197 ins_encode %{
8198 assert($addr->index_position() < 0, "should be");
8199 assert($addr$$disp == 0, "should be");
8200 __ cache_wb(Address($addr$$base$$Register, 0));
8201 %}
8202 ins_pipe(pipe_slow); // XXX
8203 %}
8204
8205 instruct cacheWBPreSync()
8206 %{
8207 predicate(VM_Version::supports_data_cache_line_flush());
8208 match(CacheWBPreSync);
8209
8210 ins_cost(100);
8211 format %{"cache wb presync" %}
8212 ins_encode %{
8213 __ cache_wbsync(true);
8214 %}
8215 ins_pipe(pipe_slow); // XXX
8216 %}
8217
8218 instruct cacheWBPostSync()
8219 %{
8220 predicate(VM_Version::supports_data_cache_line_flush());
8221 match(CacheWBPostSync);
8222
8223 ins_cost(100);
8224 format %{"cache wb postsync" %}
8225 ins_encode %{
8226 __ cache_wbsync(false);
8227 %}
8228 ins_pipe(pipe_slow); // XXX
8229 %}
8230
8231 // ============================================================================
8232 // BSWAP Instructions
8233
8234 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
8235 match(Set dst (ReverseBytesI src));
8236
8237 ins_cost(INSN_COST);
8238 format %{ "revw $dst, $src" %}
8239
8240 ins_encode %{
8241 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
8242 %}
8243
8244 ins_pipe(ialu_reg);
8245 %}
8246
8247 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
8248 match(Set dst (ReverseBytesL src));
8249
8250 ins_cost(INSN_COST);
8251 format %{ "rev $dst, $src" %}
8252
8253 ins_encode %{
8254 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
8255 %}
8256
8257 ins_pipe(ialu_reg);
8258 %}
8259
8260 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
8261 match(Set dst (ReverseBytesUS src));
8262
8263 ins_cost(INSN_COST);
8264 format %{ "rev16w $dst, $src" %}
8265
8266 ins_encode %{
8267 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8268 %}
8269
8270 ins_pipe(ialu_reg);
8271 %}
8272
8273 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
8274 match(Set dst (ReverseBytesS src));
8275
8276 ins_cost(INSN_COST);
8277 format %{ "rev16w $dst, $src\n\t"
8278 "sbfmw $dst, $dst, #0, #15" %}
8279
8280 ins_encode %{
8281 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8282 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
8283 %}
8284
8285 ins_pipe(ialu_reg);
8286 %}
8287
8288 // ============================================================================
8289 // Zero Count Instructions
8290
8291 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8292 match(Set dst (CountLeadingZerosI src));
8293
8294 ins_cost(INSN_COST);
8295 format %{ "clzw $dst, $src" %}
8296 ins_encode %{
8297 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
8298 %}
8299
8300 ins_pipe(ialu_reg);
8301 %}
8302
8303 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
8304 match(Set dst (CountLeadingZerosL src));
8305
8306 ins_cost(INSN_COST);
8307 format %{ "clz $dst, $src" %}
8308 ins_encode %{
8309 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
8310 %}
8311
8312 ins_pipe(ialu_reg);
8313 %}
8314
8315 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8316 match(Set dst (CountTrailingZerosI src));
8317
8318 ins_cost(INSN_COST * 2);
8319 format %{ "rbitw $dst, $src\n\t"
8320 "clzw $dst, $dst" %}
8321 ins_encode %{
8322 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
8323 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
8324 %}
8325
8326 ins_pipe(ialu_reg);
8327 %}
8328
8329 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
8330 match(Set dst (CountTrailingZerosL src));
8331
8332 ins_cost(INSN_COST * 2);
8333 format %{ "rbit $dst, $src\n\t"
8334 "clz $dst, $dst" %}
8335 ins_encode %{
8336 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
8337 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
8338 %}
8339
8340 ins_pipe(ialu_reg);
8341 %}
8342
8343 //---------- Population Count Instructions -------------------------------------
8344 //
8345
8346 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
8347 match(Set dst (PopCountI src));
8348 effect(TEMP tmp);
8349 ins_cost(INSN_COST * 13);
8350
8351 format %{ "movw $src, $src\n\t"
8352 "mov $tmp, $src\t# vector (1D)\n\t"
8353 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8354 "addv $tmp, $tmp\t# vector (8B)\n\t"
8355 "mov $dst, $tmp\t# vector (1D)" %}
8356 ins_encode %{
8357 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
8358 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8359 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8360 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8361 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8362 %}
8363
8364 ins_pipe(pipe_class_default);
8365 %}
8366
8367 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
8368 match(Set dst (PopCountI (LoadI mem)));
8369 effect(TEMP tmp);
8370 ins_cost(INSN_COST * 13);
8371
8372 format %{ "ldrs $tmp, $mem\n\t"
8373 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8374 "addv $tmp, $tmp\t# vector (8B)\n\t"
8375 "mov $dst, $tmp\t# vector (1D)" %}
8376 ins_encode %{
8377 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8378 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
8379 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
8380 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8381 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8382 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8383 %}
8384
8385 ins_pipe(pipe_class_default);
8386 %}
8387
8388 // Note: Long.bitCount(long) returns an int.
8389 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
8390 match(Set dst (PopCountL src));
8391 effect(TEMP tmp);
8392 ins_cost(INSN_COST * 13);
8393
8394 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
8395 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8396 "addv $tmp, $tmp\t# vector (8B)\n\t"
8397 "mov $dst, $tmp\t# vector (1D)" %}
8398 ins_encode %{
8399 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8400 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8401 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8402 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8403 %}
8404
8405 ins_pipe(pipe_class_default);
8406 %}
8407
8408 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
8409 match(Set dst (PopCountL (LoadL mem)));
8410 effect(TEMP tmp);
8411 ins_cost(INSN_COST * 13);
8412
8413 format %{ "ldrd $tmp, $mem\n\t"
8414 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8415 "addv $tmp, $tmp\t# vector (8B)\n\t"
8416 "mov $dst, $tmp\t# vector (1D)" %}
8417 ins_encode %{
8418 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8419 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8420 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
8421 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8422 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8423 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8424 %}
8425
8426 ins_pipe(pipe_class_default);
8427 %}
8428
8429 // ============================================================================
8430 // MemBar Instruction
8431
8432 instruct load_fence() %{
8433 match(LoadFence);
8434 ins_cost(VOLATILE_REF_COST);
8435
8436 format %{ "load_fence" %}
8437
8438 ins_encode %{
8439 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8440 %}
8441 ins_pipe(pipe_serial);
8442 %}
8443
8444 instruct unnecessary_membar_acquire() %{
8445 predicate(unnecessary_acquire(n));
8446 match(MemBarAcquire);
8447 ins_cost(0);
8448
8449 format %{ "membar_acquire (elided)" %}
8450
8451 ins_encode %{
8452 __ block_comment("membar_acquire (elided)");
8453 %}
8454
8455 ins_pipe(pipe_class_empty);
8456 %}
8457
8458 instruct membar_acquire() %{
8459 match(MemBarAcquire);
8460 ins_cost(VOLATILE_REF_COST);
8461
8462 format %{ "membar_acquire\n\t"
8463 "dmb ish" %}
8464
8465 ins_encode %{
8466 __ block_comment("membar_acquire");
8467 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8468 %}
8469
8470 ins_pipe(pipe_serial);
8471 %}
8472
8473
8474 instruct membar_acquire_lock() %{
8475 match(MemBarAcquireLock);
8476 ins_cost(VOLATILE_REF_COST);
8477
8478 format %{ "membar_acquire_lock (elided)" %}
8479
8480 ins_encode %{
8481 __ block_comment("membar_acquire_lock (elided)");
8482 %}
8483
8484 ins_pipe(pipe_serial);
8485 %}
8486
8487 instruct store_fence() %{
8488 match(StoreFence);
8489 ins_cost(VOLATILE_REF_COST);
8490
8491 format %{ "store_fence" %}
8492
8493 ins_encode %{
8494 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8495 %}
8496 ins_pipe(pipe_serial);
8497 %}
8498
8499 instruct unnecessary_membar_release() %{
8500 predicate(unnecessary_release(n));
8501 match(MemBarRelease);
8502 ins_cost(0);
8503
8504 format %{ "membar_release (elided)" %}
8505
8506 ins_encode %{
8507 __ block_comment("membar_release (elided)");
8508 %}
8509 ins_pipe(pipe_serial);
8510 %}
8511
8512 instruct membar_release() %{
8513 match(MemBarRelease);
8514 ins_cost(VOLATILE_REF_COST);
8515
8516 format %{ "membar_release\n\t"
8517 "dmb ish" %}
8518
8519 ins_encode %{
8520 __ block_comment("membar_release");
8521 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8522 %}
8523 ins_pipe(pipe_serial);
8524 %}
8525
8526 instruct membar_storestore() %{
8527 match(MemBarStoreStore);
8528 match(StoreStoreFence);
8529 ins_cost(VOLATILE_REF_COST);
8530
8531 format %{ "MEMBAR-store-store" %}
8532
8533 ins_encode %{
8534 __ membar(Assembler::StoreStore);
8535 %}
8536 ins_pipe(pipe_serial);
8537 %}
8538
8539 instruct membar_release_lock() %{
8540 match(MemBarReleaseLock);
8541 ins_cost(VOLATILE_REF_COST);
8542
8543 format %{ "membar_release_lock (elided)" %}
8544
8545 ins_encode %{
8546 __ block_comment("membar_release_lock (elided)");
8547 %}
8548
8549 ins_pipe(pipe_serial);
8550 %}
8551
8552 instruct unnecessary_membar_volatile() %{
8553 predicate(unnecessary_volatile(n));
8554 match(MemBarVolatile);
8555 ins_cost(0);
8556
8557 format %{ "membar_volatile (elided)" %}
8558
8559 ins_encode %{
8560 __ block_comment("membar_volatile (elided)");
8561 %}
8562
8563 ins_pipe(pipe_serial);
8564 %}
8565
8566 instruct membar_volatile() %{
8567 match(MemBarVolatile);
8568 ins_cost(VOLATILE_REF_COST*100);
8569
8570 format %{ "membar_volatile\n\t"
8571 "dmb ish"%}
8572
8573 ins_encode %{
8574 __ block_comment("membar_volatile");
8575 __ membar(Assembler::StoreLoad);
8576 %}
8577
8578 ins_pipe(pipe_serial);
8579 %}
8580
8581 // ============================================================================
8582 // Cast/Convert Instructions
8583
8584 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8585 match(Set dst (CastX2P src));
8586
8587 ins_cost(INSN_COST);
8588 format %{ "mov $dst, $src\t# long -> ptr" %}
8589
8590 ins_encode %{
8591 if ($dst$$reg != $src$$reg) {
8592 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8593 }
8594 %}
8595
8596 ins_pipe(ialu_reg);
8597 %}
8598
8599 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8600 match(Set dst (CastP2X src));
8601
8602 ins_cost(INSN_COST);
8603 format %{ "mov $dst, $src\t# ptr -> long" %}
8604
8605 ins_encode %{
8606 if ($dst$$reg != $src$$reg) {
8607 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8608 }
8609 %}
8610
8611 ins_pipe(ialu_reg);
8612 %}
8613
8614 // Convert oop into int for vectors alignment masking
8615 instruct convP2I(iRegINoSp dst, iRegP src) %{
8616 match(Set dst (ConvL2I (CastP2X src)));
8617
8618 ins_cost(INSN_COST);
8619 format %{ "movw $dst, $src\t# ptr -> int" %}
8620 ins_encode %{
8621 __ movw($dst$$Register, $src$$Register);
8622 %}
8623
8624 ins_pipe(ialu_reg);
8625 %}
8626
8627 // Convert compressed oop into int for vectors alignment masking
8628 // in case of 32bit oops (heap < 4Gb).
8629 instruct convN2I(iRegINoSp dst, iRegN src)
8630 %{
8631 predicate(CompressedOops::shift() == 0);
8632 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8633
8634 ins_cost(INSN_COST);
8635 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8636 ins_encode %{
8637 __ movw($dst$$Register, $src$$Register);
8638 %}
8639
8640 ins_pipe(ialu_reg);
8641 %}
8642
8643
8644 // Convert oop pointer into compressed form
8645 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8646 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8647 match(Set dst (EncodeP src));
8648 effect(KILL cr);
8649 ins_cost(INSN_COST * 3);
8650 format %{ "encode_heap_oop $dst, $src" %}
8651 ins_encode %{
8652 Register s = $src$$Register;
8653 Register d = $dst$$Register;
8654 __ encode_heap_oop(d, s);
8655 %}
8656 ins_pipe(ialu_reg);
8657 %}
8658
8659 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8660 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8661 match(Set dst (EncodeP src));
8662 ins_cost(INSN_COST * 3);
8663 format %{ "encode_heap_oop_not_null $dst, $src" %}
8664 ins_encode %{
8665 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8666 %}
8667 ins_pipe(ialu_reg);
8668 %}
8669
8670 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8671 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8672 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8673 match(Set dst (DecodeN src));
8674 ins_cost(INSN_COST * 3);
8675 format %{ "decode_heap_oop $dst, $src" %}
8676 ins_encode %{
8677 Register s = $src$$Register;
8678 Register d = $dst$$Register;
8679 __ decode_heap_oop(d, s);
8680 %}
8681 ins_pipe(ialu_reg);
8682 %}
8683
8684 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8685 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8686 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8687 match(Set dst (DecodeN src));
8688 ins_cost(INSN_COST * 3);
8689 format %{ "decode_heap_oop_not_null $dst, $src" %}
8690 ins_encode %{
8691 Register s = $src$$Register;
8692 Register d = $dst$$Register;
8693 __ decode_heap_oop_not_null(d, s);
8694 %}
8695 ins_pipe(ialu_reg);
8696 %}
8697
8698 // n.b. AArch64 implementations of encode_klass_not_null and
8699 // decode_klass_not_null do not modify the flags register so, unlike
8700 // Intel, we don't kill CR as a side effect here
8701
8702 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8703 match(Set dst (EncodePKlass src));
8704
8705 ins_cost(INSN_COST * 3);
8706 format %{ "encode_klass_not_null $dst,$src" %}
8707
8708 ins_encode %{
8709 Register src_reg = as_Register($src$$reg);
8710 Register dst_reg = as_Register($dst$$reg);
8711 __ encode_klass_not_null(dst_reg, src_reg);
8712 %}
8713
8714 ins_pipe(ialu_reg);
8715 %}
8716
8717 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8718 match(Set dst (DecodeNKlass src));
8719
8720 ins_cost(INSN_COST * 3);
8721 format %{ "decode_klass_not_null $dst,$src" %}
8722
8723 ins_encode %{
8724 Register src_reg = as_Register($src$$reg);
8725 Register dst_reg = as_Register($dst$$reg);
8726 if (dst_reg != src_reg) {
8727 __ decode_klass_not_null(dst_reg, src_reg);
8728 } else {
8729 __ decode_klass_not_null(dst_reg);
8730 }
8731 %}
8732
8733 ins_pipe(ialu_reg);
8734 %}
8735
8736 instruct checkCastPP(iRegPNoSp dst)
8737 %{
8738 match(Set dst (CheckCastPP dst));
8739
8740 size(0);
8741 format %{ "# checkcastPP of $dst" %}
8742 ins_encode(/* empty encoding */);
8743 ins_pipe(pipe_class_empty);
8744 %}
8745
8746 instruct castPP(iRegPNoSp dst)
8747 %{
8748 match(Set dst (CastPP dst));
8749
8750 size(0);
8751 format %{ "# castPP of $dst" %}
8752 ins_encode(/* empty encoding */);
8753 ins_pipe(pipe_class_empty);
8754 %}
8755
8756 instruct castII(iRegI dst)
8757 %{
8758 match(Set dst (CastII dst));
8759
8760 size(0);
8761 format %{ "# castII of $dst" %}
8762 ins_encode(/* empty encoding */);
8763 ins_cost(0);
8764 ins_pipe(pipe_class_empty);
8765 %}
8766
8767 instruct castLL(iRegL dst)
8768 %{
8769 match(Set dst (CastLL dst));
8770
8771 size(0);
8772 format %{ "# castLL of $dst" %}
8773 ins_encode(/* empty encoding */);
8774 ins_cost(0);
8775 ins_pipe(pipe_class_empty);
8776 %}
8777
8778 instruct castFF(vRegF dst)
8779 %{
8780 match(Set dst (CastFF dst));
8781
8782 size(0);
8783 format %{ "# castFF of $dst" %}
8784 ins_encode(/* empty encoding */);
8785 ins_cost(0);
8786 ins_pipe(pipe_class_empty);
8787 %}
8788
8789 instruct castDD(vRegD dst)
8790 %{
8791 match(Set dst (CastDD dst));
8792
8793 size(0);
8794 format %{ "# castDD of $dst" %}
8795 ins_encode(/* empty encoding */);
8796 ins_cost(0);
8797 ins_pipe(pipe_class_empty);
8798 %}
8799
8800 instruct castVV(vReg dst)
8801 %{
8802 match(Set dst (CastVV dst));
8803
8804 size(0);
8805 format %{ "# castVV of $dst" %}
8806 ins_encode(/* empty encoding */);
8807 ins_cost(0);
8808 ins_pipe(pipe_class_empty);
8809 %}
8810
8811 instruct castVVMask(pRegGov dst)
8812 %{
8813 match(Set dst (CastVV dst));
8814
8815 size(0);
8816 format %{ "# castVV of $dst" %}
8817 ins_encode(/* empty encoding */);
8818 ins_cost(0);
8819 ins_pipe(pipe_class_empty);
8820 %}
8821
8822 // ============================================================================
8823 // Atomic operation instructions
8824 //
8825
8826 // standard CompareAndSwapX when we are using barriers
8827 // these have higher priority than the rules selected by a predicate
8828
8829 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8830 // can't match them
8831
8832 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8833
8834 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8835 ins_cost(2 * VOLATILE_REF_COST);
8836
8837 effect(KILL cr);
8838
8839 format %{
8840 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8841 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8842 %}
8843
8844 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8845 aarch64_enc_cset_eq(res));
8846
8847 ins_pipe(pipe_slow);
8848 %}
8849
8850 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8851
8852 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8853 ins_cost(2 * VOLATILE_REF_COST);
8854
8855 effect(KILL cr);
8856
8857 format %{
8858 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8859 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8860 %}
8861
8862 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8863 aarch64_enc_cset_eq(res));
8864
8865 ins_pipe(pipe_slow);
8866 %}
8867
8868 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8869
8870 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8871 ins_cost(2 * VOLATILE_REF_COST);
8872
8873 effect(KILL cr);
8874
8875 format %{
8876 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8877 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8878 %}
8879
8880 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8881 aarch64_enc_cset_eq(res));
8882
8883 ins_pipe(pipe_slow);
8884 %}
8885
8886 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8887
8888 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8889 ins_cost(2 * VOLATILE_REF_COST);
8890
8891 effect(KILL cr);
8892
8893 format %{
8894 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8895 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8896 %}
8897
8898 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8899 aarch64_enc_cset_eq(res));
8900
8901 ins_pipe(pipe_slow);
8902 %}
8903
8904 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8905
8906 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8907 predicate(n->as_LoadStore()->barrier_data() == 0);
8908 ins_cost(2 * VOLATILE_REF_COST);
8909
8910 effect(KILL cr);
8911
8912 format %{
8913 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8914 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8915 %}
8916
8917 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8918 aarch64_enc_cset_eq(res));
8919
8920 ins_pipe(pipe_slow);
8921 %}
8922
8923 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8924
8925 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8926 ins_cost(2 * VOLATILE_REF_COST);
8927
8928 effect(KILL cr);
8929
8930 format %{
8931 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8932 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8933 %}
8934
8935 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8936 aarch64_enc_cset_eq(res));
8937
8938 ins_pipe(pipe_slow);
8939 %}
8940
8941 // alternative CompareAndSwapX when we are eliding barriers
8942
8943 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8944
8945 predicate(needs_acquiring_load_exclusive(n));
8946 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8947 ins_cost(VOLATILE_REF_COST);
8948
8949 effect(KILL cr);
8950
8951 format %{
8952 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8953 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8954 %}
8955
8956 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8957 aarch64_enc_cset_eq(res));
8958
8959 ins_pipe(pipe_slow);
8960 %}
8961
8962 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8963
8964 predicate(needs_acquiring_load_exclusive(n));
8965 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8966 ins_cost(VOLATILE_REF_COST);
8967
8968 effect(KILL cr);
8969
8970 format %{
8971 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8972 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8973 %}
8974
8975 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8976 aarch64_enc_cset_eq(res));
8977
8978 ins_pipe(pipe_slow);
8979 %}
8980
8981 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8982
8983 predicate(needs_acquiring_load_exclusive(n));
8984 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8985 ins_cost(VOLATILE_REF_COST);
8986
8987 effect(KILL cr);
8988
8989 format %{
8990 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8991 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8992 %}
8993
8994 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8995 aarch64_enc_cset_eq(res));
8996
8997 ins_pipe(pipe_slow);
8998 %}
8999
9000 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9001
9002 predicate(needs_acquiring_load_exclusive(n));
9003 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9004 ins_cost(VOLATILE_REF_COST);
9005
9006 effect(KILL cr);
9007
9008 format %{
9009 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9010 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9011 %}
9012
9013 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9014 aarch64_enc_cset_eq(res));
9015
9016 ins_pipe(pipe_slow);
9017 %}
9018
9019 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9020
9021 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9022 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9023 ins_cost(VOLATILE_REF_COST);
9024
9025 effect(KILL cr);
9026
9027 format %{
9028 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9029 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9030 %}
9031
9032 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9033 aarch64_enc_cset_eq(res));
9034
9035 ins_pipe(pipe_slow);
9036 %}
9037
9038 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9039
9040 predicate(needs_acquiring_load_exclusive(n));
9041 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9042 ins_cost(VOLATILE_REF_COST);
9043
9044 effect(KILL cr);
9045
9046 format %{
9047 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9048 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9049 %}
9050
9051 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9052 aarch64_enc_cset_eq(res));
9053
9054 ins_pipe(pipe_slow);
9055 %}
9056
9057
9058 // ---------------------------------------------------------------------
9059
9060 // BEGIN This section of the file is automatically generated. Do not edit --------------
9061
9062 // Sundry CAS operations. Note that release is always true,
9063 // regardless of the memory ordering of the CAS. This is because we
9064 // need the volatile case to be sequentially consistent but there is
9065 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
9066 // can't check the type of memory ordering here, so we always emit a
9067 // STLXR.
9068
9069 // This section is generated from cas.m4
9070
9071
9072 // This pattern is generated automatically from cas.m4.
9073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9074 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9075 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9076 ins_cost(2 * VOLATILE_REF_COST);
9077 effect(TEMP_DEF res, KILL cr);
9078 format %{
9079 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9080 %}
9081 ins_encode %{
9082 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9083 Assembler::byte, /*acquire*/ false, /*release*/ true,
9084 /*weak*/ false, $res$$Register);
9085 __ sxtbw($res$$Register, $res$$Register);
9086 %}
9087 ins_pipe(pipe_slow);
9088 %}
9089
9090 // This pattern is generated automatically from cas.m4.
9091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9092 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9093 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9094 ins_cost(2 * VOLATILE_REF_COST);
9095 effect(TEMP_DEF res, KILL cr);
9096 format %{
9097 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9098 %}
9099 ins_encode %{
9100 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9101 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9102 /*weak*/ false, $res$$Register);
9103 __ sxthw($res$$Register, $res$$Register);
9104 %}
9105 ins_pipe(pipe_slow);
9106 %}
9107
9108 // This pattern is generated automatically from cas.m4.
9109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9110 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9111 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9112 ins_cost(2 * VOLATILE_REF_COST);
9113 effect(TEMP_DEF res, KILL cr);
9114 format %{
9115 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9116 %}
9117 ins_encode %{
9118 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9119 Assembler::word, /*acquire*/ false, /*release*/ true,
9120 /*weak*/ false, $res$$Register);
9121 %}
9122 ins_pipe(pipe_slow);
9123 %}
9124
9125 // This pattern is generated automatically from cas.m4.
9126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9127 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9128 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9129 ins_cost(2 * VOLATILE_REF_COST);
9130 effect(TEMP_DEF res, KILL cr);
9131 format %{
9132 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9133 %}
9134 ins_encode %{
9135 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9136 Assembler::xword, /*acquire*/ false, /*release*/ true,
9137 /*weak*/ false, $res$$Register);
9138 %}
9139 ins_pipe(pipe_slow);
9140 %}
9141
9142 // This pattern is generated automatically from cas.m4.
9143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9144 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9145 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9146 ins_cost(2 * VOLATILE_REF_COST);
9147 effect(TEMP_DEF res, KILL cr);
9148 format %{
9149 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9150 %}
9151 ins_encode %{
9152 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9153 Assembler::word, /*acquire*/ false, /*release*/ true,
9154 /*weak*/ false, $res$$Register);
9155 %}
9156 ins_pipe(pipe_slow);
9157 %}
9158
9159 // This pattern is generated automatically from cas.m4.
9160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9161 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9162 predicate(n->as_LoadStore()->barrier_data() == 0);
9163 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9164 ins_cost(2 * VOLATILE_REF_COST);
9165 effect(TEMP_DEF res, KILL cr);
9166 format %{
9167 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9168 %}
9169 ins_encode %{
9170 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9171 Assembler::xword, /*acquire*/ false, /*release*/ true,
9172 /*weak*/ false, $res$$Register);
9173 %}
9174 ins_pipe(pipe_slow);
9175 %}
9176
9177 // This pattern is generated automatically from cas.m4.
9178 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9179 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9180 predicate(needs_acquiring_load_exclusive(n));
9181 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9182 ins_cost(VOLATILE_REF_COST);
9183 effect(TEMP_DEF res, KILL cr);
9184 format %{
9185 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9186 %}
9187 ins_encode %{
9188 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9189 Assembler::byte, /*acquire*/ true, /*release*/ true,
9190 /*weak*/ false, $res$$Register);
9191 __ sxtbw($res$$Register, $res$$Register);
9192 %}
9193 ins_pipe(pipe_slow);
9194 %}
9195
9196 // This pattern is generated automatically from cas.m4.
9197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9198 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9199 predicate(needs_acquiring_load_exclusive(n));
9200 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9201 ins_cost(VOLATILE_REF_COST);
9202 effect(TEMP_DEF res, KILL cr);
9203 format %{
9204 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9205 %}
9206 ins_encode %{
9207 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9208 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9209 /*weak*/ false, $res$$Register);
9210 __ sxthw($res$$Register, $res$$Register);
9211 %}
9212 ins_pipe(pipe_slow);
9213 %}
9214
9215 // This pattern is generated automatically from cas.m4.
9216 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9217 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9218 predicate(needs_acquiring_load_exclusive(n));
9219 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9220 ins_cost(VOLATILE_REF_COST);
9221 effect(TEMP_DEF res, KILL cr);
9222 format %{
9223 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9224 %}
9225 ins_encode %{
9226 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9227 Assembler::word, /*acquire*/ true, /*release*/ true,
9228 /*weak*/ false, $res$$Register);
9229 %}
9230 ins_pipe(pipe_slow);
9231 %}
9232
9233 // This pattern is generated automatically from cas.m4.
9234 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9235 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9236 predicate(needs_acquiring_load_exclusive(n));
9237 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9238 ins_cost(VOLATILE_REF_COST);
9239 effect(TEMP_DEF res, KILL cr);
9240 format %{
9241 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9242 %}
9243 ins_encode %{
9244 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9245 Assembler::xword, /*acquire*/ true, /*release*/ true,
9246 /*weak*/ false, $res$$Register);
9247 %}
9248 ins_pipe(pipe_slow);
9249 %}
9250
9251 // This pattern is generated automatically from cas.m4.
9252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9253 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9254 predicate(needs_acquiring_load_exclusive(n));
9255 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9256 ins_cost(VOLATILE_REF_COST);
9257 effect(TEMP_DEF res, KILL cr);
9258 format %{
9259 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9260 %}
9261 ins_encode %{
9262 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9263 Assembler::word, /*acquire*/ true, /*release*/ true,
9264 /*weak*/ false, $res$$Register);
9265 %}
9266 ins_pipe(pipe_slow);
9267 %}
9268
9269 // This pattern is generated automatically from cas.m4.
9270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9271 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9272 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9273 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9274 ins_cost(VOLATILE_REF_COST);
9275 effect(TEMP_DEF res, KILL cr);
9276 format %{
9277 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9278 %}
9279 ins_encode %{
9280 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9281 Assembler::xword, /*acquire*/ true, /*release*/ true,
9282 /*weak*/ false, $res$$Register);
9283 %}
9284 ins_pipe(pipe_slow);
9285 %}
9286
9287 // This pattern is generated automatically from cas.m4.
9288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9289 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9290 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9291 ins_cost(2 * VOLATILE_REF_COST);
9292 effect(KILL cr);
9293 format %{
9294 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9295 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9296 %}
9297 ins_encode %{
9298 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9299 Assembler::byte, /*acquire*/ false, /*release*/ true,
9300 /*weak*/ true, noreg);
9301 __ csetw($res$$Register, Assembler::EQ);
9302 %}
9303 ins_pipe(pipe_slow);
9304 %}
9305
9306 // This pattern is generated automatically from cas.m4.
9307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9308 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9309 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9310 ins_cost(2 * VOLATILE_REF_COST);
9311 effect(KILL cr);
9312 format %{
9313 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9314 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9315 %}
9316 ins_encode %{
9317 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9318 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9319 /*weak*/ true, noreg);
9320 __ csetw($res$$Register, Assembler::EQ);
9321 %}
9322 ins_pipe(pipe_slow);
9323 %}
9324
9325 // This pattern is generated automatically from cas.m4.
9326 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9327 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9328 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9329 ins_cost(2 * VOLATILE_REF_COST);
9330 effect(KILL cr);
9331 format %{
9332 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9333 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9334 %}
9335 ins_encode %{
9336 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9337 Assembler::word, /*acquire*/ false, /*release*/ true,
9338 /*weak*/ true, noreg);
9339 __ csetw($res$$Register, Assembler::EQ);
9340 %}
9341 ins_pipe(pipe_slow);
9342 %}
9343
9344 // This pattern is generated automatically from cas.m4.
9345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9346 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9347 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9348 ins_cost(2 * VOLATILE_REF_COST);
9349 effect(KILL cr);
9350 format %{
9351 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9352 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9353 %}
9354 ins_encode %{
9355 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9356 Assembler::xword, /*acquire*/ false, /*release*/ true,
9357 /*weak*/ true, noreg);
9358 __ csetw($res$$Register, Assembler::EQ);
9359 %}
9360 ins_pipe(pipe_slow);
9361 %}
9362
9363 // This pattern is generated automatically from cas.m4.
9364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9365 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9366 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9367 ins_cost(2 * VOLATILE_REF_COST);
9368 effect(KILL cr);
9369 format %{
9370 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9371 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9372 %}
9373 ins_encode %{
9374 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9375 Assembler::word, /*acquire*/ false, /*release*/ true,
9376 /*weak*/ true, noreg);
9377 __ csetw($res$$Register, Assembler::EQ);
9378 %}
9379 ins_pipe(pipe_slow);
9380 %}
9381
9382 // This pattern is generated automatically from cas.m4.
9383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9384 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9385 predicate(n->as_LoadStore()->barrier_data() == 0);
9386 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9387 ins_cost(2 * VOLATILE_REF_COST);
9388 effect(KILL cr);
9389 format %{
9390 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9391 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9392 %}
9393 ins_encode %{
9394 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9395 Assembler::xword, /*acquire*/ false, /*release*/ true,
9396 /*weak*/ true, noreg);
9397 __ csetw($res$$Register, Assembler::EQ);
9398 %}
9399 ins_pipe(pipe_slow);
9400 %}
9401
9402 // This pattern is generated automatically from cas.m4.
9403 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9404 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9405 predicate(needs_acquiring_load_exclusive(n));
9406 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9407 ins_cost(VOLATILE_REF_COST);
9408 effect(KILL cr);
9409 format %{
9410 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9411 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9412 %}
9413 ins_encode %{
9414 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9415 Assembler::byte, /*acquire*/ true, /*release*/ true,
9416 /*weak*/ true, noreg);
9417 __ csetw($res$$Register, Assembler::EQ);
9418 %}
9419 ins_pipe(pipe_slow);
9420 %}
9421
9422 // This pattern is generated automatically from cas.m4.
9423 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9424 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9425 predicate(needs_acquiring_load_exclusive(n));
9426 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9427 ins_cost(VOLATILE_REF_COST);
9428 effect(KILL cr);
9429 format %{
9430 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9431 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9432 %}
9433 ins_encode %{
9434 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9435 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9436 /*weak*/ true, noreg);
9437 __ csetw($res$$Register, Assembler::EQ);
9438 %}
9439 ins_pipe(pipe_slow);
9440 %}
9441
9442 // This pattern is generated automatically from cas.m4.
9443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9444 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9445 predicate(needs_acquiring_load_exclusive(n));
9446 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9447 ins_cost(VOLATILE_REF_COST);
9448 effect(KILL cr);
9449 format %{
9450 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9451 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9452 %}
9453 ins_encode %{
9454 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9455 Assembler::word, /*acquire*/ true, /*release*/ true,
9456 /*weak*/ true, noreg);
9457 __ csetw($res$$Register, Assembler::EQ);
9458 %}
9459 ins_pipe(pipe_slow);
9460 %}
9461
9462 // This pattern is generated automatically from cas.m4.
9463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9464 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9465 predicate(needs_acquiring_load_exclusive(n));
9466 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9467 ins_cost(VOLATILE_REF_COST);
9468 effect(KILL cr);
9469 format %{
9470 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9471 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9472 %}
9473 ins_encode %{
9474 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9475 Assembler::xword, /*acquire*/ true, /*release*/ true,
9476 /*weak*/ true, noreg);
9477 __ csetw($res$$Register, Assembler::EQ);
9478 %}
9479 ins_pipe(pipe_slow);
9480 %}
9481
9482 // This pattern is generated automatically from cas.m4.
9483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9484 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9485 predicate(needs_acquiring_load_exclusive(n));
9486 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9487 ins_cost(VOLATILE_REF_COST);
9488 effect(KILL cr);
9489 format %{
9490 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9491 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9492 %}
9493 ins_encode %{
9494 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9495 Assembler::word, /*acquire*/ true, /*release*/ true,
9496 /*weak*/ true, noreg);
9497 __ csetw($res$$Register, Assembler::EQ);
9498 %}
9499 ins_pipe(pipe_slow);
9500 %}
9501
9502 // This pattern is generated automatically from cas.m4.
9503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9504 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9505 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9506 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9507 ins_cost(VOLATILE_REF_COST);
9508 effect(KILL cr);
9509 format %{
9510 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9511 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9512 %}
9513 ins_encode %{
9514 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9515 Assembler::xword, /*acquire*/ true, /*release*/ true,
9516 /*weak*/ true, noreg);
9517 __ csetw($res$$Register, Assembler::EQ);
9518 %}
9519 ins_pipe(pipe_slow);
9520 %}
9521
9522 // END This section of the file is automatically generated. Do not edit --------------
9523 // ---------------------------------------------------------------------
9524
9525 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9526 match(Set prev (GetAndSetI mem newv));
9527 ins_cost(2 * VOLATILE_REF_COST);
9528 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9529 ins_encode %{
9530 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9531 %}
9532 ins_pipe(pipe_serial);
9533 %}
9534
9535 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9536 match(Set prev (GetAndSetL mem newv));
9537 ins_cost(2 * VOLATILE_REF_COST);
9538 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9539 ins_encode %{
9540 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9541 %}
9542 ins_pipe(pipe_serial);
9543 %}
9544
9545 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9546 match(Set prev (GetAndSetN mem newv));
9547 ins_cost(2 * VOLATILE_REF_COST);
9548 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9549 ins_encode %{
9550 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9551 %}
9552 ins_pipe(pipe_serial);
9553 %}
9554
9555 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9556 predicate(n->as_LoadStore()->barrier_data() == 0);
9557 match(Set prev (GetAndSetP mem newv));
9558 ins_cost(2 * VOLATILE_REF_COST);
9559 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9560 ins_encode %{
9561 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9562 %}
9563 ins_pipe(pipe_serial);
9564 %}
9565
9566 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9567 predicate(needs_acquiring_load_exclusive(n));
9568 match(Set prev (GetAndSetI mem newv));
9569 ins_cost(VOLATILE_REF_COST);
9570 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9571 ins_encode %{
9572 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9573 %}
9574 ins_pipe(pipe_serial);
9575 %}
9576
9577 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9578 predicate(needs_acquiring_load_exclusive(n));
9579 match(Set prev (GetAndSetL mem newv));
9580 ins_cost(VOLATILE_REF_COST);
9581 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9582 ins_encode %{
9583 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9584 %}
9585 ins_pipe(pipe_serial);
9586 %}
9587
9588 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9589 predicate(needs_acquiring_load_exclusive(n));
9590 match(Set prev (GetAndSetN mem newv));
9591 ins_cost(VOLATILE_REF_COST);
9592 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9593 ins_encode %{
9594 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9595 %}
9596 ins_pipe(pipe_serial);
9597 %}
9598
9599 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9600 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9601 match(Set prev (GetAndSetP mem newv));
9602 ins_cost(VOLATILE_REF_COST);
9603 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9604 ins_encode %{
9605 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9606 %}
9607 ins_pipe(pipe_serial);
9608 %}
9609
9610
9611 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9612 match(Set newval (GetAndAddL mem incr));
9613 ins_cost(2 * VOLATILE_REF_COST + 1);
9614 format %{ "get_and_addL $newval, [$mem], $incr" %}
9615 ins_encode %{
9616 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9617 %}
9618 ins_pipe(pipe_serial);
9619 %}
9620
9621 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9622 predicate(n->as_LoadStore()->result_not_used());
9623 match(Set dummy (GetAndAddL mem incr));
9624 ins_cost(2 * VOLATILE_REF_COST);
9625 format %{ "get_and_addL [$mem], $incr" %}
9626 ins_encode %{
9627 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9628 %}
9629 ins_pipe(pipe_serial);
9630 %}
9631
9632 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9633 match(Set newval (GetAndAddL mem incr));
9634 ins_cost(2 * VOLATILE_REF_COST + 1);
9635 format %{ "get_and_addL $newval, [$mem], $incr" %}
9636 ins_encode %{
9637 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9638 %}
9639 ins_pipe(pipe_serial);
9640 %}
9641
9642 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9643 predicate(n->as_LoadStore()->result_not_used());
9644 match(Set dummy (GetAndAddL mem incr));
9645 ins_cost(2 * VOLATILE_REF_COST);
9646 format %{ "get_and_addL [$mem], $incr" %}
9647 ins_encode %{
9648 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9649 %}
9650 ins_pipe(pipe_serial);
9651 %}
9652
9653 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9654 match(Set newval (GetAndAddI mem incr));
9655 ins_cost(2 * VOLATILE_REF_COST + 1);
9656 format %{ "get_and_addI $newval, [$mem], $incr" %}
9657 ins_encode %{
9658 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9659 %}
9660 ins_pipe(pipe_serial);
9661 %}
9662
9663 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9664 predicate(n->as_LoadStore()->result_not_used());
9665 match(Set dummy (GetAndAddI mem incr));
9666 ins_cost(2 * VOLATILE_REF_COST);
9667 format %{ "get_and_addI [$mem], $incr" %}
9668 ins_encode %{
9669 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9670 %}
9671 ins_pipe(pipe_serial);
9672 %}
9673
9674 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9675 match(Set newval (GetAndAddI mem incr));
9676 ins_cost(2 * VOLATILE_REF_COST + 1);
9677 format %{ "get_and_addI $newval, [$mem], $incr" %}
9678 ins_encode %{
9679 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9680 %}
9681 ins_pipe(pipe_serial);
9682 %}
9683
9684 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9685 predicate(n->as_LoadStore()->result_not_used());
9686 match(Set dummy (GetAndAddI mem incr));
9687 ins_cost(2 * VOLATILE_REF_COST);
9688 format %{ "get_and_addI [$mem], $incr" %}
9689 ins_encode %{
9690 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9691 %}
9692 ins_pipe(pipe_serial);
9693 %}
9694
9695 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9696 predicate(needs_acquiring_load_exclusive(n));
9697 match(Set newval (GetAndAddL mem incr));
9698 ins_cost(VOLATILE_REF_COST + 1);
9699 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9700 ins_encode %{
9701 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9702 %}
9703 ins_pipe(pipe_serial);
9704 %}
9705
9706 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9707 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9708 match(Set dummy (GetAndAddL mem incr));
9709 ins_cost(VOLATILE_REF_COST);
9710 format %{ "get_and_addL_acq [$mem], $incr" %}
9711 ins_encode %{
9712 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9713 %}
9714 ins_pipe(pipe_serial);
9715 %}
9716
9717 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9718 predicate(needs_acquiring_load_exclusive(n));
9719 match(Set newval (GetAndAddL mem incr));
9720 ins_cost(VOLATILE_REF_COST + 1);
9721 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9722 ins_encode %{
9723 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9724 %}
9725 ins_pipe(pipe_serial);
9726 %}
9727
9728 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9729 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9730 match(Set dummy (GetAndAddL mem incr));
9731 ins_cost(VOLATILE_REF_COST);
9732 format %{ "get_and_addL_acq [$mem], $incr" %}
9733 ins_encode %{
9734 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9735 %}
9736 ins_pipe(pipe_serial);
9737 %}
9738
9739 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9740 predicate(needs_acquiring_load_exclusive(n));
9741 match(Set newval (GetAndAddI mem incr));
9742 ins_cost(VOLATILE_REF_COST + 1);
9743 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9744 ins_encode %{
9745 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9746 %}
9747 ins_pipe(pipe_serial);
9748 %}
9749
9750 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9751 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9752 match(Set dummy (GetAndAddI mem incr));
9753 ins_cost(VOLATILE_REF_COST);
9754 format %{ "get_and_addI_acq [$mem], $incr" %}
9755 ins_encode %{
9756 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9757 %}
9758 ins_pipe(pipe_serial);
9759 %}
9760
9761 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9762 predicate(needs_acquiring_load_exclusive(n));
9763 match(Set newval (GetAndAddI mem incr));
9764 ins_cost(VOLATILE_REF_COST + 1);
9765 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9766 ins_encode %{
9767 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9768 %}
9769 ins_pipe(pipe_serial);
9770 %}
9771
9772 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9773 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9774 match(Set dummy (GetAndAddI mem incr));
9775 ins_cost(VOLATILE_REF_COST);
9776 format %{ "get_and_addI_acq [$mem], $incr" %}
9777 ins_encode %{
9778 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9779 %}
9780 ins_pipe(pipe_serial);
9781 %}
9782
9783 // Manifest a CmpU result in an integer register.
9784 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9785 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9786 %{
9787 match(Set dst (CmpU3 src1 src2));
9788 effect(KILL flags);
9789
9790 ins_cost(INSN_COST * 3);
9791 format %{
9792 "cmpw $src1, $src2\n\t"
9793 "csetw $dst, ne\n\t"
9794 "cnegw $dst, lo\t# CmpU3(reg)"
9795 %}
9796 ins_encode %{
9797 __ cmpw($src1$$Register, $src2$$Register);
9798 __ csetw($dst$$Register, Assembler::NE);
9799 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9800 %}
9801
9802 ins_pipe(pipe_class_default);
9803 %}
9804
9805 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9806 %{
9807 match(Set dst (CmpU3 src1 src2));
9808 effect(KILL flags);
9809
9810 ins_cost(INSN_COST * 3);
9811 format %{
9812 "subsw zr, $src1, $src2\n\t"
9813 "csetw $dst, ne\n\t"
9814 "cnegw $dst, lo\t# CmpU3(imm)"
9815 %}
9816 ins_encode %{
9817 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9818 __ csetw($dst$$Register, Assembler::NE);
9819 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9820 %}
9821
9822 ins_pipe(pipe_class_default);
9823 %}
9824
9825 // Manifest a CmpUL result in an integer register.
9826 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9827 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9828 %{
9829 match(Set dst (CmpUL3 src1 src2));
9830 effect(KILL flags);
9831
9832 ins_cost(INSN_COST * 3);
9833 format %{
9834 "cmp $src1, $src2\n\t"
9835 "csetw $dst, ne\n\t"
9836 "cnegw $dst, lo\t# CmpUL3(reg)"
9837 %}
9838 ins_encode %{
9839 __ cmp($src1$$Register, $src2$$Register);
9840 __ csetw($dst$$Register, Assembler::NE);
9841 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9842 %}
9843
9844 ins_pipe(pipe_class_default);
9845 %}
9846
9847 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9848 %{
9849 match(Set dst (CmpUL3 src1 src2));
9850 effect(KILL flags);
9851
9852 ins_cost(INSN_COST * 3);
9853 format %{
9854 "subs zr, $src1, $src2\n\t"
9855 "csetw $dst, ne\n\t"
9856 "cnegw $dst, lo\t# CmpUL3(imm)"
9857 %}
9858 ins_encode %{
9859 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9860 __ csetw($dst$$Register, Assembler::NE);
9861 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9862 %}
9863
9864 ins_pipe(pipe_class_default);
9865 %}
9866
9867 // Manifest a CmpL result in an integer register.
9868 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9869 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9870 %{
9871 match(Set dst (CmpL3 src1 src2));
9872 effect(KILL flags);
9873
9874 ins_cost(INSN_COST * 3);
9875 format %{
9876 "cmp $src1, $src2\n\t"
9877 "csetw $dst, ne\n\t"
9878 "cnegw $dst, lt\t# CmpL3(reg)"
9879 %}
9880 ins_encode %{
9881 __ cmp($src1$$Register, $src2$$Register);
9882 __ csetw($dst$$Register, Assembler::NE);
9883 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9884 %}
9885
9886 ins_pipe(pipe_class_default);
9887 %}
9888
9889 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9890 %{
9891 match(Set dst (CmpL3 src1 src2));
9892 effect(KILL flags);
9893
9894 ins_cost(INSN_COST * 3);
9895 format %{
9896 "subs zr, $src1, $src2\n\t"
9897 "csetw $dst, ne\n\t"
9898 "cnegw $dst, lt\t# CmpL3(imm)"
9899 %}
9900 ins_encode %{
9901 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9902 __ csetw($dst$$Register, Assembler::NE);
9903 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9904 %}
9905
9906 ins_pipe(pipe_class_default);
9907 %}
9908
9909 // ============================================================================
9910 // Conditional Move Instructions
9911
9912 // n.b. we have identical rules for both a signed compare op (cmpOp)
9913 // and an unsigned compare op (cmpOpU). it would be nice if we could
9914 // define an op class which merged both inputs and use it to type the
9915 // argument to a single rule. unfortunatelyt his fails because the
9916 // opclass does not live up to the COND_INTER interface of its
9917 // component operands. When the generic code tries to negate the
9918 // operand it ends up running the generci Machoper::negate method
9919 // which throws a ShouldNotHappen. So, we have to provide two flavours
9920 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9921
9922 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9923 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9924
9925 ins_cost(INSN_COST * 2);
9926 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9927
9928 ins_encode %{
9929 __ cselw(as_Register($dst$$reg),
9930 as_Register($src2$$reg),
9931 as_Register($src1$$reg),
9932 (Assembler::Condition)$cmp$$cmpcode);
9933 %}
9934
9935 ins_pipe(icond_reg_reg);
9936 %}
9937
9938 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9939 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9940
9941 ins_cost(INSN_COST * 2);
9942 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9943
9944 ins_encode %{
9945 __ cselw(as_Register($dst$$reg),
9946 as_Register($src2$$reg),
9947 as_Register($src1$$reg),
9948 (Assembler::Condition)$cmp$$cmpcode);
9949 %}
9950
9951 ins_pipe(icond_reg_reg);
9952 %}
9953
9954 // special cases where one arg is zero
9955
9956 // n.b. this is selected in preference to the rule above because it
9957 // avoids loading constant 0 into a source register
9958
9959 // TODO
9960 // we ought only to be able to cull one of these variants as the ideal
9961 // transforms ought always to order the zero consistently (to left/right?)
9962
9963 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9964 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9965
9966 ins_cost(INSN_COST * 2);
9967 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9968
9969 ins_encode %{
9970 __ cselw(as_Register($dst$$reg),
9971 as_Register($src$$reg),
9972 zr,
9973 (Assembler::Condition)$cmp$$cmpcode);
9974 %}
9975
9976 ins_pipe(icond_reg);
9977 %}
9978
9979 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9980 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9981
9982 ins_cost(INSN_COST * 2);
9983 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9984
9985 ins_encode %{
9986 __ cselw(as_Register($dst$$reg),
9987 as_Register($src$$reg),
9988 zr,
9989 (Assembler::Condition)$cmp$$cmpcode);
9990 %}
9991
9992 ins_pipe(icond_reg);
9993 %}
9994
9995 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9996 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9997
9998 ins_cost(INSN_COST * 2);
9999 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
10000
10001 ins_encode %{
10002 __ cselw(as_Register($dst$$reg),
10003 zr,
10004 as_Register($src$$reg),
10005 (Assembler::Condition)$cmp$$cmpcode);
10006 %}
10007
10008 ins_pipe(icond_reg);
10009 %}
10010
10011 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10012 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10013
10014 ins_cost(INSN_COST * 2);
10015 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
10016
10017 ins_encode %{
10018 __ cselw(as_Register($dst$$reg),
10019 zr,
10020 as_Register($src$$reg),
10021 (Assembler::Condition)$cmp$$cmpcode);
10022 %}
10023
10024 ins_pipe(icond_reg);
10025 %}
10026
10027 // special case for creating a boolean 0 or 1
10028
10029 // n.b. this is selected in preference to the rule above because it
10030 // avoids loading constants 0 and 1 into a source register
10031
10032 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10033 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10034
10035 ins_cost(INSN_COST * 2);
10036 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
10037
10038 ins_encode %{
10039 // equivalently
10040 // cset(as_Register($dst$$reg),
10041 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10042 __ csincw(as_Register($dst$$reg),
10043 zr,
10044 zr,
10045 (Assembler::Condition)$cmp$$cmpcode);
10046 %}
10047
10048 ins_pipe(icond_none);
10049 %}
10050
10051 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10052 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10053
10054 ins_cost(INSN_COST * 2);
10055 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
10056
10057 ins_encode %{
10058 // equivalently
10059 // cset(as_Register($dst$$reg),
10060 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10061 __ csincw(as_Register($dst$$reg),
10062 zr,
10063 zr,
10064 (Assembler::Condition)$cmp$$cmpcode);
10065 %}
10066
10067 ins_pipe(icond_none);
10068 %}
10069
10070 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10071 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10072
10073 ins_cost(INSN_COST * 2);
10074 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
10075
10076 ins_encode %{
10077 __ csel(as_Register($dst$$reg),
10078 as_Register($src2$$reg),
10079 as_Register($src1$$reg),
10080 (Assembler::Condition)$cmp$$cmpcode);
10081 %}
10082
10083 ins_pipe(icond_reg_reg);
10084 %}
10085
10086 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10087 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10088
10089 ins_cost(INSN_COST * 2);
10090 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
10091
10092 ins_encode %{
10093 __ csel(as_Register($dst$$reg),
10094 as_Register($src2$$reg),
10095 as_Register($src1$$reg),
10096 (Assembler::Condition)$cmp$$cmpcode);
10097 %}
10098
10099 ins_pipe(icond_reg_reg);
10100 %}
10101
10102 // special cases where one arg is zero
10103
10104 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10105 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10106
10107 ins_cost(INSN_COST * 2);
10108 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
10109
10110 ins_encode %{
10111 __ csel(as_Register($dst$$reg),
10112 zr,
10113 as_Register($src$$reg),
10114 (Assembler::Condition)$cmp$$cmpcode);
10115 %}
10116
10117 ins_pipe(icond_reg);
10118 %}
10119
10120 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10121 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10122
10123 ins_cost(INSN_COST * 2);
10124 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
10125
10126 ins_encode %{
10127 __ csel(as_Register($dst$$reg),
10128 zr,
10129 as_Register($src$$reg),
10130 (Assembler::Condition)$cmp$$cmpcode);
10131 %}
10132
10133 ins_pipe(icond_reg);
10134 %}
10135
10136 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10137 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10138
10139 ins_cost(INSN_COST * 2);
10140 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
10141
10142 ins_encode %{
10143 __ csel(as_Register($dst$$reg),
10144 as_Register($src$$reg),
10145 zr,
10146 (Assembler::Condition)$cmp$$cmpcode);
10147 %}
10148
10149 ins_pipe(icond_reg);
10150 %}
10151
10152 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10153 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10154
10155 ins_cost(INSN_COST * 2);
10156 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
10157
10158 ins_encode %{
10159 __ csel(as_Register($dst$$reg),
10160 as_Register($src$$reg),
10161 zr,
10162 (Assembler::Condition)$cmp$$cmpcode);
10163 %}
10164
10165 ins_pipe(icond_reg);
10166 %}
10167
10168 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10169 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10170
10171 ins_cost(INSN_COST * 2);
10172 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
10173
10174 ins_encode %{
10175 __ csel(as_Register($dst$$reg),
10176 as_Register($src2$$reg),
10177 as_Register($src1$$reg),
10178 (Assembler::Condition)$cmp$$cmpcode);
10179 %}
10180
10181 ins_pipe(icond_reg_reg);
10182 %}
10183
10184 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10185 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10186
10187 ins_cost(INSN_COST * 2);
10188 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
10189
10190 ins_encode %{
10191 __ csel(as_Register($dst$$reg),
10192 as_Register($src2$$reg),
10193 as_Register($src1$$reg),
10194 (Assembler::Condition)$cmp$$cmpcode);
10195 %}
10196
10197 ins_pipe(icond_reg_reg);
10198 %}
10199
10200 // special cases where one arg is zero
10201
10202 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10203 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10204
10205 ins_cost(INSN_COST * 2);
10206 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
10207
10208 ins_encode %{
10209 __ csel(as_Register($dst$$reg),
10210 zr,
10211 as_Register($src$$reg),
10212 (Assembler::Condition)$cmp$$cmpcode);
10213 %}
10214
10215 ins_pipe(icond_reg);
10216 %}
10217
10218 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10219 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10220
10221 ins_cost(INSN_COST * 2);
10222 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
10223
10224 ins_encode %{
10225 __ csel(as_Register($dst$$reg),
10226 zr,
10227 as_Register($src$$reg),
10228 (Assembler::Condition)$cmp$$cmpcode);
10229 %}
10230
10231 ins_pipe(icond_reg);
10232 %}
10233
10234 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10235 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10236
10237 ins_cost(INSN_COST * 2);
10238 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
10239
10240 ins_encode %{
10241 __ csel(as_Register($dst$$reg),
10242 as_Register($src$$reg),
10243 zr,
10244 (Assembler::Condition)$cmp$$cmpcode);
10245 %}
10246
10247 ins_pipe(icond_reg);
10248 %}
10249
10250 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10251 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10252
10253 ins_cost(INSN_COST * 2);
10254 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
10255
10256 ins_encode %{
10257 __ csel(as_Register($dst$$reg),
10258 as_Register($src$$reg),
10259 zr,
10260 (Assembler::Condition)$cmp$$cmpcode);
10261 %}
10262
10263 ins_pipe(icond_reg);
10264 %}
10265
10266 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10267 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10268
10269 ins_cost(INSN_COST * 2);
10270 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10271
10272 ins_encode %{
10273 __ cselw(as_Register($dst$$reg),
10274 as_Register($src2$$reg),
10275 as_Register($src1$$reg),
10276 (Assembler::Condition)$cmp$$cmpcode);
10277 %}
10278
10279 ins_pipe(icond_reg_reg);
10280 %}
10281
10282 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10283 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10284
10285 ins_cost(INSN_COST * 2);
10286 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10287
10288 ins_encode %{
10289 __ cselw(as_Register($dst$$reg),
10290 as_Register($src2$$reg),
10291 as_Register($src1$$reg),
10292 (Assembler::Condition)$cmp$$cmpcode);
10293 %}
10294
10295 ins_pipe(icond_reg_reg);
10296 %}
10297
10298 // special cases where one arg is zero
10299
10300 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10301 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10302
10303 ins_cost(INSN_COST * 2);
10304 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
10305
10306 ins_encode %{
10307 __ cselw(as_Register($dst$$reg),
10308 zr,
10309 as_Register($src$$reg),
10310 (Assembler::Condition)$cmp$$cmpcode);
10311 %}
10312
10313 ins_pipe(icond_reg);
10314 %}
10315
10316 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10317 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10318
10319 ins_cost(INSN_COST * 2);
10320 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
10321
10322 ins_encode %{
10323 __ cselw(as_Register($dst$$reg),
10324 zr,
10325 as_Register($src$$reg),
10326 (Assembler::Condition)$cmp$$cmpcode);
10327 %}
10328
10329 ins_pipe(icond_reg);
10330 %}
10331
10332 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10333 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10334
10335 ins_cost(INSN_COST * 2);
10336 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
10337
10338 ins_encode %{
10339 __ cselw(as_Register($dst$$reg),
10340 as_Register($src$$reg),
10341 zr,
10342 (Assembler::Condition)$cmp$$cmpcode);
10343 %}
10344
10345 ins_pipe(icond_reg);
10346 %}
10347
10348 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10349 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10350
10351 ins_cost(INSN_COST * 2);
10352 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
10353
10354 ins_encode %{
10355 __ cselw(as_Register($dst$$reg),
10356 as_Register($src$$reg),
10357 zr,
10358 (Assembler::Condition)$cmp$$cmpcode);
10359 %}
10360
10361 ins_pipe(icond_reg);
10362 %}
10363
10364 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
10365 %{
10366 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10367
10368 ins_cost(INSN_COST * 3);
10369
10370 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10371 ins_encode %{
10372 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10373 __ fcsels(as_FloatRegister($dst$$reg),
10374 as_FloatRegister($src2$$reg),
10375 as_FloatRegister($src1$$reg),
10376 cond);
10377 %}
10378
10379 ins_pipe(fp_cond_reg_reg_s);
10380 %}
10381
10382 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
10383 %{
10384 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10385
10386 ins_cost(INSN_COST * 3);
10387
10388 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10389 ins_encode %{
10390 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10391 __ fcsels(as_FloatRegister($dst$$reg),
10392 as_FloatRegister($src2$$reg),
10393 as_FloatRegister($src1$$reg),
10394 cond);
10395 %}
10396
10397 ins_pipe(fp_cond_reg_reg_s);
10398 %}
10399
10400 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10401 %{
10402 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10403
10404 ins_cost(INSN_COST * 3);
10405
10406 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10407 ins_encode %{
10408 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10409 __ fcseld(as_FloatRegister($dst$$reg),
10410 as_FloatRegister($src2$$reg),
10411 as_FloatRegister($src1$$reg),
10412 cond);
10413 %}
10414
10415 ins_pipe(fp_cond_reg_reg_d);
10416 %}
10417
10418 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10419 %{
10420 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10421
10422 ins_cost(INSN_COST * 3);
10423
10424 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10425 ins_encode %{
10426 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10427 __ fcseld(as_FloatRegister($dst$$reg),
10428 as_FloatRegister($src2$$reg),
10429 as_FloatRegister($src1$$reg),
10430 cond);
10431 %}
10432
10433 ins_pipe(fp_cond_reg_reg_d);
10434 %}
10435
10436 // ============================================================================
10437 // Arithmetic Instructions
10438 //
10439
10440 // Integer Addition
10441
10442 // TODO
10443 // these currently employ operations which do not set CR and hence are
10444 // not flagged as killing CR but we would like to isolate the cases
10445 // where we want to set flags from those where we don't. need to work
10446 // out how to do that.
10447
10448 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10449 match(Set dst (AddI src1 src2));
10450
10451 ins_cost(INSN_COST);
10452 format %{ "addw $dst, $src1, $src2" %}
10453
10454 ins_encode %{
10455 __ addw(as_Register($dst$$reg),
10456 as_Register($src1$$reg),
10457 as_Register($src2$$reg));
10458 %}
10459
10460 ins_pipe(ialu_reg_reg);
10461 %}
10462
10463 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10464 match(Set dst (AddI src1 src2));
10465
10466 ins_cost(INSN_COST);
10467 format %{ "addw $dst, $src1, $src2" %}
10468
10469 // use opcode to indicate that this is an add not a sub
10470 opcode(0x0);
10471
10472 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10473
10474 ins_pipe(ialu_reg_imm);
10475 %}
10476
10477 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10478 match(Set dst (AddI (ConvL2I src1) src2));
10479
10480 ins_cost(INSN_COST);
10481 format %{ "addw $dst, $src1, $src2" %}
10482
10483 // use opcode to indicate that this is an add not a sub
10484 opcode(0x0);
10485
10486 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10487
10488 ins_pipe(ialu_reg_imm);
10489 %}
10490
10491 // Pointer Addition
10492 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10493 match(Set dst (AddP src1 src2));
10494
10495 ins_cost(INSN_COST);
10496 format %{ "add $dst, $src1, $src2\t# ptr" %}
10497
10498 ins_encode %{
10499 __ add(as_Register($dst$$reg),
10500 as_Register($src1$$reg),
10501 as_Register($src2$$reg));
10502 %}
10503
10504 ins_pipe(ialu_reg_reg);
10505 %}
10506
10507 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10508 match(Set dst (AddP src1 (ConvI2L src2)));
10509
10510 ins_cost(1.9 * INSN_COST);
10511 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10512
10513 ins_encode %{
10514 __ add(as_Register($dst$$reg),
10515 as_Register($src1$$reg),
10516 as_Register($src2$$reg), ext::sxtw);
10517 %}
10518
10519 ins_pipe(ialu_reg_reg);
10520 %}
10521
10522 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10523 match(Set dst (AddP src1 (LShiftL src2 scale)));
10524
10525 ins_cost(1.9 * INSN_COST);
10526 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10527
10528 ins_encode %{
10529 __ lea(as_Register($dst$$reg),
10530 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10531 Address::lsl($scale$$constant)));
10532 %}
10533
10534 ins_pipe(ialu_reg_reg_shift);
10535 %}
10536
10537 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10538 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10539
10540 ins_cost(1.9 * INSN_COST);
10541 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10542
10543 ins_encode %{
10544 __ lea(as_Register($dst$$reg),
10545 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10546 Address::sxtw($scale$$constant)));
10547 %}
10548
10549 ins_pipe(ialu_reg_reg_shift);
10550 %}
10551
10552 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10553 match(Set dst (LShiftL (ConvI2L src) scale));
10554
10555 ins_cost(INSN_COST);
10556 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10557
10558 ins_encode %{
10559 __ sbfiz(as_Register($dst$$reg),
10560 as_Register($src$$reg),
10561 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10562 %}
10563
10564 ins_pipe(ialu_reg_shift);
10565 %}
10566
10567 // Pointer Immediate Addition
10568 // n.b. this needs to be more expensive than using an indirect memory
10569 // operand
10570 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10571 match(Set dst (AddP src1 src2));
10572
10573 ins_cost(INSN_COST);
10574 format %{ "add $dst, $src1, $src2\t# ptr" %}
10575
10576 // use opcode to indicate that this is an add not a sub
10577 opcode(0x0);
10578
10579 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10580
10581 ins_pipe(ialu_reg_imm);
10582 %}
10583
10584 // Long Addition
10585 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10586
10587 match(Set dst (AddL src1 src2));
10588
10589 ins_cost(INSN_COST);
10590 format %{ "add $dst, $src1, $src2" %}
10591
10592 ins_encode %{
10593 __ add(as_Register($dst$$reg),
10594 as_Register($src1$$reg),
10595 as_Register($src2$$reg));
10596 %}
10597
10598 ins_pipe(ialu_reg_reg);
10599 %}
10600
10601 // No constant pool entries requiredLong Immediate Addition.
10602 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10603 match(Set dst (AddL src1 src2));
10604
10605 ins_cost(INSN_COST);
10606 format %{ "add $dst, $src1, $src2" %}
10607
10608 // use opcode to indicate that this is an add not a sub
10609 opcode(0x0);
10610
10611 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10612
10613 ins_pipe(ialu_reg_imm);
10614 %}
10615
10616 // Integer Subtraction
10617 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10618 match(Set dst (SubI src1 src2));
10619
10620 ins_cost(INSN_COST);
10621 format %{ "subw $dst, $src1, $src2" %}
10622
10623 ins_encode %{
10624 __ subw(as_Register($dst$$reg),
10625 as_Register($src1$$reg),
10626 as_Register($src2$$reg));
10627 %}
10628
10629 ins_pipe(ialu_reg_reg);
10630 %}
10631
10632 // Immediate Subtraction
10633 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10634 match(Set dst (SubI src1 src2));
10635
10636 ins_cost(INSN_COST);
10637 format %{ "subw $dst, $src1, $src2" %}
10638
10639 // use opcode to indicate that this is a sub not an add
10640 opcode(0x1);
10641
10642 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10643
10644 ins_pipe(ialu_reg_imm);
10645 %}
10646
10647 // Long Subtraction
10648 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10649
10650 match(Set dst (SubL src1 src2));
10651
10652 ins_cost(INSN_COST);
10653 format %{ "sub $dst, $src1, $src2" %}
10654
10655 ins_encode %{
10656 __ sub(as_Register($dst$$reg),
10657 as_Register($src1$$reg),
10658 as_Register($src2$$reg));
10659 %}
10660
10661 ins_pipe(ialu_reg_reg);
10662 %}
10663
10664 // No constant pool entries requiredLong Immediate Subtraction.
10665 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10666 match(Set dst (SubL src1 src2));
10667
10668 ins_cost(INSN_COST);
10669 format %{ "sub$dst, $src1, $src2" %}
10670
10671 // use opcode to indicate that this is a sub not an add
10672 opcode(0x1);
10673
10674 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10675
10676 ins_pipe(ialu_reg_imm);
10677 %}
10678
10679 // Integer Negation (special case for sub)
10680
10681 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10682 match(Set dst (SubI zero src));
10683
10684 ins_cost(INSN_COST);
10685 format %{ "negw $dst, $src\t# int" %}
10686
10687 ins_encode %{
10688 __ negw(as_Register($dst$$reg),
10689 as_Register($src$$reg));
10690 %}
10691
10692 ins_pipe(ialu_reg);
10693 %}
10694
10695 // Long Negation
10696
10697 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10698 match(Set dst (SubL zero src));
10699
10700 ins_cost(INSN_COST);
10701 format %{ "neg $dst, $src\t# long" %}
10702
10703 ins_encode %{
10704 __ neg(as_Register($dst$$reg),
10705 as_Register($src$$reg));
10706 %}
10707
10708 ins_pipe(ialu_reg);
10709 %}
10710
10711 // Integer Multiply
10712
10713 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10714 match(Set dst (MulI src1 src2));
10715
10716 ins_cost(INSN_COST * 3);
10717 format %{ "mulw $dst, $src1, $src2" %}
10718
10719 ins_encode %{
10720 __ mulw(as_Register($dst$$reg),
10721 as_Register($src1$$reg),
10722 as_Register($src2$$reg));
10723 %}
10724
10725 ins_pipe(imul_reg_reg);
10726 %}
10727
10728 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10729 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10730
10731 ins_cost(INSN_COST * 3);
10732 format %{ "smull $dst, $src1, $src2" %}
10733
10734 ins_encode %{
10735 __ smull(as_Register($dst$$reg),
10736 as_Register($src1$$reg),
10737 as_Register($src2$$reg));
10738 %}
10739
10740 ins_pipe(imul_reg_reg);
10741 %}
10742
10743 // Long Multiply
10744
10745 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10746 match(Set dst (MulL src1 src2));
10747
10748 ins_cost(INSN_COST * 5);
10749 format %{ "mul $dst, $src1, $src2" %}
10750
10751 ins_encode %{
10752 __ mul(as_Register($dst$$reg),
10753 as_Register($src1$$reg),
10754 as_Register($src2$$reg));
10755 %}
10756
10757 ins_pipe(lmul_reg_reg);
10758 %}
10759
10760 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10761 %{
10762 match(Set dst (MulHiL src1 src2));
10763
10764 ins_cost(INSN_COST * 7);
10765 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10766
10767 ins_encode %{
10768 __ smulh(as_Register($dst$$reg),
10769 as_Register($src1$$reg),
10770 as_Register($src2$$reg));
10771 %}
10772
10773 ins_pipe(lmul_reg_reg);
10774 %}
10775
10776 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10777 %{
10778 match(Set dst (UMulHiL src1 src2));
10779
10780 ins_cost(INSN_COST * 7);
10781 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10782
10783 ins_encode %{
10784 __ umulh(as_Register($dst$$reg),
10785 as_Register($src1$$reg),
10786 as_Register($src2$$reg));
10787 %}
10788
10789 ins_pipe(lmul_reg_reg);
10790 %}
10791
10792 // Combined Integer Multiply & Add/Sub
10793
10794 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10795 match(Set dst (AddI src3 (MulI src1 src2)));
10796
10797 ins_cost(INSN_COST * 3);
10798 format %{ "madd $dst, $src1, $src2, $src3" %}
10799
10800 ins_encode %{
10801 __ maddw(as_Register($dst$$reg),
10802 as_Register($src1$$reg),
10803 as_Register($src2$$reg),
10804 as_Register($src3$$reg));
10805 %}
10806
10807 ins_pipe(imac_reg_reg);
10808 %}
10809
10810 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10811 match(Set dst (SubI src3 (MulI src1 src2)));
10812
10813 ins_cost(INSN_COST * 3);
10814 format %{ "msub $dst, $src1, $src2, $src3" %}
10815
10816 ins_encode %{
10817 __ msubw(as_Register($dst$$reg),
10818 as_Register($src1$$reg),
10819 as_Register($src2$$reg),
10820 as_Register($src3$$reg));
10821 %}
10822
10823 ins_pipe(imac_reg_reg);
10824 %}
10825
10826 // Combined Integer Multiply & Neg
10827
10828 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10829 match(Set dst (MulI (SubI zero src1) src2));
10830
10831 ins_cost(INSN_COST * 3);
10832 format %{ "mneg $dst, $src1, $src2" %}
10833
10834 ins_encode %{
10835 __ mnegw(as_Register($dst$$reg),
10836 as_Register($src1$$reg),
10837 as_Register($src2$$reg));
10838 %}
10839
10840 ins_pipe(imac_reg_reg);
10841 %}
10842
10843 // Combined Long Multiply & Add/Sub
10844
10845 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10846 match(Set dst (AddL src3 (MulL src1 src2)));
10847
10848 ins_cost(INSN_COST * 5);
10849 format %{ "madd $dst, $src1, $src2, $src3" %}
10850
10851 ins_encode %{
10852 __ madd(as_Register($dst$$reg),
10853 as_Register($src1$$reg),
10854 as_Register($src2$$reg),
10855 as_Register($src3$$reg));
10856 %}
10857
10858 ins_pipe(lmac_reg_reg);
10859 %}
10860
10861 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10862 match(Set dst (SubL src3 (MulL src1 src2)));
10863
10864 ins_cost(INSN_COST * 5);
10865 format %{ "msub $dst, $src1, $src2, $src3" %}
10866
10867 ins_encode %{
10868 __ msub(as_Register($dst$$reg),
10869 as_Register($src1$$reg),
10870 as_Register($src2$$reg),
10871 as_Register($src3$$reg));
10872 %}
10873
10874 ins_pipe(lmac_reg_reg);
10875 %}
10876
10877 // Combined Long Multiply & Neg
10878
10879 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10880 match(Set dst (MulL (SubL zero src1) src2));
10881
10882 ins_cost(INSN_COST * 5);
10883 format %{ "mneg $dst, $src1, $src2" %}
10884
10885 ins_encode %{
10886 __ mneg(as_Register($dst$$reg),
10887 as_Register($src1$$reg),
10888 as_Register($src2$$reg));
10889 %}
10890
10891 ins_pipe(lmac_reg_reg);
10892 %}
10893
10894 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10895
10896 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10897 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10898
10899 ins_cost(INSN_COST * 3);
10900 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10901
10902 ins_encode %{
10903 __ smaddl(as_Register($dst$$reg),
10904 as_Register($src1$$reg),
10905 as_Register($src2$$reg),
10906 as_Register($src3$$reg));
10907 %}
10908
10909 ins_pipe(imac_reg_reg);
10910 %}
10911
10912 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10913 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10914
10915 ins_cost(INSN_COST * 3);
10916 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10917
10918 ins_encode %{
10919 __ smsubl(as_Register($dst$$reg),
10920 as_Register($src1$$reg),
10921 as_Register($src2$$reg),
10922 as_Register($src3$$reg));
10923 %}
10924
10925 ins_pipe(imac_reg_reg);
10926 %}
10927
10928 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10929 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10930
10931 ins_cost(INSN_COST * 3);
10932 format %{ "smnegl $dst, $src1, $src2" %}
10933
10934 ins_encode %{
10935 __ smnegl(as_Register($dst$$reg),
10936 as_Register($src1$$reg),
10937 as_Register($src2$$reg));
10938 %}
10939
10940 ins_pipe(imac_reg_reg);
10941 %}
10942
10943 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10944
10945 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10946 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10947
10948 ins_cost(INSN_COST * 5);
10949 format %{ "mulw rscratch1, $src1, $src2\n\t"
10950 "maddw $dst, $src3, $src4, rscratch1" %}
10951
10952 ins_encode %{
10953 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10954 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10955
10956 ins_pipe(imac_reg_reg);
10957 %}
10958
10959 // Integer Divide
10960
10961 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10962 match(Set dst (DivI src1 src2));
10963
10964 ins_cost(INSN_COST * 19);
10965 format %{ "sdivw $dst, $src1, $src2" %}
10966
10967 ins_encode(aarch64_enc_divw(dst, src1, src2));
10968 ins_pipe(idiv_reg_reg);
10969 %}
10970
10971 // Long Divide
10972
10973 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10974 match(Set dst (DivL src1 src2));
10975
10976 ins_cost(INSN_COST * 35);
10977 format %{ "sdiv $dst, $src1, $src2" %}
10978
10979 ins_encode(aarch64_enc_div(dst, src1, src2));
10980 ins_pipe(ldiv_reg_reg);
10981 %}
10982
10983 // Integer Remainder
10984
10985 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10986 match(Set dst (ModI src1 src2));
10987
10988 ins_cost(INSN_COST * 22);
10989 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10990 "msubw $dst, rscratch1, $src2, $src1" %}
10991
10992 ins_encode(aarch64_enc_modw(dst, src1, src2));
10993 ins_pipe(idiv_reg_reg);
10994 %}
10995
10996 // Long Remainder
10997
10998 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10999 match(Set dst (ModL src1 src2));
11000
11001 ins_cost(INSN_COST * 38);
11002 format %{ "sdiv rscratch1, $src1, $src2\n"
11003 "msub $dst, rscratch1, $src2, $src1" %}
11004
11005 ins_encode(aarch64_enc_mod(dst, src1, src2));
11006 ins_pipe(ldiv_reg_reg);
11007 %}
11008
11009 // Unsigned Integer Divide
11010
11011 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11012 match(Set dst (UDivI src1 src2));
11013
11014 ins_cost(INSN_COST * 19);
11015 format %{ "udivw $dst, $src1, $src2" %}
11016
11017 ins_encode %{
11018 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
11019 %}
11020
11021 ins_pipe(idiv_reg_reg);
11022 %}
11023
11024 // Unsigned Long Divide
11025
11026 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11027 match(Set dst (UDivL src1 src2));
11028
11029 ins_cost(INSN_COST * 35);
11030 format %{ "udiv $dst, $src1, $src2" %}
11031
11032 ins_encode %{
11033 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
11034 %}
11035
11036 ins_pipe(ldiv_reg_reg);
11037 %}
11038
11039 // Unsigned Integer Remainder
11040
11041 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11042 match(Set dst (UModI src1 src2));
11043
11044 ins_cost(INSN_COST * 22);
11045 format %{ "udivw rscratch1, $src1, $src2\n\t"
11046 "msubw $dst, rscratch1, $src2, $src1" %}
11047
11048 ins_encode %{
11049 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
11050 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
11051 %}
11052
11053 ins_pipe(idiv_reg_reg);
11054 %}
11055
11056 // Unsigned Long Remainder
11057
11058 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11059 match(Set dst (UModL src1 src2));
11060
11061 ins_cost(INSN_COST * 38);
11062 format %{ "udiv rscratch1, $src1, $src2\n"
11063 "msub $dst, rscratch1, $src2, $src1" %}
11064
11065 ins_encode %{
11066 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
11067 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
11068 %}
11069
11070 ins_pipe(ldiv_reg_reg);
11071 %}
11072
11073 // Integer Shifts
11074
11075 // Shift Left Register
11076 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11077 match(Set dst (LShiftI src1 src2));
11078
11079 ins_cost(INSN_COST * 2);
11080 format %{ "lslvw $dst, $src1, $src2" %}
11081
11082 ins_encode %{
11083 __ lslvw(as_Register($dst$$reg),
11084 as_Register($src1$$reg),
11085 as_Register($src2$$reg));
11086 %}
11087
11088 ins_pipe(ialu_reg_reg_vshift);
11089 %}
11090
11091 // Shift Left Immediate
11092 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11093 match(Set dst (LShiftI src1 src2));
11094
11095 ins_cost(INSN_COST);
11096 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11097
11098 ins_encode %{
11099 __ lslw(as_Register($dst$$reg),
11100 as_Register($src1$$reg),
11101 $src2$$constant & 0x1f);
11102 %}
11103
11104 ins_pipe(ialu_reg_shift);
11105 %}
11106
11107 // Shift Right Logical Register
11108 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11109 match(Set dst (URShiftI src1 src2));
11110
11111 ins_cost(INSN_COST * 2);
11112 format %{ "lsrvw $dst, $src1, $src2" %}
11113
11114 ins_encode %{
11115 __ lsrvw(as_Register($dst$$reg),
11116 as_Register($src1$$reg),
11117 as_Register($src2$$reg));
11118 %}
11119
11120 ins_pipe(ialu_reg_reg_vshift);
11121 %}
11122
11123 // Shift Right Logical Immediate
11124 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11125 match(Set dst (URShiftI src1 src2));
11126
11127 ins_cost(INSN_COST);
11128 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11129
11130 ins_encode %{
11131 __ lsrw(as_Register($dst$$reg),
11132 as_Register($src1$$reg),
11133 $src2$$constant & 0x1f);
11134 %}
11135
11136 ins_pipe(ialu_reg_shift);
11137 %}
11138
11139 // Shift Right Arithmetic Register
11140 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11141 match(Set dst (RShiftI src1 src2));
11142
11143 ins_cost(INSN_COST * 2);
11144 format %{ "asrvw $dst, $src1, $src2" %}
11145
11146 ins_encode %{
11147 __ asrvw(as_Register($dst$$reg),
11148 as_Register($src1$$reg),
11149 as_Register($src2$$reg));
11150 %}
11151
11152 ins_pipe(ialu_reg_reg_vshift);
11153 %}
11154
11155 // Shift Right Arithmetic Immediate
11156 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11157 match(Set dst (RShiftI src1 src2));
11158
11159 ins_cost(INSN_COST);
11160 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11161
11162 ins_encode %{
11163 __ asrw(as_Register($dst$$reg),
11164 as_Register($src1$$reg),
11165 $src2$$constant & 0x1f);
11166 %}
11167
11168 ins_pipe(ialu_reg_shift);
11169 %}
11170
11171 // Combined Int Mask and Right Shift (using UBFM)
11172 // TODO
11173
11174 // Long Shifts
11175
11176 // Shift Left Register
11177 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11178 match(Set dst (LShiftL src1 src2));
11179
11180 ins_cost(INSN_COST * 2);
11181 format %{ "lslv $dst, $src1, $src2" %}
11182
11183 ins_encode %{
11184 __ lslv(as_Register($dst$$reg),
11185 as_Register($src1$$reg),
11186 as_Register($src2$$reg));
11187 %}
11188
11189 ins_pipe(ialu_reg_reg_vshift);
11190 %}
11191
11192 // Shift Left Immediate
11193 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11194 match(Set dst (LShiftL src1 src2));
11195
11196 ins_cost(INSN_COST);
11197 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11198
11199 ins_encode %{
11200 __ lsl(as_Register($dst$$reg),
11201 as_Register($src1$$reg),
11202 $src2$$constant & 0x3f);
11203 %}
11204
11205 ins_pipe(ialu_reg_shift);
11206 %}
11207
11208 // Shift Right Logical Register
11209 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11210 match(Set dst (URShiftL src1 src2));
11211
11212 ins_cost(INSN_COST * 2);
11213 format %{ "lsrv $dst, $src1, $src2" %}
11214
11215 ins_encode %{
11216 __ lsrv(as_Register($dst$$reg),
11217 as_Register($src1$$reg),
11218 as_Register($src2$$reg));
11219 %}
11220
11221 ins_pipe(ialu_reg_reg_vshift);
11222 %}
11223
11224 // Shift Right Logical Immediate
11225 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11226 match(Set dst (URShiftL src1 src2));
11227
11228 ins_cost(INSN_COST);
11229 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11230
11231 ins_encode %{
11232 __ lsr(as_Register($dst$$reg),
11233 as_Register($src1$$reg),
11234 $src2$$constant & 0x3f);
11235 %}
11236
11237 ins_pipe(ialu_reg_shift);
11238 %}
11239
11240 // A special-case pattern for card table stores.
11241 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11242 match(Set dst (URShiftL (CastP2X src1) src2));
11243
11244 ins_cost(INSN_COST);
11245 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11246
11247 ins_encode %{
11248 __ lsr(as_Register($dst$$reg),
11249 as_Register($src1$$reg),
11250 $src2$$constant & 0x3f);
11251 %}
11252
11253 ins_pipe(ialu_reg_shift);
11254 %}
11255
11256 // Shift Right Arithmetic Register
11257 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11258 match(Set dst (RShiftL src1 src2));
11259
11260 ins_cost(INSN_COST * 2);
11261 format %{ "asrv $dst, $src1, $src2" %}
11262
11263 ins_encode %{
11264 __ asrv(as_Register($dst$$reg),
11265 as_Register($src1$$reg),
11266 as_Register($src2$$reg));
11267 %}
11268
11269 ins_pipe(ialu_reg_reg_vshift);
11270 %}
11271
11272 // Shift Right Arithmetic Immediate
11273 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11274 match(Set dst (RShiftL src1 src2));
11275
11276 ins_cost(INSN_COST);
11277 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11278
11279 ins_encode %{
11280 __ asr(as_Register($dst$$reg),
11281 as_Register($src1$$reg),
11282 $src2$$constant & 0x3f);
11283 %}
11284
11285 ins_pipe(ialu_reg_shift);
11286 %}
11287
11288 // BEGIN This section of the file is automatically generated. Do not edit --------------
11289 // This section is generated from aarch64_ad.m4
11290
11291 // This pattern is automatically generated from aarch64_ad.m4
11292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11293 instruct regL_not_reg(iRegLNoSp dst,
11294 iRegL src1, immL_M1 m1,
11295 rFlagsReg cr) %{
11296 match(Set dst (XorL src1 m1));
11297 ins_cost(INSN_COST);
11298 format %{ "eon $dst, $src1, zr" %}
11299
11300 ins_encode %{
11301 __ eon(as_Register($dst$$reg),
11302 as_Register($src1$$reg),
11303 zr,
11304 Assembler::LSL, 0);
11305 %}
11306
11307 ins_pipe(ialu_reg);
11308 %}
11309
11310 // This pattern is automatically generated from aarch64_ad.m4
11311 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11312 instruct regI_not_reg(iRegINoSp dst,
11313 iRegIorL2I src1, immI_M1 m1,
11314 rFlagsReg cr) %{
11315 match(Set dst (XorI src1 m1));
11316 ins_cost(INSN_COST);
11317 format %{ "eonw $dst, $src1, zr" %}
11318
11319 ins_encode %{
11320 __ eonw(as_Register($dst$$reg),
11321 as_Register($src1$$reg),
11322 zr,
11323 Assembler::LSL, 0);
11324 %}
11325
11326 ins_pipe(ialu_reg);
11327 %}
11328
11329 // This pattern is automatically generated from aarch64_ad.m4
11330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11331 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11332 immI0 zero, iRegIorL2I src1, immI src2) %{
11333 match(Set dst (SubI zero (URShiftI src1 src2)));
11334
11335 ins_cost(1.9 * INSN_COST);
11336 format %{ "negw $dst, $src1, LSR $src2" %}
11337
11338 ins_encode %{
11339 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11340 Assembler::LSR, $src2$$constant & 0x1f);
11341 %}
11342
11343 ins_pipe(ialu_reg_shift);
11344 %}
11345
11346 // This pattern is automatically generated from aarch64_ad.m4
11347 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11348 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11349 immI0 zero, iRegIorL2I src1, immI src2) %{
11350 match(Set dst (SubI zero (RShiftI src1 src2)));
11351
11352 ins_cost(1.9 * INSN_COST);
11353 format %{ "negw $dst, $src1, ASR $src2" %}
11354
11355 ins_encode %{
11356 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11357 Assembler::ASR, $src2$$constant & 0x1f);
11358 %}
11359
11360 ins_pipe(ialu_reg_shift);
11361 %}
11362
11363 // This pattern is automatically generated from aarch64_ad.m4
11364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11365 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11366 immI0 zero, iRegIorL2I src1, immI src2) %{
11367 match(Set dst (SubI zero (LShiftI src1 src2)));
11368
11369 ins_cost(1.9 * INSN_COST);
11370 format %{ "negw $dst, $src1, LSL $src2" %}
11371
11372 ins_encode %{
11373 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11374 Assembler::LSL, $src2$$constant & 0x1f);
11375 %}
11376
11377 ins_pipe(ialu_reg_shift);
11378 %}
11379
11380 // This pattern is automatically generated from aarch64_ad.m4
11381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11382 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11383 immL0 zero, iRegL src1, immI src2) %{
11384 match(Set dst (SubL zero (URShiftL src1 src2)));
11385
11386 ins_cost(1.9 * INSN_COST);
11387 format %{ "neg $dst, $src1, LSR $src2" %}
11388
11389 ins_encode %{
11390 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11391 Assembler::LSR, $src2$$constant & 0x3f);
11392 %}
11393
11394 ins_pipe(ialu_reg_shift);
11395 %}
11396
11397 // This pattern is automatically generated from aarch64_ad.m4
11398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11399 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11400 immL0 zero, iRegL src1, immI src2) %{
11401 match(Set dst (SubL zero (RShiftL src1 src2)));
11402
11403 ins_cost(1.9 * INSN_COST);
11404 format %{ "neg $dst, $src1, ASR $src2" %}
11405
11406 ins_encode %{
11407 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11408 Assembler::ASR, $src2$$constant & 0x3f);
11409 %}
11410
11411 ins_pipe(ialu_reg_shift);
11412 %}
11413
11414 // This pattern is automatically generated from aarch64_ad.m4
11415 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11416 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11417 immL0 zero, iRegL src1, immI src2) %{
11418 match(Set dst (SubL zero (LShiftL src1 src2)));
11419
11420 ins_cost(1.9 * INSN_COST);
11421 format %{ "neg $dst, $src1, LSL $src2" %}
11422
11423 ins_encode %{
11424 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11425 Assembler::LSL, $src2$$constant & 0x3f);
11426 %}
11427
11428 ins_pipe(ialu_reg_shift);
11429 %}
11430
11431 // This pattern is automatically generated from aarch64_ad.m4
11432 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11433 instruct AndI_reg_not_reg(iRegINoSp dst,
11434 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11435 match(Set dst (AndI src1 (XorI src2 m1)));
11436 ins_cost(INSN_COST);
11437 format %{ "bicw $dst, $src1, $src2" %}
11438
11439 ins_encode %{
11440 __ bicw(as_Register($dst$$reg),
11441 as_Register($src1$$reg),
11442 as_Register($src2$$reg),
11443 Assembler::LSL, 0);
11444 %}
11445
11446 ins_pipe(ialu_reg_reg);
11447 %}
11448
11449 // This pattern is automatically generated from aarch64_ad.m4
11450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11451 instruct AndL_reg_not_reg(iRegLNoSp dst,
11452 iRegL src1, iRegL src2, immL_M1 m1) %{
11453 match(Set dst (AndL src1 (XorL src2 m1)));
11454 ins_cost(INSN_COST);
11455 format %{ "bic $dst, $src1, $src2" %}
11456
11457 ins_encode %{
11458 __ bic(as_Register($dst$$reg),
11459 as_Register($src1$$reg),
11460 as_Register($src2$$reg),
11461 Assembler::LSL, 0);
11462 %}
11463
11464 ins_pipe(ialu_reg_reg);
11465 %}
11466
11467 // This pattern is automatically generated from aarch64_ad.m4
11468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11469 instruct OrI_reg_not_reg(iRegINoSp dst,
11470 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11471 match(Set dst (OrI src1 (XorI src2 m1)));
11472 ins_cost(INSN_COST);
11473 format %{ "ornw $dst, $src1, $src2" %}
11474
11475 ins_encode %{
11476 __ ornw(as_Register($dst$$reg),
11477 as_Register($src1$$reg),
11478 as_Register($src2$$reg),
11479 Assembler::LSL, 0);
11480 %}
11481
11482 ins_pipe(ialu_reg_reg);
11483 %}
11484
11485 // This pattern is automatically generated from aarch64_ad.m4
11486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11487 instruct OrL_reg_not_reg(iRegLNoSp dst,
11488 iRegL src1, iRegL src2, immL_M1 m1) %{
11489 match(Set dst (OrL src1 (XorL src2 m1)));
11490 ins_cost(INSN_COST);
11491 format %{ "orn $dst, $src1, $src2" %}
11492
11493 ins_encode %{
11494 __ orn(as_Register($dst$$reg),
11495 as_Register($src1$$reg),
11496 as_Register($src2$$reg),
11497 Assembler::LSL, 0);
11498 %}
11499
11500 ins_pipe(ialu_reg_reg);
11501 %}
11502
11503 // This pattern is automatically generated from aarch64_ad.m4
11504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11505 instruct XorI_reg_not_reg(iRegINoSp dst,
11506 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11507 match(Set dst (XorI m1 (XorI src2 src1)));
11508 ins_cost(INSN_COST);
11509 format %{ "eonw $dst, $src1, $src2" %}
11510
11511 ins_encode %{
11512 __ eonw(as_Register($dst$$reg),
11513 as_Register($src1$$reg),
11514 as_Register($src2$$reg),
11515 Assembler::LSL, 0);
11516 %}
11517
11518 ins_pipe(ialu_reg_reg);
11519 %}
11520
11521 // This pattern is automatically generated from aarch64_ad.m4
11522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11523 instruct XorL_reg_not_reg(iRegLNoSp dst,
11524 iRegL src1, iRegL src2, immL_M1 m1) %{
11525 match(Set dst (XorL m1 (XorL src2 src1)));
11526 ins_cost(INSN_COST);
11527 format %{ "eon $dst, $src1, $src2" %}
11528
11529 ins_encode %{
11530 __ eon(as_Register($dst$$reg),
11531 as_Register($src1$$reg),
11532 as_Register($src2$$reg),
11533 Assembler::LSL, 0);
11534 %}
11535
11536 ins_pipe(ialu_reg_reg);
11537 %}
11538
11539 // This pattern is automatically generated from aarch64_ad.m4
11540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11541 // val & (-1 ^ (val >>> shift)) ==> bicw
11542 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11543 iRegIorL2I src1, iRegIorL2I src2,
11544 immI src3, immI_M1 src4) %{
11545 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11546 ins_cost(1.9 * INSN_COST);
11547 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11548
11549 ins_encode %{
11550 __ bicw(as_Register($dst$$reg),
11551 as_Register($src1$$reg),
11552 as_Register($src2$$reg),
11553 Assembler::LSR,
11554 $src3$$constant & 0x1f);
11555 %}
11556
11557 ins_pipe(ialu_reg_reg_shift);
11558 %}
11559
11560 // This pattern is automatically generated from aarch64_ad.m4
11561 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11562 // val & (-1 ^ (val >>> shift)) ==> bic
11563 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11564 iRegL src1, iRegL src2,
11565 immI src3, immL_M1 src4) %{
11566 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11567 ins_cost(1.9 * INSN_COST);
11568 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11569
11570 ins_encode %{
11571 __ bic(as_Register($dst$$reg),
11572 as_Register($src1$$reg),
11573 as_Register($src2$$reg),
11574 Assembler::LSR,
11575 $src3$$constant & 0x3f);
11576 %}
11577
11578 ins_pipe(ialu_reg_reg_shift);
11579 %}
11580
11581 // This pattern is automatically generated from aarch64_ad.m4
11582 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11583 // val & (-1 ^ (val >> shift)) ==> bicw
11584 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11585 iRegIorL2I src1, iRegIorL2I src2,
11586 immI src3, immI_M1 src4) %{
11587 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11588 ins_cost(1.9 * INSN_COST);
11589 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11590
11591 ins_encode %{
11592 __ bicw(as_Register($dst$$reg),
11593 as_Register($src1$$reg),
11594 as_Register($src2$$reg),
11595 Assembler::ASR,
11596 $src3$$constant & 0x1f);
11597 %}
11598
11599 ins_pipe(ialu_reg_reg_shift);
11600 %}
11601
11602 // This pattern is automatically generated from aarch64_ad.m4
11603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11604 // val & (-1 ^ (val >> shift)) ==> bic
11605 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11606 iRegL src1, iRegL src2,
11607 immI src3, immL_M1 src4) %{
11608 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11609 ins_cost(1.9 * INSN_COST);
11610 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11611
11612 ins_encode %{
11613 __ bic(as_Register($dst$$reg),
11614 as_Register($src1$$reg),
11615 as_Register($src2$$reg),
11616 Assembler::ASR,
11617 $src3$$constant & 0x3f);
11618 %}
11619
11620 ins_pipe(ialu_reg_reg_shift);
11621 %}
11622
11623 // This pattern is automatically generated from aarch64_ad.m4
11624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11625 // val & (-1 ^ (val ror shift)) ==> bicw
11626 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11627 iRegIorL2I src1, iRegIorL2I src2,
11628 immI src3, immI_M1 src4) %{
11629 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11630 ins_cost(1.9 * INSN_COST);
11631 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11632
11633 ins_encode %{
11634 __ bicw(as_Register($dst$$reg),
11635 as_Register($src1$$reg),
11636 as_Register($src2$$reg),
11637 Assembler::ROR,
11638 $src3$$constant & 0x1f);
11639 %}
11640
11641 ins_pipe(ialu_reg_reg_shift);
11642 %}
11643
11644 // This pattern is automatically generated from aarch64_ad.m4
11645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11646 // val & (-1 ^ (val ror shift)) ==> bic
11647 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11648 iRegL src1, iRegL src2,
11649 immI src3, immL_M1 src4) %{
11650 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11651 ins_cost(1.9 * INSN_COST);
11652 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11653
11654 ins_encode %{
11655 __ bic(as_Register($dst$$reg),
11656 as_Register($src1$$reg),
11657 as_Register($src2$$reg),
11658 Assembler::ROR,
11659 $src3$$constant & 0x3f);
11660 %}
11661
11662 ins_pipe(ialu_reg_reg_shift);
11663 %}
11664
11665 // This pattern is automatically generated from aarch64_ad.m4
11666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11667 // val & (-1 ^ (val << shift)) ==> bicw
11668 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11669 iRegIorL2I src1, iRegIorL2I src2,
11670 immI src3, immI_M1 src4) %{
11671 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11672 ins_cost(1.9 * INSN_COST);
11673 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11674
11675 ins_encode %{
11676 __ bicw(as_Register($dst$$reg),
11677 as_Register($src1$$reg),
11678 as_Register($src2$$reg),
11679 Assembler::LSL,
11680 $src3$$constant & 0x1f);
11681 %}
11682
11683 ins_pipe(ialu_reg_reg_shift);
11684 %}
11685
11686 // This pattern is automatically generated from aarch64_ad.m4
11687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11688 // val & (-1 ^ (val << shift)) ==> bic
11689 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11690 iRegL src1, iRegL src2,
11691 immI src3, immL_M1 src4) %{
11692 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11693 ins_cost(1.9 * INSN_COST);
11694 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11695
11696 ins_encode %{
11697 __ bic(as_Register($dst$$reg),
11698 as_Register($src1$$reg),
11699 as_Register($src2$$reg),
11700 Assembler::LSL,
11701 $src3$$constant & 0x3f);
11702 %}
11703
11704 ins_pipe(ialu_reg_reg_shift);
11705 %}
11706
11707 // This pattern is automatically generated from aarch64_ad.m4
11708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11709 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11710 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11711 iRegIorL2I src1, iRegIorL2I src2,
11712 immI src3, immI_M1 src4) %{
11713 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11714 ins_cost(1.9 * INSN_COST);
11715 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11716
11717 ins_encode %{
11718 __ eonw(as_Register($dst$$reg),
11719 as_Register($src1$$reg),
11720 as_Register($src2$$reg),
11721 Assembler::LSR,
11722 $src3$$constant & 0x1f);
11723 %}
11724
11725 ins_pipe(ialu_reg_reg_shift);
11726 %}
11727
11728 // This pattern is automatically generated from aarch64_ad.m4
11729 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11730 // val ^ (-1 ^ (val >>> shift)) ==> eon
11731 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11732 iRegL src1, iRegL src2,
11733 immI src3, immL_M1 src4) %{
11734 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11735 ins_cost(1.9 * INSN_COST);
11736 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11737
11738 ins_encode %{
11739 __ eon(as_Register($dst$$reg),
11740 as_Register($src1$$reg),
11741 as_Register($src2$$reg),
11742 Assembler::LSR,
11743 $src3$$constant & 0x3f);
11744 %}
11745
11746 ins_pipe(ialu_reg_reg_shift);
11747 %}
11748
11749 // This pattern is automatically generated from aarch64_ad.m4
11750 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11751 // val ^ (-1 ^ (val >> shift)) ==> eonw
11752 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11753 iRegIorL2I src1, iRegIorL2I src2,
11754 immI src3, immI_M1 src4) %{
11755 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11756 ins_cost(1.9 * INSN_COST);
11757 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11758
11759 ins_encode %{
11760 __ eonw(as_Register($dst$$reg),
11761 as_Register($src1$$reg),
11762 as_Register($src2$$reg),
11763 Assembler::ASR,
11764 $src3$$constant & 0x1f);
11765 %}
11766
11767 ins_pipe(ialu_reg_reg_shift);
11768 %}
11769
11770 // This pattern is automatically generated from aarch64_ad.m4
11771 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11772 // val ^ (-1 ^ (val >> shift)) ==> eon
11773 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11774 iRegL src1, iRegL src2,
11775 immI src3, immL_M1 src4) %{
11776 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11777 ins_cost(1.9 * INSN_COST);
11778 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11779
11780 ins_encode %{
11781 __ eon(as_Register($dst$$reg),
11782 as_Register($src1$$reg),
11783 as_Register($src2$$reg),
11784 Assembler::ASR,
11785 $src3$$constant & 0x3f);
11786 %}
11787
11788 ins_pipe(ialu_reg_reg_shift);
11789 %}
11790
11791 // This pattern is automatically generated from aarch64_ad.m4
11792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11793 // val ^ (-1 ^ (val ror shift)) ==> eonw
11794 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11795 iRegIorL2I src1, iRegIorL2I src2,
11796 immI src3, immI_M1 src4) %{
11797 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11798 ins_cost(1.9 * INSN_COST);
11799 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11800
11801 ins_encode %{
11802 __ eonw(as_Register($dst$$reg),
11803 as_Register($src1$$reg),
11804 as_Register($src2$$reg),
11805 Assembler::ROR,
11806 $src3$$constant & 0x1f);
11807 %}
11808
11809 ins_pipe(ialu_reg_reg_shift);
11810 %}
11811
11812 // This pattern is automatically generated from aarch64_ad.m4
11813 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11814 // val ^ (-1 ^ (val ror shift)) ==> eon
11815 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11816 iRegL src1, iRegL src2,
11817 immI src3, immL_M1 src4) %{
11818 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11819 ins_cost(1.9 * INSN_COST);
11820 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11821
11822 ins_encode %{
11823 __ eon(as_Register($dst$$reg),
11824 as_Register($src1$$reg),
11825 as_Register($src2$$reg),
11826 Assembler::ROR,
11827 $src3$$constant & 0x3f);
11828 %}
11829
11830 ins_pipe(ialu_reg_reg_shift);
11831 %}
11832
11833 // This pattern is automatically generated from aarch64_ad.m4
11834 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11835 // val ^ (-1 ^ (val << shift)) ==> eonw
11836 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11837 iRegIorL2I src1, iRegIorL2I src2,
11838 immI src3, immI_M1 src4) %{
11839 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11840 ins_cost(1.9 * INSN_COST);
11841 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11842
11843 ins_encode %{
11844 __ eonw(as_Register($dst$$reg),
11845 as_Register($src1$$reg),
11846 as_Register($src2$$reg),
11847 Assembler::LSL,
11848 $src3$$constant & 0x1f);
11849 %}
11850
11851 ins_pipe(ialu_reg_reg_shift);
11852 %}
11853
11854 // This pattern is automatically generated from aarch64_ad.m4
11855 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11856 // val ^ (-1 ^ (val << shift)) ==> eon
11857 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11858 iRegL src1, iRegL src2,
11859 immI src3, immL_M1 src4) %{
11860 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11861 ins_cost(1.9 * INSN_COST);
11862 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11863
11864 ins_encode %{
11865 __ eon(as_Register($dst$$reg),
11866 as_Register($src1$$reg),
11867 as_Register($src2$$reg),
11868 Assembler::LSL,
11869 $src3$$constant & 0x3f);
11870 %}
11871
11872 ins_pipe(ialu_reg_reg_shift);
11873 %}
11874
11875 // This pattern is automatically generated from aarch64_ad.m4
11876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11877 // val | (-1 ^ (val >>> shift)) ==> ornw
11878 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11879 iRegIorL2I src1, iRegIorL2I src2,
11880 immI src3, immI_M1 src4) %{
11881 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11882 ins_cost(1.9 * INSN_COST);
11883 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11884
11885 ins_encode %{
11886 __ ornw(as_Register($dst$$reg),
11887 as_Register($src1$$reg),
11888 as_Register($src2$$reg),
11889 Assembler::LSR,
11890 $src3$$constant & 0x1f);
11891 %}
11892
11893 ins_pipe(ialu_reg_reg_shift);
11894 %}
11895
11896 // This pattern is automatically generated from aarch64_ad.m4
11897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11898 // val | (-1 ^ (val >>> shift)) ==> orn
11899 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11900 iRegL src1, iRegL src2,
11901 immI src3, immL_M1 src4) %{
11902 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11903 ins_cost(1.9 * INSN_COST);
11904 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11905
11906 ins_encode %{
11907 __ orn(as_Register($dst$$reg),
11908 as_Register($src1$$reg),
11909 as_Register($src2$$reg),
11910 Assembler::LSR,
11911 $src3$$constant & 0x3f);
11912 %}
11913
11914 ins_pipe(ialu_reg_reg_shift);
11915 %}
11916
11917 // This pattern is automatically generated from aarch64_ad.m4
11918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11919 // val | (-1 ^ (val >> shift)) ==> ornw
11920 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11921 iRegIorL2I src1, iRegIorL2I src2,
11922 immI src3, immI_M1 src4) %{
11923 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11924 ins_cost(1.9 * INSN_COST);
11925 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11926
11927 ins_encode %{
11928 __ ornw(as_Register($dst$$reg),
11929 as_Register($src1$$reg),
11930 as_Register($src2$$reg),
11931 Assembler::ASR,
11932 $src3$$constant & 0x1f);
11933 %}
11934
11935 ins_pipe(ialu_reg_reg_shift);
11936 %}
11937
11938 // This pattern is automatically generated from aarch64_ad.m4
11939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11940 // val | (-1 ^ (val >> shift)) ==> orn
11941 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11942 iRegL src1, iRegL src2,
11943 immI src3, immL_M1 src4) %{
11944 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11945 ins_cost(1.9 * INSN_COST);
11946 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11947
11948 ins_encode %{
11949 __ orn(as_Register($dst$$reg),
11950 as_Register($src1$$reg),
11951 as_Register($src2$$reg),
11952 Assembler::ASR,
11953 $src3$$constant & 0x3f);
11954 %}
11955
11956 ins_pipe(ialu_reg_reg_shift);
11957 %}
11958
11959 // This pattern is automatically generated from aarch64_ad.m4
11960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11961 // val | (-1 ^ (val ror shift)) ==> ornw
11962 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11963 iRegIorL2I src1, iRegIorL2I src2,
11964 immI src3, immI_M1 src4) %{
11965 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11966 ins_cost(1.9 * INSN_COST);
11967 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11968
11969 ins_encode %{
11970 __ ornw(as_Register($dst$$reg),
11971 as_Register($src1$$reg),
11972 as_Register($src2$$reg),
11973 Assembler::ROR,
11974 $src3$$constant & 0x1f);
11975 %}
11976
11977 ins_pipe(ialu_reg_reg_shift);
11978 %}
11979
11980 // This pattern is automatically generated from aarch64_ad.m4
11981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11982 // val | (-1 ^ (val ror shift)) ==> orn
11983 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11984 iRegL src1, iRegL src2,
11985 immI src3, immL_M1 src4) %{
11986 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11987 ins_cost(1.9 * INSN_COST);
11988 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11989
11990 ins_encode %{
11991 __ orn(as_Register($dst$$reg),
11992 as_Register($src1$$reg),
11993 as_Register($src2$$reg),
11994 Assembler::ROR,
11995 $src3$$constant & 0x3f);
11996 %}
11997
11998 ins_pipe(ialu_reg_reg_shift);
11999 %}
12000
12001 // This pattern is automatically generated from aarch64_ad.m4
12002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12003 // val | (-1 ^ (val << shift)) ==> ornw
12004 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12005 iRegIorL2I src1, iRegIorL2I src2,
12006 immI src3, immI_M1 src4) %{
12007 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12008 ins_cost(1.9 * INSN_COST);
12009 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
12010
12011 ins_encode %{
12012 __ ornw(as_Register($dst$$reg),
12013 as_Register($src1$$reg),
12014 as_Register($src2$$reg),
12015 Assembler::LSL,
12016 $src3$$constant & 0x1f);
12017 %}
12018
12019 ins_pipe(ialu_reg_reg_shift);
12020 %}
12021
12022 // This pattern is automatically generated from aarch64_ad.m4
12023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12024 // val | (-1 ^ (val << shift)) ==> orn
12025 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12026 iRegL src1, iRegL src2,
12027 immI src3, immL_M1 src4) %{
12028 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12029 ins_cost(1.9 * INSN_COST);
12030 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
12031
12032 ins_encode %{
12033 __ orn(as_Register($dst$$reg),
12034 as_Register($src1$$reg),
12035 as_Register($src2$$reg),
12036 Assembler::LSL,
12037 $src3$$constant & 0x3f);
12038 %}
12039
12040 ins_pipe(ialu_reg_reg_shift);
12041 %}
12042
12043 // This pattern is automatically generated from aarch64_ad.m4
12044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12045 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12046 iRegIorL2I src1, iRegIorL2I src2,
12047 immI src3) %{
12048 match(Set dst (AndI src1 (URShiftI src2 src3)));
12049
12050 ins_cost(1.9 * INSN_COST);
12051 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
12052
12053 ins_encode %{
12054 __ andw(as_Register($dst$$reg),
12055 as_Register($src1$$reg),
12056 as_Register($src2$$reg),
12057 Assembler::LSR,
12058 $src3$$constant & 0x1f);
12059 %}
12060
12061 ins_pipe(ialu_reg_reg_shift);
12062 %}
12063
12064 // This pattern is automatically generated from aarch64_ad.m4
12065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12066 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12067 iRegL src1, iRegL src2,
12068 immI src3) %{
12069 match(Set dst (AndL src1 (URShiftL src2 src3)));
12070
12071 ins_cost(1.9 * INSN_COST);
12072 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
12073
12074 ins_encode %{
12075 __ andr(as_Register($dst$$reg),
12076 as_Register($src1$$reg),
12077 as_Register($src2$$reg),
12078 Assembler::LSR,
12079 $src3$$constant & 0x3f);
12080 %}
12081
12082 ins_pipe(ialu_reg_reg_shift);
12083 %}
12084
12085 // This pattern is automatically generated from aarch64_ad.m4
12086 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12087 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12088 iRegIorL2I src1, iRegIorL2I src2,
12089 immI src3) %{
12090 match(Set dst (AndI src1 (RShiftI src2 src3)));
12091
12092 ins_cost(1.9 * INSN_COST);
12093 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
12094
12095 ins_encode %{
12096 __ andw(as_Register($dst$$reg),
12097 as_Register($src1$$reg),
12098 as_Register($src2$$reg),
12099 Assembler::ASR,
12100 $src3$$constant & 0x1f);
12101 %}
12102
12103 ins_pipe(ialu_reg_reg_shift);
12104 %}
12105
12106 // This pattern is automatically generated from aarch64_ad.m4
12107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12108 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12109 iRegL src1, iRegL src2,
12110 immI src3) %{
12111 match(Set dst (AndL src1 (RShiftL src2 src3)));
12112
12113 ins_cost(1.9 * INSN_COST);
12114 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
12115
12116 ins_encode %{
12117 __ andr(as_Register($dst$$reg),
12118 as_Register($src1$$reg),
12119 as_Register($src2$$reg),
12120 Assembler::ASR,
12121 $src3$$constant & 0x3f);
12122 %}
12123
12124 ins_pipe(ialu_reg_reg_shift);
12125 %}
12126
12127 // This pattern is automatically generated from aarch64_ad.m4
12128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12129 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12130 iRegIorL2I src1, iRegIorL2I src2,
12131 immI src3) %{
12132 match(Set dst (AndI src1 (LShiftI src2 src3)));
12133
12134 ins_cost(1.9 * INSN_COST);
12135 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
12136
12137 ins_encode %{
12138 __ andw(as_Register($dst$$reg),
12139 as_Register($src1$$reg),
12140 as_Register($src2$$reg),
12141 Assembler::LSL,
12142 $src3$$constant & 0x1f);
12143 %}
12144
12145 ins_pipe(ialu_reg_reg_shift);
12146 %}
12147
12148 // This pattern is automatically generated from aarch64_ad.m4
12149 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12150 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12151 iRegL src1, iRegL src2,
12152 immI src3) %{
12153 match(Set dst (AndL src1 (LShiftL src2 src3)));
12154
12155 ins_cost(1.9 * INSN_COST);
12156 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
12157
12158 ins_encode %{
12159 __ andr(as_Register($dst$$reg),
12160 as_Register($src1$$reg),
12161 as_Register($src2$$reg),
12162 Assembler::LSL,
12163 $src3$$constant & 0x3f);
12164 %}
12165
12166 ins_pipe(ialu_reg_reg_shift);
12167 %}
12168
12169 // This pattern is automatically generated from aarch64_ad.m4
12170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12171 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12172 iRegIorL2I src1, iRegIorL2I src2,
12173 immI src3) %{
12174 match(Set dst (AndI src1 (RotateRight src2 src3)));
12175
12176 ins_cost(1.9 * INSN_COST);
12177 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
12178
12179 ins_encode %{
12180 __ andw(as_Register($dst$$reg),
12181 as_Register($src1$$reg),
12182 as_Register($src2$$reg),
12183 Assembler::ROR,
12184 $src3$$constant & 0x1f);
12185 %}
12186
12187 ins_pipe(ialu_reg_reg_shift);
12188 %}
12189
12190 // This pattern is automatically generated from aarch64_ad.m4
12191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12192 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12193 iRegL src1, iRegL src2,
12194 immI src3) %{
12195 match(Set dst (AndL src1 (RotateRight src2 src3)));
12196
12197 ins_cost(1.9 * INSN_COST);
12198 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
12199
12200 ins_encode %{
12201 __ andr(as_Register($dst$$reg),
12202 as_Register($src1$$reg),
12203 as_Register($src2$$reg),
12204 Assembler::ROR,
12205 $src3$$constant & 0x3f);
12206 %}
12207
12208 ins_pipe(ialu_reg_reg_shift);
12209 %}
12210
12211 // This pattern is automatically generated from aarch64_ad.m4
12212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12213 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12214 iRegIorL2I src1, iRegIorL2I src2,
12215 immI src3) %{
12216 match(Set dst (XorI src1 (URShiftI src2 src3)));
12217
12218 ins_cost(1.9 * INSN_COST);
12219 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
12220
12221 ins_encode %{
12222 __ eorw(as_Register($dst$$reg),
12223 as_Register($src1$$reg),
12224 as_Register($src2$$reg),
12225 Assembler::LSR,
12226 $src3$$constant & 0x1f);
12227 %}
12228
12229 ins_pipe(ialu_reg_reg_shift);
12230 %}
12231
12232 // This pattern is automatically generated from aarch64_ad.m4
12233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12234 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12235 iRegL src1, iRegL src2,
12236 immI src3) %{
12237 match(Set dst (XorL src1 (URShiftL src2 src3)));
12238
12239 ins_cost(1.9 * INSN_COST);
12240 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
12241
12242 ins_encode %{
12243 __ eor(as_Register($dst$$reg),
12244 as_Register($src1$$reg),
12245 as_Register($src2$$reg),
12246 Assembler::LSR,
12247 $src3$$constant & 0x3f);
12248 %}
12249
12250 ins_pipe(ialu_reg_reg_shift);
12251 %}
12252
12253 // This pattern is automatically generated from aarch64_ad.m4
12254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12255 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12256 iRegIorL2I src1, iRegIorL2I src2,
12257 immI src3) %{
12258 match(Set dst (XorI src1 (RShiftI src2 src3)));
12259
12260 ins_cost(1.9 * INSN_COST);
12261 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
12262
12263 ins_encode %{
12264 __ eorw(as_Register($dst$$reg),
12265 as_Register($src1$$reg),
12266 as_Register($src2$$reg),
12267 Assembler::ASR,
12268 $src3$$constant & 0x1f);
12269 %}
12270
12271 ins_pipe(ialu_reg_reg_shift);
12272 %}
12273
12274 // This pattern is automatically generated from aarch64_ad.m4
12275 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12276 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12277 iRegL src1, iRegL src2,
12278 immI src3) %{
12279 match(Set dst (XorL src1 (RShiftL src2 src3)));
12280
12281 ins_cost(1.9 * INSN_COST);
12282 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
12283
12284 ins_encode %{
12285 __ eor(as_Register($dst$$reg),
12286 as_Register($src1$$reg),
12287 as_Register($src2$$reg),
12288 Assembler::ASR,
12289 $src3$$constant & 0x3f);
12290 %}
12291
12292 ins_pipe(ialu_reg_reg_shift);
12293 %}
12294
12295 // This pattern is automatically generated from aarch64_ad.m4
12296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12297 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12298 iRegIorL2I src1, iRegIorL2I src2,
12299 immI src3) %{
12300 match(Set dst (XorI src1 (LShiftI src2 src3)));
12301
12302 ins_cost(1.9 * INSN_COST);
12303 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
12304
12305 ins_encode %{
12306 __ eorw(as_Register($dst$$reg),
12307 as_Register($src1$$reg),
12308 as_Register($src2$$reg),
12309 Assembler::LSL,
12310 $src3$$constant & 0x1f);
12311 %}
12312
12313 ins_pipe(ialu_reg_reg_shift);
12314 %}
12315
12316 // This pattern is automatically generated from aarch64_ad.m4
12317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12318 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12319 iRegL src1, iRegL src2,
12320 immI src3) %{
12321 match(Set dst (XorL src1 (LShiftL src2 src3)));
12322
12323 ins_cost(1.9 * INSN_COST);
12324 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
12325
12326 ins_encode %{
12327 __ eor(as_Register($dst$$reg),
12328 as_Register($src1$$reg),
12329 as_Register($src2$$reg),
12330 Assembler::LSL,
12331 $src3$$constant & 0x3f);
12332 %}
12333
12334 ins_pipe(ialu_reg_reg_shift);
12335 %}
12336
12337 // This pattern is automatically generated from aarch64_ad.m4
12338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12339 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12340 iRegIorL2I src1, iRegIorL2I src2,
12341 immI src3) %{
12342 match(Set dst (XorI src1 (RotateRight src2 src3)));
12343
12344 ins_cost(1.9 * INSN_COST);
12345 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
12346
12347 ins_encode %{
12348 __ eorw(as_Register($dst$$reg),
12349 as_Register($src1$$reg),
12350 as_Register($src2$$reg),
12351 Assembler::ROR,
12352 $src3$$constant & 0x1f);
12353 %}
12354
12355 ins_pipe(ialu_reg_reg_shift);
12356 %}
12357
12358 // This pattern is automatically generated from aarch64_ad.m4
12359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12360 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12361 iRegL src1, iRegL src2,
12362 immI src3) %{
12363 match(Set dst (XorL src1 (RotateRight src2 src3)));
12364
12365 ins_cost(1.9 * INSN_COST);
12366 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
12367
12368 ins_encode %{
12369 __ eor(as_Register($dst$$reg),
12370 as_Register($src1$$reg),
12371 as_Register($src2$$reg),
12372 Assembler::ROR,
12373 $src3$$constant & 0x3f);
12374 %}
12375
12376 ins_pipe(ialu_reg_reg_shift);
12377 %}
12378
12379 // This pattern is automatically generated from aarch64_ad.m4
12380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12381 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12382 iRegIorL2I src1, iRegIorL2I src2,
12383 immI src3) %{
12384 match(Set dst (OrI src1 (URShiftI src2 src3)));
12385
12386 ins_cost(1.9 * INSN_COST);
12387 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
12388
12389 ins_encode %{
12390 __ orrw(as_Register($dst$$reg),
12391 as_Register($src1$$reg),
12392 as_Register($src2$$reg),
12393 Assembler::LSR,
12394 $src3$$constant & 0x1f);
12395 %}
12396
12397 ins_pipe(ialu_reg_reg_shift);
12398 %}
12399
12400 // This pattern is automatically generated from aarch64_ad.m4
12401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12402 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12403 iRegL src1, iRegL src2,
12404 immI src3) %{
12405 match(Set dst (OrL src1 (URShiftL src2 src3)));
12406
12407 ins_cost(1.9 * INSN_COST);
12408 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12409
12410 ins_encode %{
12411 __ orr(as_Register($dst$$reg),
12412 as_Register($src1$$reg),
12413 as_Register($src2$$reg),
12414 Assembler::LSR,
12415 $src3$$constant & 0x3f);
12416 %}
12417
12418 ins_pipe(ialu_reg_reg_shift);
12419 %}
12420
12421 // This pattern is automatically generated from aarch64_ad.m4
12422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12423 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12424 iRegIorL2I src1, iRegIorL2I src2,
12425 immI src3) %{
12426 match(Set dst (OrI src1 (RShiftI src2 src3)));
12427
12428 ins_cost(1.9 * INSN_COST);
12429 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12430
12431 ins_encode %{
12432 __ orrw(as_Register($dst$$reg),
12433 as_Register($src1$$reg),
12434 as_Register($src2$$reg),
12435 Assembler::ASR,
12436 $src3$$constant & 0x1f);
12437 %}
12438
12439 ins_pipe(ialu_reg_reg_shift);
12440 %}
12441
12442 // This pattern is automatically generated from aarch64_ad.m4
12443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12444 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12445 iRegL src1, iRegL src2,
12446 immI src3) %{
12447 match(Set dst (OrL src1 (RShiftL src2 src3)));
12448
12449 ins_cost(1.9 * INSN_COST);
12450 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12451
12452 ins_encode %{
12453 __ orr(as_Register($dst$$reg),
12454 as_Register($src1$$reg),
12455 as_Register($src2$$reg),
12456 Assembler::ASR,
12457 $src3$$constant & 0x3f);
12458 %}
12459
12460 ins_pipe(ialu_reg_reg_shift);
12461 %}
12462
12463 // This pattern is automatically generated from aarch64_ad.m4
12464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12465 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12466 iRegIorL2I src1, iRegIorL2I src2,
12467 immI src3) %{
12468 match(Set dst (OrI src1 (LShiftI src2 src3)));
12469
12470 ins_cost(1.9 * INSN_COST);
12471 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12472
12473 ins_encode %{
12474 __ orrw(as_Register($dst$$reg),
12475 as_Register($src1$$reg),
12476 as_Register($src2$$reg),
12477 Assembler::LSL,
12478 $src3$$constant & 0x1f);
12479 %}
12480
12481 ins_pipe(ialu_reg_reg_shift);
12482 %}
12483
12484 // This pattern is automatically generated from aarch64_ad.m4
12485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12486 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12487 iRegL src1, iRegL src2,
12488 immI src3) %{
12489 match(Set dst (OrL src1 (LShiftL src2 src3)));
12490
12491 ins_cost(1.9 * INSN_COST);
12492 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12493
12494 ins_encode %{
12495 __ orr(as_Register($dst$$reg),
12496 as_Register($src1$$reg),
12497 as_Register($src2$$reg),
12498 Assembler::LSL,
12499 $src3$$constant & 0x3f);
12500 %}
12501
12502 ins_pipe(ialu_reg_reg_shift);
12503 %}
12504
12505 // This pattern is automatically generated from aarch64_ad.m4
12506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12507 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12508 iRegIorL2I src1, iRegIorL2I src2,
12509 immI src3) %{
12510 match(Set dst (OrI src1 (RotateRight src2 src3)));
12511
12512 ins_cost(1.9 * INSN_COST);
12513 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12514
12515 ins_encode %{
12516 __ orrw(as_Register($dst$$reg),
12517 as_Register($src1$$reg),
12518 as_Register($src2$$reg),
12519 Assembler::ROR,
12520 $src3$$constant & 0x1f);
12521 %}
12522
12523 ins_pipe(ialu_reg_reg_shift);
12524 %}
12525
12526 // This pattern is automatically generated from aarch64_ad.m4
12527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12528 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12529 iRegL src1, iRegL src2,
12530 immI src3) %{
12531 match(Set dst (OrL src1 (RotateRight src2 src3)));
12532
12533 ins_cost(1.9 * INSN_COST);
12534 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12535
12536 ins_encode %{
12537 __ orr(as_Register($dst$$reg),
12538 as_Register($src1$$reg),
12539 as_Register($src2$$reg),
12540 Assembler::ROR,
12541 $src3$$constant & 0x3f);
12542 %}
12543
12544 ins_pipe(ialu_reg_reg_shift);
12545 %}
12546
12547 // This pattern is automatically generated from aarch64_ad.m4
12548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12549 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12550 iRegIorL2I src1, iRegIorL2I src2,
12551 immI src3) %{
12552 match(Set dst (AddI src1 (URShiftI src2 src3)));
12553
12554 ins_cost(1.9 * INSN_COST);
12555 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12556
12557 ins_encode %{
12558 __ addw(as_Register($dst$$reg),
12559 as_Register($src1$$reg),
12560 as_Register($src2$$reg),
12561 Assembler::LSR,
12562 $src3$$constant & 0x1f);
12563 %}
12564
12565 ins_pipe(ialu_reg_reg_shift);
12566 %}
12567
12568 // This pattern is automatically generated from aarch64_ad.m4
12569 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12570 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12571 iRegL src1, iRegL src2,
12572 immI src3) %{
12573 match(Set dst (AddL src1 (URShiftL src2 src3)));
12574
12575 ins_cost(1.9 * INSN_COST);
12576 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12577
12578 ins_encode %{
12579 __ add(as_Register($dst$$reg),
12580 as_Register($src1$$reg),
12581 as_Register($src2$$reg),
12582 Assembler::LSR,
12583 $src3$$constant & 0x3f);
12584 %}
12585
12586 ins_pipe(ialu_reg_reg_shift);
12587 %}
12588
12589 // This pattern is automatically generated from aarch64_ad.m4
12590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12591 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12592 iRegIorL2I src1, iRegIorL2I src2,
12593 immI src3) %{
12594 match(Set dst (AddI src1 (RShiftI src2 src3)));
12595
12596 ins_cost(1.9 * INSN_COST);
12597 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12598
12599 ins_encode %{
12600 __ addw(as_Register($dst$$reg),
12601 as_Register($src1$$reg),
12602 as_Register($src2$$reg),
12603 Assembler::ASR,
12604 $src3$$constant & 0x1f);
12605 %}
12606
12607 ins_pipe(ialu_reg_reg_shift);
12608 %}
12609
12610 // This pattern is automatically generated from aarch64_ad.m4
12611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12612 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12613 iRegL src1, iRegL src2,
12614 immI src3) %{
12615 match(Set dst (AddL src1 (RShiftL src2 src3)));
12616
12617 ins_cost(1.9 * INSN_COST);
12618 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12619
12620 ins_encode %{
12621 __ add(as_Register($dst$$reg),
12622 as_Register($src1$$reg),
12623 as_Register($src2$$reg),
12624 Assembler::ASR,
12625 $src3$$constant & 0x3f);
12626 %}
12627
12628 ins_pipe(ialu_reg_reg_shift);
12629 %}
12630
12631 // This pattern is automatically generated from aarch64_ad.m4
12632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12633 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12634 iRegIorL2I src1, iRegIorL2I src2,
12635 immI src3) %{
12636 match(Set dst (AddI src1 (LShiftI src2 src3)));
12637
12638 ins_cost(1.9 * INSN_COST);
12639 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12640
12641 ins_encode %{
12642 __ addw(as_Register($dst$$reg),
12643 as_Register($src1$$reg),
12644 as_Register($src2$$reg),
12645 Assembler::LSL,
12646 $src3$$constant & 0x1f);
12647 %}
12648
12649 ins_pipe(ialu_reg_reg_shift);
12650 %}
12651
12652 // This pattern is automatically generated from aarch64_ad.m4
12653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12654 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12655 iRegL src1, iRegL src2,
12656 immI src3) %{
12657 match(Set dst (AddL src1 (LShiftL src2 src3)));
12658
12659 ins_cost(1.9 * INSN_COST);
12660 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12661
12662 ins_encode %{
12663 __ add(as_Register($dst$$reg),
12664 as_Register($src1$$reg),
12665 as_Register($src2$$reg),
12666 Assembler::LSL,
12667 $src3$$constant & 0x3f);
12668 %}
12669
12670 ins_pipe(ialu_reg_reg_shift);
12671 %}
12672
12673 // This pattern is automatically generated from aarch64_ad.m4
12674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12675 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12676 iRegIorL2I src1, iRegIorL2I src2,
12677 immI src3) %{
12678 match(Set dst (SubI src1 (URShiftI src2 src3)));
12679
12680 ins_cost(1.9 * INSN_COST);
12681 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12682
12683 ins_encode %{
12684 __ subw(as_Register($dst$$reg),
12685 as_Register($src1$$reg),
12686 as_Register($src2$$reg),
12687 Assembler::LSR,
12688 $src3$$constant & 0x1f);
12689 %}
12690
12691 ins_pipe(ialu_reg_reg_shift);
12692 %}
12693
12694 // This pattern is automatically generated from aarch64_ad.m4
12695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12696 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12697 iRegL src1, iRegL src2,
12698 immI src3) %{
12699 match(Set dst (SubL src1 (URShiftL src2 src3)));
12700
12701 ins_cost(1.9 * INSN_COST);
12702 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12703
12704 ins_encode %{
12705 __ sub(as_Register($dst$$reg),
12706 as_Register($src1$$reg),
12707 as_Register($src2$$reg),
12708 Assembler::LSR,
12709 $src3$$constant & 0x3f);
12710 %}
12711
12712 ins_pipe(ialu_reg_reg_shift);
12713 %}
12714
12715 // This pattern is automatically generated from aarch64_ad.m4
12716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12717 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12718 iRegIorL2I src1, iRegIorL2I src2,
12719 immI src3) %{
12720 match(Set dst (SubI src1 (RShiftI src2 src3)));
12721
12722 ins_cost(1.9 * INSN_COST);
12723 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12724
12725 ins_encode %{
12726 __ subw(as_Register($dst$$reg),
12727 as_Register($src1$$reg),
12728 as_Register($src2$$reg),
12729 Assembler::ASR,
12730 $src3$$constant & 0x1f);
12731 %}
12732
12733 ins_pipe(ialu_reg_reg_shift);
12734 %}
12735
12736 // This pattern is automatically generated from aarch64_ad.m4
12737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12738 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12739 iRegL src1, iRegL src2,
12740 immI src3) %{
12741 match(Set dst (SubL src1 (RShiftL src2 src3)));
12742
12743 ins_cost(1.9 * INSN_COST);
12744 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12745
12746 ins_encode %{
12747 __ sub(as_Register($dst$$reg),
12748 as_Register($src1$$reg),
12749 as_Register($src2$$reg),
12750 Assembler::ASR,
12751 $src3$$constant & 0x3f);
12752 %}
12753
12754 ins_pipe(ialu_reg_reg_shift);
12755 %}
12756
12757 // This pattern is automatically generated from aarch64_ad.m4
12758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12759 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12760 iRegIorL2I src1, iRegIorL2I src2,
12761 immI src3) %{
12762 match(Set dst (SubI src1 (LShiftI src2 src3)));
12763
12764 ins_cost(1.9 * INSN_COST);
12765 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12766
12767 ins_encode %{
12768 __ subw(as_Register($dst$$reg),
12769 as_Register($src1$$reg),
12770 as_Register($src2$$reg),
12771 Assembler::LSL,
12772 $src3$$constant & 0x1f);
12773 %}
12774
12775 ins_pipe(ialu_reg_reg_shift);
12776 %}
12777
12778 // This pattern is automatically generated from aarch64_ad.m4
12779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12780 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12781 iRegL src1, iRegL src2,
12782 immI src3) %{
12783 match(Set dst (SubL src1 (LShiftL src2 src3)));
12784
12785 ins_cost(1.9 * INSN_COST);
12786 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12787
12788 ins_encode %{
12789 __ sub(as_Register($dst$$reg),
12790 as_Register($src1$$reg),
12791 as_Register($src2$$reg),
12792 Assembler::LSL,
12793 $src3$$constant & 0x3f);
12794 %}
12795
12796 ins_pipe(ialu_reg_reg_shift);
12797 %}
12798
12799 // This pattern is automatically generated from aarch64_ad.m4
12800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12801
12802 // Shift Left followed by Shift Right.
12803 // This idiom is used by the compiler for the i2b bytecode etc.
12804 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12805 %{
12806 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12807 ins_cost(INSN_COST * 2);
12808 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12809 ins_encode %{
12810 int lshift = $lshift_count$$constant & 63;
12811 int rshift = $rshift_count$$constant & 63;
12812 int s = 63 - lshift;
12813 int r = (rshift - lshift) & 63;
12814 __ sbfm(as_Register($dst$$reg),
12815 as_Register($src$$reg),
12816 r, s);
12817 %}
12818
12819 ins_pipe(ialu_reg_shift);
12820 %}
12821
12822 // This pattern is automatically generated from aarch64_ad.m4
12823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12824
12825 // Shift Left followed by Shift Right.
12826 // This idiom is used by the compiler for the i2b bytecode etc.
12827 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12828 %{
12829 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12830 ins_cost(INSN_COST * 2);
12831 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12832 ins_encode %{
12833 int lshift = $lshift_count$$constant & 31;
12834 int rshift = $rshift_count$$constant & 31;
12835 int s = 31 - lshift;
12836 int r = (rshift - lshift) & 31;
12837 __ sbfmw(as_Register($dst$$reg),
12838 as_Register($src$$reg),
12839 r, s);
12840 %}
12841
12842 ins_pipe(ialu_reg_shift);
12843 %}
12844
12845 // This pattern is automatically generated from aarch64_ad.m4
12846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12847
12848 // Shift Left followed by Shift Right.
12849 // This idiom is used by the compiler for the i2b bytecode etc.
12850 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12851 %{
12852 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12853 ins_cost(INSN_COST * 2);
12854 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12855 ins_encode %{
12856 int lshift = $lshift_count$$constant & 63;
12857 int rshift = $rshift_count$$constant & 63;
12858 int s = 63 - lshift;
12859 int r = (rshift - lshift) & 63;
12860 __ ubfm(as_Register($dst$$reg),
12861 as_Register($src$$reg),
12862 r, s);
12863 %}
12864
12865 ins_pipe(ialu_reg_shift);
12866 %}
12867
12868 // This pattern is automatically generated from aarch64_ad.m4
12869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12870
12871 // Shift Left followed by Shift Right.
12872 // This idiom is used by the compiler for the i2b bytecode etc.
12873 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12874 %{
12875 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12876 ins_cost(INSN_COST * 2);
12877 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12878 ins_encode %{
12879 int lshift = $lshift_count$$constant & 31;
12880 int rshift = $rshift_count$$constant & 31;
12881 int s = 31 - lshift;
12882 int r = (rshift - lshift) & 31;
12883 __ ubfmw(as_Register($dst$$reg),
12884 as_Register($src$$reg),
12885 r, s);
12886 %}
12887
12888 ins_pipe(ialu_reg_shift);
12889 %}
12890
12891 // Bitfield extract with shift & mask
12892
12893 // This pattern is automatically generated from aarch64_ad.m4
12894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12895 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12896 %{
12897 match(Set dst (AndI (URShiftI src rshift) mask));
12898 // Make sure we are not going to exceed what ubfxw can do.
12899 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12900
12901 ins_cost(INSN_COST);
12902 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12903 ins_encode %{
12904 int rshift = $rshift$$constant & 31;
12905 intptr_t mask = $mask$$constant;
12906 int width = exact_log2(mask+1);
12907 __ ubfxw(as_Register($dst$$reg),
12908 as_Register($src$$reg), rshift, width);
12909 %}
12910 ins_pipe(ialu_reg_shift);
12911 %}
12912
12913 // This pattern is automatically generated from aarch64_ad.m4
12914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12915 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12916 %{
12917 match(Set dst (AndL (URShiftL src rshift) mask));
12918 // Make sure we are not going to exceed what ubfx can do.
12919 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12920
12921 ins_cost(INSN_COST);
12922 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12923 ins_encode %{
12924 int rshift = $rshift$$constant & 63;
12925 intptr_t mask = $mask$$constant;
12926 int width = exact_log2_long(mask+1);
12927 __ ubfx(as_Register($dst$$reg),
12928 as_Register($src$$reg), rshift, width);
12929 %}
12930 ins_pipe(ialu_reg_shift);
12931 %}
12932
12933
12934 // This pattern is automatically generated from aarch64_ad.m4
12935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12936
12937 // We can use ubfx when extending an And with a mask when we know mask
12938 // is positive. We know that because immI_bitmask guarantees it.
12939 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12940 %{
12941 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12942 // Make sure we are not going to exceed what ubfxw can do.
12943 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12944
12945 ins_cost(INSN_COST * 2);
12946 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12947 ins_encode %{
12948 int rshift = $rshift$$constant & 31;
12949 intptr_t mask = $mask$$constant;
12950 int width = exact_log2(mask+1);
12951 __ ubfx(as_Register($dst$$reg),
12952 as_Register($src$$reg), rshift, width);
12953 %}
12954 ins_pipe(ialu_reg_shift);
12955 %}
12956
12957
12958 // This pattern is automatically generated from aarch64_ad.m4
12959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12960
12961 // We can use ubfiz when masking by a positive number and then left shifting the result.
12962 // We know that the mask is positive because immI_bitmask guarantees it.
12963 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12964 %{
12965 match(Set dst (LShiftI (AndI src mask) lshift));
12966 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12967
12968 ins_cost(INSN_COST);
12969 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12970 ins_encode %{
12971 int lshift = $lshift$$constant & 31;
12972 intptr_t mask = $mask$$constant;
12973 int width = exact_log2(mask+1);
12974 __ ubfizw(as_Register($dst$$reg),
12975 as_Register($src$$reg), lshift, width);
12976 %}
12977 ins_pipe(ialu_reg_shift);
12978 %}
12979
12980 // This pattern is automatically generated from aarch64_ad.m4
12981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12982
12983 // We can use ubfiz when masking by a positive number and then left shifting the result.
12984 // We know that the mask is positive because immL_bitmask guarantees it.
12985 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12986 %{
12987 match(Set dst (LShiftL (AndL src mask) lshift));
12988 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12989
12990 ins_cost(INSN_COST);
12991 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12992 ins_encode %{
12993 int lshift = $lshift$$constant & 63;
12994 intptr_t mask = $mask$$constant;
12995 int width = exact_log2_long(mask+1);
12996 __ ubfiz(as_Register($dst$$reg),
12997 as_Register($src$$reg), lshift, width);
12998 %}
12999 ins_pipe(ialu_reg_shift);
13000 %}
13001
13002 // This pattern is automatically generated from aarch64_ad.m4
13003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13004
13005 // We can use ubfiz when masking by a positive number and then left shifting the result.
13006 // We know that the mask is positive because immI_bitmask guarantees it.
13007 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13008 %{
13009 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13010 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13011
13012 ins_cost(INSN_COST);
13013 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13014 ins_encode %{
13015 int lshift = $lshift$$constant & 31;
13016 intptr_t mask = $mask$$constant;
13017 int width = exact_log2(mask+1);
13018 __ ubfizw(as_Register($dst$$reg),
13019 as_Register($src$$reg), lshift, width);
13020 %}
13021 ins_pipe(ialu_reg_shift);
13022 %}
13023
13024 // This pattern is automatically generated from aarch64_ad.m4
13025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13026
13027 // We can use ubfiz when masking by a positive number and then left shifting the result.
13028 // We know that the mask is positive because immL_bitmask guarantees it.
13029 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13030 %{
13031 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13032 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13033
13034 ins_cost(INSN_COST);
13035 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13036 ins_encode %{
13037 int lshift = $lshift$$constant & 63;
13038 intptr_t mask = $mask$$constant;
13039 int width = exact_log2_long(mask+1);
13040 __ ubfiz(as_Register($dst$$reg),
13041 as_Register($src$$reg), lshift, width);
13042 %}
13043 ins_pipe(ialu_reg_shift);
13044 %}
13045
13046
13047 // This pattern is automatically generated from aarch64_ad.m4
13048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13049
13050 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13051 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13052 %{
13053 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13054 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13055
13056 ins_cost(INSN_COST);
13057 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13058 ins_encode %{
13059 int lshift = $lshift$$constant & 63;
13060 intptr_t mask = $mask$$constant;
13061 int width = exact_log2(mask+1);
13062 __ ubfiz(as_Register($dst$$reg),
13063 as_Register($src$$reg), lshift, width);
13064 %}
13065 ins_pipe(ialu_reg_shift);
13066 %}
13067
13068 // This pattern is automatically generated from aarch64_ad.m4
13069 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13070
13071 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13072 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13073 %{
13074 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13075 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13076
13077 ins_cost(INSN_COST);
13078 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13079 ins_encode %{
13080 int lshift = $lshift$$constant & 31;
13081 intptr_t mask = $mask$$constant;
13082 int width = exact_log2(mask+1);
13083 __ ubfiz(as_Register($dst$$reg),
13084 as_Register($src$$reg), lshift, width);
13085 %}
13086 ins_pipe(ialu_reg_shift);
13087 %}
13088
13089 // This pattern is automatically generated from aarch64_ad.m4
13090 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13091
13092 // Can skip int2long conversions after AND with small bitmask
13093 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13094 %{
13095 match(Set dst (ConvI2L (AndI src msk)));
13096 ins_cost(INSN_COST);
13097 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13098 ins_encode %{
13099 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13100 %}
13101 ins_pipe(ialu_reg_shift);
13102 %}
13103
13104
13105 // Rotations
13106
13107 // This pattern is automatically generated from aarch64_ad.m4
13108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13109 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13110 %{
13111 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13112 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13113
13114 ins_cost(INSN_COST);
13115 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13116
13117 ins_encode %{
13118 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13119 $rshift$$constant & 63);
13120 %}
13121 ins_pipe(ialu_reg_reg_extr);
13122 %}
13123
13124
13125 // This pattern is automatically generated from aarch64_ad.m4
13126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13127 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13128 %{
13129 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13130 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13131
13132 ins_cost(INSN_COST);
13133 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13134
13135 ins_encode %{
13136 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13137 $rshift$$constant & 31);
13138 %}
13139 ins_pipe(ialu_reg_reg_extr);
13140 %}
13141
13142
13143 // This pattern is automatically generated from aarch64_ad.m4
13144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13145 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13146 %{
13147 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13148 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13149
13150 ins_cost(INSN_COST);
13151 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13152
13153 ins_encode %{
13154 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13155 $rshift$$constant & 63);
13156 %}
13157 ins_pipe(ialu_reg_reg_extr);
13158 %}
13159
13160
13161 // This pattern is automatically generated from aarch64_ad.m4
13162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13163 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13164 %{
13165 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13166 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13167
13168 ins_cost(INSN_COST);
13169 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13170
13171 ins_encode %{
13172 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13173 $rshift$$constant & 31);
13174 %}
13175 ins_pipe(ialu_reg_reg_extr);
13176 %}
13177
13178 // This pattern is automatically generated from aarch64_ad.m4
13179 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13180 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13181 %{
13182 match(Set dst (RotateRight src shift));
13183
13184 ins_cost(INSN_COST);
13185 format %{ "ror $dst, $src, $shift" %}
13186
13187 ins_encode %{
13188 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13189 $shift$$constant & 0x1f);
13190 %}
13191 ins_pipe(ialu_reg_reg_vshift);
13192 %}
13193
13194 // This pattern is automatically generated from aarch64_ad.m4
13195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13196 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13197 %{
13198 match(Set dst (RotateRight src shift));
13199
13200 ins_cost(INSN_COST);
13201 format %{ "ror $dst, $src, $shift" %}
13202
13203 ins_encode %{
13204 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13205 $shift$$constant & 0x3f);
13206 %}
13207 ins_pipe(ialu_reg_reg_vshift);
13208 %}
13209
13210 // This pattern is automatically generated from aarch64_ad.m4
13211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13212 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13213 %{
13214 match(Set dst (RotateRight src shift));
13215
13216 ins_cost(INSN_COST);
13217 format %{ "ror $dst, $src, $shift" %}
13218
13219 ins_encode %{
13220 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13221 %}
13222 ins_pipe(ialu_reg_reg_vshift);
13223 %}
13224
13225 // This pattern is automatically generated from aarch64_ad.m4
13226 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13227 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13228 %{
13229 match(Set dst (RotateRight src shift));
13230
13231 ins_cost(INSN_COST);
13232 format %{ "ror $dst, $src, $shift" %}
13233
13234 ins_encode %{
13235 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13236 %}
13237 ins_pipe(ialu_reg_reg_vshift);
13238 %}
13239
13240 // This pattern is automatically generated from aarch64_ad.m4
13241 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13242 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13243 %{
13244 match(Set dst (RotateLeft src shift));
13245
13246 ins_cost(INSN_COST);
13247 format %{ "rol $dst, $src, $shift" %}
13248
13249 ins_encode %{
13250 __ subw(rscratch1, zr, as_Register($shift$$reg));
13251 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13252 %}
13253 ins_pipe(ialu_reg_reg_vshift);
13254 %}
13255
13256 // This pattern is automatically generated from aarch64_ad.m4
13257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13258 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13259 %{
13260 match(Set dst (RotateLeft src shift));
13261
13262 ins_cost(INSN_COST);
13263 format %{ "rol $dst, $src, $shift" %}
13264
13265 ins_encode %{
13266 __ subw(rscratch1, zr, as_Register($shift$$reg));
13267 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13268 %}
13269 ins_pipe(ialu_reg_reg_vshift);
13270 %}
13271
13272
13273 // Add/subtract (extended)
13274
13275 // This pattern is automatically generated from aarch64_ad.m4
13276 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13277 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13278 %{
13279 match(Set dst (AddL src1 (ConvI2L src2)));
13280 ins_cost(INSN_COST);
13281 format %{ "add $dst, $src1, $src2, sxtw" %}
13282
13283 ins_encode %{
13284 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13285 as_Register($src2$$reg), ext::sxtw);
13286 %}
13287 ins_pipe(ialu_reg_reg);
13288 %}
13289
13290 // This pattern is automatically generated from aarch64_ad.m4
13291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13292 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13293 %{
13294 match(Set dst (SubL src1 (ConvI2L src2)));
13295 ins_cost(INSN_COST);
13296 format %{ "sub $dst, $src1, $src2, sxtw" %}
13297
13298 ins_encode %{
13299 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13300 as_Register($src2$$reg), ext::sxtw);
13301 %}
13302 ins_pipe(ialu_reg_reg);
13303 %}
13304
13305 // This pattern is automatically generated from aarch64_ad.m4
13306 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13307 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13308 %{
13309 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13310 ins_cost(INSN_COST);
13311 format %{ "add $dst, $src1, $src2, sxth" %}
13312
13313 ins_encode %{
13314 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13315 as_Register($src2$$reg), ext::sxth);
13316 %}
13317 ins_pipe(ialu_reg_reg);
13318 %}
13319
13320 // This pattern is automatically generated from aarch64_ad.m4
13321 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13322 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13323 %{
13324 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13325 ins_cost(INSN_COST);
13326 format %{ "add $dst, $src1, $src2, sxtb" %}
13327
13328 ins_encode %{
13329 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13330 as_Register($src2$$reg), ext::sxtb);
13331 %}
13332 ins_pipe(ialu_reg_reg);
13333 %}
13334
13335 // This pattern is automatically generated from aarch64_ad.m4
13336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13337 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13338 %{
13339 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13340 ins_cost(INSN_COST);
13341 format %{ "add $dst, $src1, $src2, uxtb" %}
13342
13343 ins_encode %{
13344 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13345 as_Register($src2$$reg), ext::uxtb);
13346 %}
13347 ins_pipe(ialu_reg_reg);
13348 %}
13349
13350 // This pattern is automatically generated from aarch64_ad.m4
13351 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13352 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13353 %{
13354 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13355 ins_cost(INSN_COST);
13356 format %{ "add $dst, $src1, $src2, sxth" %}
13357
13358 ins_encode %{
13359 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13360 as_Register($src2$$reg), ext::sxth);
13361 %}
13362 ins_pipe(ialu_reg_reg);
13363 %}
13364
13365 // This pattern is automatically generated from aarch64_ad.m4
13366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13367 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13368 %{
13369 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13370 ins_cost(INSN_COST);
13371 format %{ "add $dst, $src1, $src2, sxtw" %}
13372
13373 ins_encode %{
13374 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13375 as_Register($src2$$reg), ext::sxtw);
13376 %}
13377 ins_pipe(ialu_reg_reg);
13378 %}
13379
13380 // This pattern is automatically generated from aarch64_ad.m4
13381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13382 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13383 %{
13384 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13385 ins_cost(INSN_COST);
13386 format %{ "add $dst, $src1, $src2, sxtb" %}
13387
13388 ins_encode %{
13389 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13390 as_Register($src2$$reg), ext::sxtb);
13391 %}
13392 ins_pipe(ialu_reg_reg);
13393 %}
13394
13395 // This pattern is automatically generated from aarch64_ad.m4
13396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13397 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13398 %{
13399 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13400 ins_cost(INSN_COST);
13401 format %{ "add $dst, $src1, $src2, uxtb" %}
13402
13403 ins_encode %{
13404 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13405 as_Register($src2$$reg), ext::uxtb);
13406 %}
13407 ins_pipe(ialu_reg_reg);
13408 %}
13409
13410 // This pattern is automatically generated from aarch64_ad.m4
13411 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13412 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13413 %{
13414 match(Set dst (AddI src1 (AndI src2 mask)));
13415 ins_cost(INSN_COST);
13416 format %{ "addw $dst, $src1, $src2, uxtb" %}
13417
13418 ins_encode %{
13419 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13420 as_Register($src2$$reg), ext::uxtb);
13421 %}
13422 ins_pipe(ialu_reg_reg);
13423 %}
13424
13425 // This pattern is automatically generated from aarch64_ad.m4
13426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13427 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13428 %{
13429 match(Set dst (AddI src1 (AndI src2 mask)));
13430 ins_cost(INSN_COST);
13431 format %{ "addw $dst, $src1, $src2, uxth" %}
13432
13433 ins_encode %{
13434 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13435 as_Register($src2$$reg), ext::uxth);
13436 %}
13437 ins_pipe(ialu_reg_reg);
13438 %}
13439
13440 // This pattern is automatically generated from aarch64_ad.m4
13441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13442 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13443 %{
13444 match(Set dst (AddL src1 (AndL src2 mask)));
13445 ins_cost(INSN_COST);
13446 format %{ "add $dst, $src1, $src2, uxtb" %}
13447
13448 ins_encode %{
13449 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13450 as_Register($src2$$reg), ext::uxtb);
13451 %}
13452 ins_pipe(ialu_reg_reg);
13453 %}
13454
13455 // This pattern is automatically generated from aarch64_ad.m4
13456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13457 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13458 %{
13459 match(Set dst (AddL src1 (AndL src2 mask)));
13460 ins_cost(INSN_COST);
13461 format %{ "add $dst, $src1, $src2, uxth" %}
13462
13463 ins_encode %{
13464 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13465 as_Register($src2$$reg), ext::uxth);
13466 %}
13467 ins_pipe(ialu_reg_reg);
13468 %}
13469
13470 // This pattern is automatically generated from aarch64_ad.m4
13471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13472 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13473 %{
13474 match(Set dst (AddL src1 (AndL src2 mask)));
13475 ins_cost(INSN_COST);
13476 format %{ "add $dst, $src1, $src2, uxtw" %}
13477
13478 ins_encode %{
13479 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13480 as_Register($src2$$reg), ext::uxtw);
13481 %}
13482 ins_pipe(ialu_reg_reg);
13483 %}
13484
13485 // This pattern is automatically generated from aarch64_ad.m4
13486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13487 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13488 %{
13489 match(Set dst (SubI src1 (AndI src2 mask)));
13490 ins_cost(INSN_COST);
13491 format %{ "subw $dst, $src1, $src2, uxtb" %}
13492
13493 ins_encode %{
13494 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13495 as_Register($src2$$reg), ext::uxtb);
13496 %}
13497 ins_pipe(ialu_reg_reg);
13498 %}
13499
13500 // This pattern is automatically generated from aarch64_ad.m4
13501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13502 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13503 %{
13504 match(Set dst (SubI src1 (AndI src2 mask)));
13505 ins_cost(INSN_COST);
13506 format %{ "subw $dst, $src1, $src2, uxth" %}
13507
13508 ins_encode %{
13509 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13510 as_Register($src2$$reg), ext::uxth);
13511 %}
13512 ins_pipe(ialu_reg_reg);
13513 %}
13514
13515 // This pattern is automatically generated from aarch64_ad.m4
13516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13517 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13518 %{
13519 match(Set dst (SubL src1 (AndL src2 mask)));
13520 ins_cost(INSN_COST);
13521 format %{ "sub $dst, $src1, $src2, uxtb" %}
13522
13523 ins_encode %{
13524 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13525 as_Register($src2$$reg), ext::uxtb);
13526 %}
13527 ins_pipe(ialu_reg_reg);
13528 %}
13529
13530 // This pattern is automatically generated from aarch64_ad.m4
13531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13532 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13533 %{
13534 match(Set dst (SubL src1 (AndL src2 mask)));
13535 ins_cost(INSN_COST);
13536 format %{ "sub $dst, $src1, $src2, uxth" %}
13537
13538 ins_encode %{
13539 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13540 as_Register($src2$$reg), ext::uxth);
13541 %}
13542 ins_pipe(ialu_reg_reg);
13543 %}
13544
13545 // This pattern is automatically generated from aarch64_ad.m4
13546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13547 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13548 %{
13549 match(Set dst (SubL src1 (AndL src2 mask)));
13550 ins_cost(INSN_COST);
13551 format %{ "sub $dst, $src1, $src2, uxtw" %}
13552
13553 ins_encode %{
13554 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13555 as_Register($src2$$reg), ext::uxtw);
13556 %}
13557 ins_pipe(ialu_reg_reg);
13558 %}
13559
13560
13561 // This pattern is automatically generated from aarch64_ad.m4
13562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13563 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13564 %{
13565 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13566 ins_cost(1.9 * INSN_COST);
13567 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13568
13569 ins_encode %{
13570 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13571 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13572 %}
13573 ins_pipe(ialu_reg_reg_shift);
13574 %}
13575
13576 // This pattern is automatically generated from aarch64_ad.m4
13577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13578 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13579 %{
13580 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13581 ins_cost(1.9 * INSN_COST);
13582 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13583
13584 ins_encode %{
13585 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13586 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13587 %}
13588 ins_pipe(ialu_reg_reg_shift);
13589 %}
13590
13591 // This pattern is automatically generated from aarch64_ad.m4
13592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13593 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13594 %{
13595 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13596 ins_cost(1.9 * INSN_COST);
13597 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13598
13599 ins_encode %{
13600 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13601 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13602 %}
13603 ins_pipe(ialu_reg_reg_shift);
13604 %}
13605
13606 // This pattern is automatically generated from aarch64_ad.m4
13607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13608 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13609 %{
13610 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13611 ins_cost(1.9 * INSN_COST);
13612 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13613
13614 ins_encode %{
13615 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13616 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13617 %}
13618 ins_pipe(ialu_reg_reg_shift);
13619 %}
13620
13621 // This pattern is automatically generated from aarch64_ad.m4
13622 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13623 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13624 %{
13625 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13626 ins_cost(1.9 * INSN_COST);
13627 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13628
13629 ins_encode %{
13630 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13631 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13632 %}
13633 ins_pipe(ialu_reg_reg_shift);
13634 %}
13635
13636 // This pattern is automatically generated from aarch64_ad.m4
13637 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13638 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13639 %{
13640 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13641 ins_cost(1.9 * INSN_COST);
13642 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13643
13644 ins_encode %{
13645 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13646 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13647 %}
13648 ins_pipe(ialu_reg_reg_shift);
13649 %}
13650
13651 // This pattern is automatically generated from aarch64_ad.m4
13652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13653 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13654 %{
13655 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13656 ins_cost(1.9 * INSN_COST);
13657 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13658
13659 ins_encode %{
13660 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13661 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13662 %}
13663 ins_pipe(ialu_reg_reg_shift);
13664 %}
13665
13666 // This pattern is automatically generated from aarch64_ad.m4
13667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13668 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13669 %{
13670 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13671 ins_cost(1.9 * INSN_COST);
13672 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13673
13674 ins_encode %{
13675 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13676 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13677 %}
13678 ins_pipe(ialu_reg_reg_shift);
13679 %}
13680
13681 // This pattern is automatically generated from aarch64_ad.m4
13682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13683 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13684 %{
13685 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13686 ins_cost(1.9 * INSN_COST);
13687 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13688
13689 ins_encode %{
13690 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13691 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13692 %}
13693 ins_pipe(ialu_reg_reg_shift);
13694 %}
13695
13696 // This pattern is automatically generated from aarch64_ad.m4
13697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13698 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13699 %{
13700 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13701 ins_cost(1.9 * INSN_COST);
13702 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13703
13704 ins_encode %{
13705 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13706 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13707 %}
13708 ins_pipe(ialu_reg_reg_shift);
13709 %}
13710
13711 // This pattern is automatically generated from aarch64_ad.m4
13712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13713 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13714 %{
13715 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13716 ins_cost(1.9 * INSN_COST);
13717 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13718
13719 ins_encode %{
13720 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13721 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13722 %}
13723 ins_pipe(ialu_reg_reg_shift);
13724 %}
13725
13726 // This pattern is automatically generated from aarch64_ad.m4
13727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13728 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13729 %{
13730 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13731 ins_cost(1.9 * INSN_COST);
13732 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13733
13734 ins_encode %{
13735 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13736 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13737 %}
13738 ins_pipe(ialu_reg_reg_shift);
13739 %}
13740
13741 // This pattern is automatically generated from aarch64_ad.m4
13742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13743 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13744 %{
13745 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13746 ins_cost(1.9 * INSN_COST);
13747 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13748
13749 ins_encode %{
13750 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13751 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13752 %}
13753 ins_pipe(ialu_reg_reg_shift);
13754 %}
13755
13756 // This pattern is automatically generated from aarch64_ad.m4
13757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13758 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13759 %{
13760 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13761 ins_cost(1.9 * INSN_COST);
13762 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13763
13764 ins_encode %{
13765 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13766 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13767 %}
13768 ins_pipe(ialu_reg_reg_shift);
13769 %}
13770
13771 // This pattern is automatically generated from aarch64_ad.m4
13772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13773 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13774 %{
13775 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13776 ins_cost(1.9 * INSN_COST);
13777 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13778
13779 ins_encode %{
13780 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13781 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13782 %}
13783 ins_pipe(ialu_reg_reg_shift);
13784 %}
13785
13786 // This pattern is automatically generated from aarch64_ad.m4
13787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13788 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13789 %{
13790 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13791 ins_cost(1.9 * INSN_COST);
13792 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13793
13794 ins_encode %{
13795 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13796 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13797 %}
13798 ins_pipe(ialu_reg_reg_shift);
13799 %}
13800
13801 // This pattern is automatically generated from aarch64_ad.m4
13802 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13803 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13804 %{
13805 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13806 ins_cost(1.9 * INSN_COST);
13807 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13808
13809 ins_encode %{
13810 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13811 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13812 %}
13813 ins_pipe(ialu_reg_reg_shift);
13814 %}
13815
13816 // This pattern is automatically generated from aarch64_ad.m4
13817 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13818 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13819 %{
13820 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13821 ins_cost(1.9 * INSN_COST);
13822 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13823
13824 ins_encode %{
13825 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13826 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13827 %}
13828 ins_pipe(ialu_reg_reg_shift);
13829 %}
13830
13831 // This pattern is automatically generated from aarch64_ad.m4
13832 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13833 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13834 %{
13835 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13836 ins_cost(1.9 * INSN_COST);
13837 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13838
13839 ins_encode %{
13840 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13841 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13842 %}
13843 ins_pipe(ialu_reg_reg_shift);
13844 %}
13845
13846 // This pattern is automatically generated from aarch64_ad.m4
13847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13848 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13849 %{
13850 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13851 ins_cost(1.9 * INSN_COST);
13852 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13853
13854 ins_encode %{
13855 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13856 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13857 %}
13858 ins_pipe(ialu_reg_reg_shift);
13859 %}
13860
13861 // This pattern is automatically generated from aarch64_ad.m4
13862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13863 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13864 %{
13865 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13866 ins_cost(1.9 * INSN_COST);
13867 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13868
13869 ins_encode %{
13870 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13871 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13872 %}
13873 ins_pipe(ialu_reg_reg_shift);
13874 %}
13875
13876 // This pattern is automatically generated from aarch64_ad.m4
13877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13878 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13879 %{
13880 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13881 ins_cost(1.9 * INSN_COST);
13882 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13883
13884 ins_encode %{
13885 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13886 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13887 %}
13888 ins_pipe(ialu_reg_reg_shift);
13889 %}
13890
13891 // This pattern is automatically generated from aarch64_ad.m4
13892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13893 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13894 %{
13895 effect(DEF dst, USE src1, USE src2, USE cr);
13896 ins_cost(INSN_COST * 2);
13897 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13898
13899 ins_encode %{
13900 __ cselw($dst$$Register,
13901 $src1$$Register,
13902 $src2$$Register,
13903 Assembler::LT);
13904 %}
13905 ins_pipe(icond_reg_reg);
13906 %}
13907
13908 // This pattern is automatically generated from aarch64_ad.m4
13909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13910 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13911 %{
13912 effect(DEF dst, USE src1, USE src2, USE cr);
13913 ins_cost(INSN_COST * 2);
13914 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13915
13916 ins_encode %{
13917 __ cselw($dst$$Register,
13918 $src1$$Register,
13919 $src2$$Register,
13920 Assembler::GT);
13921 %}
13922 ins_pipe(icond_reg_reg);
13923 %}
13924
13925 // This pattern is automatically generated from aarch64_ad.m4
13926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13927 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13928 %{
13929 effect(DEF dst, USE src1, USE cr);
13930 ins_cost(INSN_COST * 2);
13931 format %{ "cselw $dst, $src1, zr lt\t" %}
13932
13933 ins_encode %{
13934 __ cselw($dst$$Register,
13935 $src1$$Register,
13936 zr,
13937 Assembler::LT);
13938 %}
13939 ins_pipe(icond_reg);
13940 %}
13941
13942 // This pattern is automatically generated from aarch64_ad.m4
13943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13944 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13945 %{
13946 effect(DEF dst, USE src1, USE cr);
13947 ins_cost(INSN_COST * 2);
13948 format %{ "cselw $dst, $src1, zr gt\t" %}
13949
13950 ins_encode %{
13951 __ cselw($dst$$Register,
13952 $src1$$Register,
13953 zr,
13954 Assembler::GT);
13955 %}
13956 ins_pipe(icond_reg);
13957 %}
13958
13959 // This pattern is automatically generated from aarch64_ad.m4
13960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13961 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13962 %{
13963 effect(DEF dst, USE src1, USE cr);
13964 ins_cost(INSN_COST * 2);
13965 format %{ "csincw $dst, $src1, zr le\t" %}
13966
13967 ins_encode %{
13968 __ csincw($dst$$Register,
13969 $src1$$Register,
13970 zr,
13971 Assembler::LE);
13972 %}
13973 ins_pipe(icond_reg);
13974 %}
13975
13976 // This pattern is automatically generated from aarch64_ad.m4
13977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13978 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13979 %{
13980 effect(DEF dst, USE src1, USE cr);
13981 ins_cost(INSN_COST * 2);
13982 format %{ "csincw $dst, $src1, zr gt\t" %}
13983
13984 ins_encode %{
13985 __ csincw($dst$$Register,
13986 $src1$$Register,
13987 zr,
13988 Assembler::GT);
13989 %}
13990 ins_pipe(icond_reg);
13991 %}
13992
13993 // This pattern is automatically generated from aarch64_ad.m4
13994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13995 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13996 %{
13997 effect(DEF dst, USE src1, USE cr);
13998 ins_cost(INSN_COST * 2);
13999 format %{ "csinvw $dst, $src1, zr lt\t" %}
14000
14001 ins_encode %{
14002 __ csinvw($dst$$Register,
14003 $src1$$Register,
14004 zr,
14005 Assembler::LT);
14006 %}
14007 ins_pipe(icond_reg);
14008 %}
14009
14010 // This pattern is automatically generated from aarch64_ad.m4
14011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14012 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14013 %{
14014 effect(DEF dst, USE src1, USE cr);
14015 ins_cost(INSN_COST * 2);
14016 format %{ "csinvw $dst, $src1, zr ge\t" %}
14017
14018 ins_encode %{
14019 __ csinvw($dst$$Register,
14020 $src1$$Register,
14021 zr,
14022 Assembler::GE);
14023 %}
14024 ins_pipe(icond_reg);
14025 %}
14026
14027 // This pattern is automatically generated from aarch64_ad.m4
14028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14029 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14030 %{
14031 match(Set dst (MinI src imm));
14032 ins_cost(INSN_COST * 3);
14033 expand %{
14034 rFlagsReg cr;
14035 compI_reg_imm0(cr, src);
14036 cmovI_reg_imm0_lt(dst, src, cr);
14037 %}
14038 %}
14039
14040 // This pattern is automatically generated from aarch64_ad.m4
14041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14042 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14043 %{
14044 match(Set dst (MinI imm src));
14045 ins_cost(INSN_COST * 3);
14046 expand %{
14047 rFlagsReg cr;
14048 compI_reg_imm0(cr, src);
14049 cmovI_reg_imm0_lt(dst, src, cr);
14050 %}
14051 %}
14052
14053 // This pattern is automatically generated from aarch64_ad.m4
14054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14055 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14056 %{
14057 match(Set dst (MinI src imm));
14058 ins_cost(INSN_COST * 3);
14059 expand %{
14060 rFlagsReg cr;
14061 compI_reg_imm0(cr, src);
14062 cmovI_reg_imm1_le(dst, src, cr);
14063 %}
14064 %}
14065
14066 // This pattern is automatically generated from aarch64_ad.m4
14067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14068 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14069 %{
14070 match(Set dst (MinI imm src));
14071 ins_cost(INSN_COST * 3);
14072 expand %{
14073 rFlagsReg cr;
14074 compI_reg_imm0(cr, src);
14075 cmovI_reg_imm1_le(dst, src, cr);
14076 %}
14077 %}
14078
14079 // This pattern is automatically generated from aarch64_ad.m4
14080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14081 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14082 %{
14083 match(Set dst (MinI src imm));
14084 ins_cost(INSN_COST * 3);
14085 expand %{
14086 rFlagsReg cr;
14087 compI_reg_imm0(cr, src);
14088 cmovI_reg_immM1_lt(dst, src, cr);
14089 %}
14090 %}
14091
14092 // This pattern is automatically generated from aarch64_ad.m4
14093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14094 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14095 %{
14096 match(Set dst (MinI imm src));
14097 ins_cost(INSN_COST * 3);
14098 expand %{
14099 rFlagsReg cr;
14100 compI_reg_imm0(cr, src);
14101 cmovI_reg_immM1_lt(dst, src, cr);
14102 %}
14103 %}
14104
14105 // This pattern is automatically generated from aarch64_ad.m4
14106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14107 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14108 %{
14109 match(Set dst (MaxI src imm));
14110 ins_cost(INSN_COST * 3);
14111 expand %{
14112 rFlagsReg cr;
14113 compI_reg_imm0(cr, src);
14114 cmovI_reg_imm0_gt(dst, src, cr);
14115 %}
14116 %}
14117
14118 // This pattern is automatically generated from aarch64_ad.m4
14119 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14120 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14121 %{
14122 match(Set dst (MaxI imm src));
14123 ins_cost(INSN_COST * 3);
14124 expand %{
14125 rFlagsReg cr;
14126 compI_reg_imm0(cr, src);
14127 cmovI_reg_imm0_gt(dst, src, cr);
14128 %}
14129 %}
14130
14131 // This pattern is automatically generated from aarch64_ad.m4
14132 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14133 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14134 %{
14135 match(Set dst (MaxI src imm));
14136 ins_cost(INSN_COST * 3);
14137 expand %{
14138 rFlagsReg cr;
14139 compI_reg_imm0(cr, src);
14140 cmovI_reg_imm1_gt(dst, src, cr);
14141 %}
14142 %}
14143
14144 // This pattern is automatically generated from aarch64_ad.m4
14145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14146 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14147 %{
14148 match(Set dst (MaxI imm src));
14149 ins_cost(INSN_COST * 3);
14150 expand %{
14151 rFlagsReg cr;
14152 compI_reg_imm0(cr, src);
14153 cmovI_reg_imm1_gt(dst, src, cr);
14154 %}
14155 %}
14156
14157 // This pattern is automatically generated from aarch64_ad.m4
14158 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14159 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14160 %{
14161 match(Set dst (MaxI src imm));
14162 ins_cost(INSN_COST * 3);
14163 expand %{
14164 rFlagsReg cr;
14165 compI_reg_imm0(cr, src);
14166 cmovI_reg_immM1_ge(dst, src, cr);
14167 %}
14168 %}
14169
14170 // This pattern is automatically generated from aarch64_ad.m4
14171 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14172 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14173 %{
14174 match(Set dst (MaxI imm src));
14175 ins_cost(INSN_COST * 3);
14176 expand %{
14177 rFlagsReg cr;
14178 compI_reg_imm0(cr, src);
14179 cmovI_reg_immM1_ge(dst, src, cr);
14180 %}
14181 %}
14182
14183 // This pattern is automatically generated from aarch64_ad.m4
14184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14185 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
14186 %{
14187 match(Set dst (ReverseI src));
14188 ins_cost(INSN_COST);
14189 format %{ "rbitw $dst, $src" %}
14190 ins_encode %{
14191 __ rbitw($dst$$Register, $src$$Register);
14192 %}
14193 ins_pipe(ialu_reg);
14194 %}
14195
14196 // This pattern is automatically generated from aarch64_ad.m4
14197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14198 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
14199 %{
14200 match(Set dst (ReverseL src));
14201 ins_cost(INSN_COST);
14202 format %{ "rbit $dst, $src" %}
14203 ins_encode %{
14204 __ rbit($dst$$Register, $src$$Register);
14205 %}
14206 ins_pipe(ialu_reg);
14207 %}
14208
14209
14210 // END This section of the file is automatically generated. Do not edit --------------
14211
14212
14213 // ============================================================================
14214 // Floating Point Arithmetic Instructions
14215
14216 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14217 match(Set dst (AddF src1 src2));
14218
14219 ins_cost(INSN_COST * 5);
14220 format %{ "fadds $dst, $src1, $src2" %}
14221
14222 ins_encode %{
14223 __ fadds(as_FloatRegister($dst$$reg),
14224 as_FloatRegister($src1$$reg),
14225 as_FloatRegister($src2$$reg));
14226 %}
14227
14228 ins_pipe(fp_dop_reg_reg_s);
14229 %}
14230
14231 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14232 match(Set dst (AddD src1 src2));
14233
14234 ins_cost(INSN_COST * 5);
14235 format %{ "faddd $dst, $src1, $src2" %}
14236
14237 ins_encode %{
14238 __ faddd(as_FloatRegister($dst$$reg),
14239 as_FloatRegister($src1$$reg),
14240 as_FloatRegister($src2$$reg));
14241 %}
14242
14243 ins_pipe(fp_dop_reg_reg_d);
14244 %}
14245
14246 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14247 match(Set dst (SubF src1 src2));
14248
14249 ins_cost(INSN_COST * 5);
14250 format %{ "fsubs $dst, $src1, $src2" %}
14251
14252 ins_encode %{
14253 __ fsubs(as_FloatRegister($dst$$reg),
14254 as_FloatRegister($src1$$reg),
14255 as_FloatRegister($src2$$reg));
14256 %}
14257
14258 ins_pipe(fp_dop_reg_reg_s);
14259 %}
14260
14261 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14262 match(Set dst (SubD src1 src2));
14263
14264 ins_cost(INSN_COST * 5);
14265 format %{ "fsubd $dst, $src1, $src2" %}
14266
14267 ins_encode %{
14268 __ fsubd(as_FloatRegister($dst$$reg),
14269 as_FloatRegister($src1$$reg),
14270 as_FloatRegister($src2$$reg));
14271 %}
14272
14273 ins_pipe(fp_dop_reg_reg_d);
14274 %}
14275
14276 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14277 match(Set dst (MulF src1 src2));
14278
14279 ins_cost(INSN_COST * 6);
14280 format %{ "fmuls $dst, $src1, $src2" %}
14281
14282 ins_encode %{
14283 __ fmuls(as_FloatRegister($dst$$reg),
14284 as_FloatRegister($src1$$reg),
14285 as_FloatRegister($src2$$reg));
14286 %}
14287
14288 ins_pipe(fp_dop_reg_reg_s);
14289 %}
14290
14291 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14292 match(Set dst (MulD src1 src2));
14293
14294 ins_cost(INSN_COST * 6);
14295 format %{ "fmuld $dst, $src1, $src2" %}
14296
14297 ins_encode %{
14298 __ fmuld(as_FloatRegister($dst$$reg),
14299 as_FloatRegister($src1$$reg),
14300 as_FloatRegister($src2$$reg));
14301 %}
14302
14303 ins_pipe(fp_dop_reg_reg_d);
14304 %}
14305
14306 // src1 * src2 + src3
14307 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14308 predicate(UseFMA);
14309 match(Set dst (FmaF src3 (Binary src1 src2)));
14310
14311 format %{ "fmadds $dst, $src1, $src2, $src3" %}
14312
14313 ins_encode %{
14314 __ fmadds(as_FloatRegister($dst$$reg),
14315 as_FloatRegister($src1$$reg),
14316 as_FloatRegister($src2$$reg),
14317 as_FloatRegister($src3$$reg));
14318 %}
14319
14320 ins_pipe(pipe_class_default);
14321 %}
14322
14323 // src1 * src2 + src3
14324 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14325 predicate(UseFMA);
14326 match(Set dst (FmaD src3 (Binary src1 src2)));
14327
14328 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
14329
14330 ins_encode %{
14331 __ fmaddd(as_FloatRegister($dst$$reg),
14332 as_FloatRegister($src1$$reg),
14333 as_FloatRegister($src2$$reg),
14334 as_FloatRegister($src3$$reg));
14335 %}
14336
14337 ins_pipe(pipe_class_default);
14338 %}
14339
14340 // -src1 * src2 + src3
14341 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14342 predicate(UseFMA);
14343 match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14344 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14345
14346 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
14347
14348 ins_encode %{
14349 __ fmsubs(as_FloatRegister($dst$$reg),
14350 as_FloatRegister($src1$$reg),
14351 as_FloatRegister($src2$$reg),
14352 as_FloatRegister($src3$$reg));
14353 %}
14354
14355 ins_pipe(pipe_class_default);
14356 %}
14357
14358 // -src1 * src2 + src3
14359 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14360 predicate(UseFMA);
14361 match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14362 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14363
14364 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
14365
14366 ins_encode %{
14367 __ fmsubd(as_FloatRegister($dst$$reg),
14368 as_FloatRegister($src1$$reg),
14369 as_FloatRegister($src2$$reg),
14370 as_FloatRegister($src3$$reg));
14371 %}
14372
14373 ins_pipe(pipe_class_default);
14374 %}
14375
14376 // -src1 * src2 - src3
14377 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14378 predicate(UseFMA);
14379 match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14380 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14381
14382 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
14383
14384 ins_encode %{
14385 __ fnmadds(as_FloatRegister($dst$$reg),
14386 as_FloatRegister($src1$$reg),
14387 as_FloatRegister($src2$$reg),
14388 as_FloatRegister($src3$$reg));
14389 %}
14390
14391 ins_pipe(pipe_class_default);
14392 %}
14393
14394 // -src1 * src2 - src3
14395 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14396 predicate(UseFMA);
14397 match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14398 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14399
14400 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14401
14402 ins_encode %{
14403 __ fnmaddd(as_FloatRegister($dst$$reg),
14404 as_FloatRegister($src1$$reg),
14405 as_FloatRegister($src2$$reg),
14406 as_FloatRegister($src3$$reg));
14407 %}
14408
14409 ins_pipe(pipe_class_default);
14410 %}
14411
14412 // src1 * src2 - src3
14413 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14414 predicate(UseFMA);
14415 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14416
14417 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14418
14419 ins_encode %{
14420 __ fnmsubs(as_FloatRegister($dst$$reg),
14421 as_FloatRegister($src1$$reg),
14422 as_FloatRegister($src2$$reg),
14423 as_FloatRegister($src3$$reg));
14424 %}
14425
14426 ins_pipe(pipe_class_default);
14427 %}
14428
14429 // src1 * src2 - src3
14430 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14431 predicate(UseFMA);
14432 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14433
14434 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14435
14436 ins_encode %{
14437 // n.b. insn name should be fnmsubd
14438 __ fnmsub(as_FloatRegister($dst$$reg),
14439 as_FloatRegister($src1$$reg),
14440 as_FloatRegister($src2$$reg),
14441 as_FloatRegister($src3$$reg));
14442 %}
14443
14444 ins_pipe(pipe_class_default);
14445 %}
14446
14447
14448 // Math.max(FF)F
14449 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14450 match(Set dst (MaxF src1 src2));
14451
14452 format %{ "fmaxs $dst, $src1, $src2" %}
14453 ins_encode %{
14454 __ fmaxs(as_FloatRegister($dst$$reg),
14455 as_FloatRegister($src1$$reg),
14456 as_FloatRegister($src2$$reg));
14457 %}
14458
14459 ins_pipe(fp_dop_reg_reg_s);
14460 %}
14461
14462 // Math.min(FF)F
14463 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14464 match(Set dst (MinF src1 src2));
14465
14466 format %{ "fmins $dst, $src1, $src2" %}
14467 ins_encode %{
14468 __ fmins(as_FloatRegister($dst$$reg),
14469 as_FloatRegister($src1$$reg),
14470 as_FloatRegister($src2$$reg));
14471 %}
14472
14473 ins_pipe(fp_dop_reg_reg_s);
14474 %}
14475
14476 // Math.max(DD)D
14477 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14478 match(Set dst (MaxD src1 src2));
14479
14480 format %{ "fmaxd $dst, $src1, $src2" %}
14481 ins_encode %{
14482 __ fmaxd(as_FloatRegister($dst$$reg),
14483 as_FloatRegister($src1$$reg),
14484 as_FloatRegister($src2$$reg));
14485 %}
14486
14487 ins_pipe(fp_dop_reg_reg_d);
14488 %}
14489
14490 // Math.min(DD)D
14491 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14492 match(Set dst (MinD src1 src2));
14493
14494 format %{ "fmind $dst, $src1, $src2" %}
14495 ins_encode %{
14496 __ fmind(as_FloatRegister($dst$$reg),
14497 as_FloatRegister($src1$$reg),
14498 as_FloatRegister($src2$$reg));
14499 %}
14500
14501 ins_pipe(fp_dop_reg_reg_d);
14502 %}
14503
14504
14505 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14506 match(Set dst (DivF src1 src2));
14507
14508 ins_cost(INSN_COST * 18);
14509 format %{ "fdivs $dst, $src1, $src2" %}
14510
14511 ins_encode %{
14512 __ fdivs(as_FloatRegister($dst$$reg),
14513 as_FloatRegister($src1$$reg),
14514 as_FloatRegister($src2$$reg));
14515 %}
14516
14517 ins_pipe(fp_div_s);
14518 %}
14519
14520 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14521 match(Set dst (DivD src1 src2));
14522
14523 ins_cost(INSN_COST * 32);
14524 format %{ "fdivd $dst, $src1, $src2" %}
14525
14526 ins_encode %{
14527 __ fdivd(as_FloatRegister($dst$$reg),
14528 as_FloatRegister($src1$$reg),
14529 as_FloatRegister($src2$$reg));
14530 %}
14531
14532 ins_pipe(fp_div_d);
14533 %}
14534
14535 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14536 match(Set dst (NegF src));
14537
14538 ins_cost(INSN_COST * 3);
14539 format %{ "fneg $dst, $src" %}
14540
14541 ins_encode %{
14542 __ fnegs(as_FloatRegister($dst$$reg),
14543 as_FloatRegister($src$$reg));
14544 %}
14545
14546 ins_pipe(fp_uop_s);
14547 %}
14548
14549 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14550 match(Set dst (NegD src));
14551
14552 ins_cost(INSN_COST * 3);
14553 format %{ "fnegd $dst, $src" %}
14554
14555 ins_encode %{
14556 __ fnegd(as_FloatRegister($dst$$reg),
14557 as_FloatRegister($src$$reg));
14558 %}
14559
14560 ins_pipe(fp_uop_d);
14561 %}
14562
14563 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14564 %{
14565 match(Set dst (AbsI src));
14566
14567 effect(KILL cr);
14568 ins_cost(INSN_COST * 2);
14569 format %{ "cmpw $src, zr\n\t"
14570 "cnegw $dst, $src, Assembler::LT\t# int abs"
14571 %}
14572
14573 ins_encode %{
14574 __ cmpw(as_Register($src$$reg), zr);
14575 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14576 %}
14577 ins_pipe(pipe_class_default);
14578 %}
14579
14580 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14581 %{
14582 match(Set dst (AbsL src));
14583
14584 effect(KILL cr);
14585 ins_cost(INSN_COST * 2);
14586 format %{ "cmp $src, zr\n\t"
14587 "cneg $dst, $src, Assembler::LT\t# long abs"
14588 %}
14589
14590 ins_encode %{
14591 __ cmp(as_Register($src$$reg), zr);
14592 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14593 %}
14594 ins_pipe(pipe_class_default);
14595 %}
14596
14597 instruct absF_reg(vRegF dst, vRegF src) %{
14598 match(Set dst (AbsF src));
14599
14600 ins_cost(INSN_COST * 3);
14601 format %{ "fabss $dst, $src" %}
14602 ins_encode %{
14603 __ fabss(as_FloatRegister($dst$$reg),
14604 as_FloatRegister($src$$reg));
14605 %}
14606
14607 ins_pipe(fp_uop_s);
14608 %}
14609
14610 instruct absD_reg(vRegD dst, vRegD src) %{
14611 match(Set dst (AbsD src));
14612
14613 ins_cost(INSN_COST * 3);
14614 format %{ "fabsd $dst, $src" %}
14615 ins_encode %{
14616 __ fabsd(as_FloatRegister($dst$$reg),
14617 as_FloatRegister($src$$reg));
14618 %}
14619
14620 ins_pipe(fp_uop_d);
14621 %}
14622
14623 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14624 match(Set dst (AbsF (SubF src1 src2)));
14625
14626 ins_cost(INSN_COST * 3);
14627 format %{ "fabds $dst, $src1, $src2" %}
14628 ins_encode %{
14629 __ fabds(as_FloatRegister($dst$$reg),
14630 as_FloatRegister($src1$$reg),
14631 as_FloatRegister($src2$$reg));
14632 %}
14633
14634 ins_pipe(fp_uop_s);
14635 %}
14636
14637 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14638 match(Set dst (AbsD (SubD src1 src2)));
14639
14640 ins_cost(INSN_COST * 3);
14641 format %{ "fabdd $dst, $src1, $src2" %}
14642 ins_encode %{
14643 __ fabdd(as_FloatRegister($dst$$reg),
14644 as_FloatRegister($src1$$reg),
14645 as_FloatRegister($src2$$reg));
14646 %}
14647
14648 ins_pipe(fp_uop_d);
14649 %}
14650
14651 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14652 match(Set dst (SqrtD src));
14653
14654 ins_cost(INSN_COST * 50);
14655 format %{ "fsqrtd $dst, $src" %}
14656 ins_encode %{
14657 __ fsqrtd(as_FloatRegister($dst$$reg),
14658 as_FloatRegister($src$$reg));
14659 %}
14660
14661 ins_pipe(fp_div_s);
14662 %}
14663
14664 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14665 match(Set dst (SqrtF src));
14666
14667 ins_cost(INSN_COST * 50);
14668 format %{ "fsqrts $dst, $src" %}
14669 ins_encode %{
14670 __ fsqrts(as_FloatRegister($dst$$reg),
14671 as_FloatRegister($src$$reg));
14672 %}
14673
14674 ins_pipe(fp_div_d);
14675 %}
14676
14677 // Math.rint, floor, ceil
14678 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14679 match(Set dst (RoundDoubleMode src rmode));
14680 format %{ "frint $dst, $src, $rmode" %}
14681 ins_encode %{
14682 switch ($rmode$$constant) {
14683 case RoundDoubleModeNode::rmode_rint:
14684 __ frintnd(as_FloatRegister($dst$$reg),
14685 as_FloatRegister($src$$reg));
14686 break;
14687 case RoundDoubleModeNode::rmode_floor:
14688 __ frintmd(as_FloatRegister($dst$$reg),
14689 as_FloatRegister($src$$reg));
14690 break;
14691 case RoundDoubleModeNode::rmode_ceil:
14692 __ frintpd(as_FloatRegister($dst$$reg),
14693 as_FloatRegister($src$$reg));
14694 break;
14695 }
14696 %}
14697 ins_pipe(fp_uop_d);
14698 %}
14699
14700 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14701 match(Set dst (CopySignD src1 (Binary src2 zero)));
14702 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14703 format %{ "CopySignD $dst $src1 $src2" %}
14704 ins_encode %{
14705 FloatRegister dst = as_FloatRegister($dst$$reg),
14706 src1 = as_FloatRegister($src1$$reg),
14707 src2 = as_FloatRegister($src2$$reg),
14708 zero = as_FloatRegister($zero$$reg);
14709 __ fnegd(dst, zero);
14710 __ bsl(dst, __ T8B, src2, src1);
14711 %}
14712 ins_pipe(fp_uop_d);
14713 %}
14714
14715 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14716 match(Set dst (CopySignF src1 src2));
14717 effect(TEMP_DEF dst, USE src1, USE src2);
14718 format %{ "CopySignF $dst $src1 $src2" %}
14719 ins_encode %{
14720 FloatRegister dst = as_FloatRegister($dst$$reg),
14721 src1 = as_FloatRegister($src1$$reg),
14722 src2 = as_FloatRegister($src2$$reg);
14723 __ movi(dst, __ T2S, 0x80, 24);
14724 __ bsl(dst, __ T8B, src2, src1);
14725 %}
14726 ins_pipe(fp_uop_d);
14727 %}
14728
14729 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14730 match(Set dst (SignumD src (Binary zero one)));
14731 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14732 format %{ "signumD $dst, $src" %}
14733 ins_encode %{
14734 FloatRegister src = as_FloatRegister($src$$reg),
14735 dst = as_FloatRegister($dst$$reg),
14736 zero = as_FloatRegister($zero$$reg),
14737 one = as_FloatRegister($one$$reg);
14738 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14739 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14740 // Bit selection instruction gets bit from "one" for each enabled bit in
14741 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14742 // NaN the whole "src" will be copied because "dst" is zero. For all other
14743 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14744 // from "src", and all other bits are copied from 1.0.
14745 __ bsl(dst, __ T8B, one, src);
14746 %}
14747 ins_pipe(fp_uop_d);
14748 %}
14749
14750 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14751 match(Set dst (SignumF src (Binary zero one)));
14752 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14753 format %{ "signumF $dst, $src" %}
14754 ins_encode %{
14755 FloatRegister src = as_FloatRegister($src$$reg),
14756 dst = as_FloatRegister($dst$$reg),
14757 zero = as_FloatRegister($zero$$reg),
14758 one = as_FloatRegister($one$$reg);
14759 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14760 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14761 // Bit selection instruction gets bit from "one" for each enabled bit in
14762 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14763 // NaN the whole "src" will be copied because "dst" is zero. For all other
14764 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14765 // from "src", and all other bits are copied from 1.0.
14766 __ bsl(dst, __ T8B, one, src);
14767 %}
14768 ins_pipe(fp_uop_d);
14769 %}
14770
14771 instruct onspinwait() %{
14772 match(OnSpinWait);
14773 ins_cost(INSN_COST);
14774
14775 format %{ "onspinwait" %}
14776
14777 ins_encode %{
14778 __ spin_wait();
14779 %}
14780 ins_pipe(pipe_class_empty);
14781 %}
14782
14783 // ============================================================================
14784 // Logical Instructions
14785
14786 // Integer Logical Instructions
14787
14788 // And Instructions
14789
14790
14791 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14792 match(Set dst (AndI src1 src2));
14793
14794 format %{ "andw $dst, $src1, $src2\t# int" %}
14795
14796 ins_cost(INSN_COST);
14797 ins_encode %{
14798 __ andw(as_Register($dst$$reg),
14799 as_Register($src1$$reg),
14800 as_Register($src2$$reg));
14801 %}
14802
14803 ins_pipe(ialu_reg_reg);
14804 %}
14805
14806 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14807 match(Set dst (AndI src1 src2));
14808
14809 format %{ "andsw $dst, $src1, $src2\t# int" %}
14810
14811 ins_cost(INSN_COST);
14812 ins_encode %{
14813 __ andw(as_Register($dst$$reg),
14814 as_Register($src1$$reg),
14815 (uint64_t)($src2$$constant));
14816 %}
14817
14818 ins_pipe(ialu_reg_imm);
14819 %}
14820
14821 // Or Instructions
14822
14823 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14824 match(Set dst (OrI src1 src2));
14825
14826 format %{ "orrw $dst, $src1, $src2\t# int" %}
14827
14828 ins_cost(INSN_COST);
14829 ins_encode %{
14830 __ orrw(as_Register($dst$$reg),
14831 as_Register($src1$$reg),
14832 as_Register($src2$$reg));
14833 %}
14834
14835 ins_pipe(ialu_reg_reg);
14836 %}
14837
14838 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14839 match(Set dst (OrI src1 src2));
14840
14841 format %{ "orrw $dst, $src1, $src2\t# int" %}
14842
14843 ins_cost(INSN_COST);
14844 ins_encode %{
14845 __ orrw(as_Register($dst$$reg),
14846 as_Register($src1$$reg),
14847 (uint64_t)($src2$$constant));
14848 %}
14849
14850 ins_pipe(ialu_reg_imm);
14851 %}
14852
14853 // Xor Instructions
14854
14855 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14856 match(Set dst (XorI src1 src2));
14857
14858 format %{ "eorw $dst, $src1, $src2\t# int" %}
14859
14860 ins_cost(INSN_COST);
14861 ins_encode %{
14862 __ eorw(as_Register($dst$$reg),
14863 as_Register($src1$$reg),
14864 as_Register($src2$$reg));
14865 %}
14866
14867 ins_pipe(ialu_reg_reg);
14868 %}
14869
14870 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14871 match(Set dst (XorI src1 src2));
14872
14873 format %{ "eorw $dst, $src1, $src2\t# int" %}
14874
14875 ins_cost(INSN_COST);
14876 ins_encode %{
14877 __ eorw(as_Register($dst$$reg),
14878 as_Register($src1$$reg),
14879 (uint64_t)($src2$$constant));
14880 %}
14881
14882 ins_pipe(ialu_reg_imm);
14883 %}
14884
14885 // Long Logical Instructions
14886 // TODO
14887
14888 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14889 match(Set dst (AndL src1 src2));
14890
14891 format %{ "and $dst, $src1, $src2\t# int" %}
14892
14893 ins_cost(INSN_COST);
14894 ins_encode %{
14895 __ andr(as_Register($dst$$reg),
14896 as_Register($src1$$reg),
14897 as_Register($src2$$reg));
14898 %}
14899
14900 ins_pipe(ialu_reg_reg);
14901 %}
14902
14903 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14904 match(Set dst (AndL src1 src2));
14905
14906 format %{ "and $dst, $src1, $src2\t# int" %}
14907
14908 ins_cost(INSN_COST);
14909 ins_encode %{
14910 __ andr(as_Register($dst$$reg),
14911 as_Register($src1$$reg),
14912 (uint64_t)($src2$$constant));
14913 %}
14914
14915 ins_pipe(ialu_reg_imm);
14916 %}
14917
14918 // Or Instructions
14919
14920 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14921 match(Set dst (OrL src1 src2));
14922
14923 format %{ "orr $dst, $src1, $src2\t# int" %}
14924
14925 ins_cost(INSN_COST);
14926 ins_encode %{
14927 __ orr(as_Register($dst$$reg),
14928 as_Register($src1$$reg),
14929 as_Register($src2$$reg));
14930 %}
14931
14932 ins_pipe(ialu_reg_reg);
14933 %}
14934
14935 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14936 match(Set dst (OrL src1 src2));
14937
14938 format %{ "orr $dst, $src1, $src2\t# int" %}
14939
14940 ins_cost(INSN_COST);
14941 ins_encode %{
14942 __ orr(as_Register($dst$$reg),
14943 as_Register($src1$$reg),
14944 (uint64_t)($src2$$constant));
14945 %}
14946
14947 ins_pipe(ialu_reg_imm);
14948 %}
14949
14950 // Xor Instructions
14951
14952 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14953 match(Set dst (XorL src1 src2));
14954
14955 format %{ "eor $dst, $src1, $src2\t# int" %}
14956
14957 ins_cost(INSN_COST);
14958 ins_encode %{
14959 __ eor(as_Register($dst$$reg),
14960 as_Register($src1$$reg),
14961 as_Register($src2$$reg));
14962 %}
14963
14964 ins_pipe(ialu_reg_reg);
14965 %}
14966
14967 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14968 match(Set dst (XorL src1 src2));
14969
14970 ins_cost(INSN_COST);
14971 format %{ "eor $dst, $src1, $src2\t# int" %}
14972
14973 ins_encode %{
14974 __ eor(as_Register($dst$$reg),
14975 as_Register($src1$$reg),
14976 (uint64_t)($src2$$constant));
14977 %}
14978
14979 ins_pipe(ialu_reg_imm);
14980 %}
14981
14982 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14983 %{
14984 match(Set dst (ConvI2L src));
14985
14986 ins_cost(INSN_COST);
14987 format %{ "sxtw $dst, $src\t# i2l" %}
14988 ins_encode %{
14989 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14990 %}
14991 ins_pipe(ialu_reg_shift);
14992 %}
14993
14994 // this pattern occurs in bigmath arithmetic
14995 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14996 %{
14997 match(Set dst (AndL (ConvI2L src) mask));
14998
14999 ins_cost(INSN_COST);
15000 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
15001 ins_encode %{
15002 __ ubfm($dst$$Register, $src$$Register, 0, 31);
15003 %}
15004
15005 ins_pipe(ialu_reg_shift);
15006 %}
15007
15008 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
15009 match(Set dst (ConvL2I src));
15010
15011 ins_cost(INSN_COST);
15012 format %{ "movw $dst, $src \t// l2i" %}
15013
15014 ins_encode %{
15015 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
15016 %}
15017
15018 ins_pipe(ialu_reg);
15019 %}
15020
15021 instruct convD2F_reg(vRegF dst, vRegD src) %{
15022 match(Set dst (ConvD2F src));
15023
15024 ins_cost(INSN_COST * 5);
15025 format %{ "fcvtd $dst, $src \t// d2f" %}
15026
15027 ins_encode %{
15028 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15029 %}
15030
15031 ins_pipe(fp_d2f);
15032 %}
15033
15034 instruct convF2D_reg(vRegD dst, vRegF src) %{
15035 match(Set dst (ConvF2D src));
15036
15037 ins_cost(INSN_COST * 5);
15038 format %{ "fcvts $dst, $src \t// f2d" %}
15039
15040 ins_encode %{
15041 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15042 %}
15043
15044 ins_pipe(fp_f2d);
15045 %}
15046
15047 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15048 match(Set dst (ConvF2I src));
15049
15050 ins_cost(INSN_COST * 5);
15051 format %{ "fcvtzsw $dst, $src \t// f2i" %}
15052
15053 ins_encode %{
15054 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15055 %}
15056
15057 ins_pipe(fp_f2i);
15058 %}
15059
15060 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
15061 match(Set dst (ConvF2L src));
15062
15063 ins_cost(INSN_COST * 5);
15064 format %{ "fcvtzs $dst, $src \t// f2l" %}
15065
15066 ins_encode %{
15067 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15068 %}
15069
15070 ins_pipe(fp_f2l);
15071 %}
15072
15073 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
15074 match(Set dst (ConvF2HF src));
15075 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
15076 "smov $dst, $tmp\t# move result from $tmp to $dst"
15077 %}
15078 effect(TEMP tmp);
15079 ins_encode %{
15080 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
15081 %}
15082 ins_pipe(pipe_slow);
15083 %}
15084
15085 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
15086 match(Set dst (ConvHF2F src));
15087 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
15088 "fcvt $dst, $tmp\t# convert half to single precision"
15089 %}
15090 effect(TEMP tmp);
15091 ins_encode %{
15092 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
15093 %}
15094 ins_pipe(pipe_slow);
15095 %}
15096
15097 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
15098 match(Set dst (ConvI2F src));
15099
15100 ins_cost(INSN_COST * 5);
15101 format %{ "scvtfws $dst, $src \t// i2f" %}
15102
15103 ins_encode %{
15104 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15105 %}
15106
15107 ins_pipe(fp_i2f);
15108 %}
15109
15110 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15111 match(Set dst (ConvL2F src));
15112
15113 ins_cost(INSN_COST * 5);
15114 format %{ "scvtfs $dst, $src \t// l2f" %}
15115
15116 ins_encode %{
15117 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15118 %}
15119
15120 ins_pipe(fp_l2f);
15121 %}
15122
15123 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15124 match(Set dst (ConvD2I src));
15125
15126 ins_cost(INSN_COST * 5);
15127 format %{ "fcvtzdw $dst, $src \t// d2i" %}
15128
15129 ins_encode %{
15130 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15131 %}
15132
15133 ins_pipe(fp_d2i);
15134 %}
15135
15136 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15137 match(Set dst (ConvD2L src));
15138
15139 ins_cost(INSN_COST * 5);
15140 format %{ "fcvtzd $dst, $src \t// d2l" %}
15141
15142 ins_encode %{
15143 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15144 %}
15145
15146 ins_pipe(fp_d2l);
15147 %}
15148
15149 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15150 match(Set dst (ConvI2D src));
15151
15152 ins_cost(INSN_COST * 5);
15153 format %{ "scvtfwd $dst, $src \t// i2d" %}
15154
15155 ins_encode %{
15156 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15157 %}
15158
15159 ins_pipe(fp_i2d);
15160 %}
15161
15162 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15163 match(Set dst (ConvL2D src));
15164
15165 ins_cost(INSN_COST * 5);
15166 format %{ "scvtfd $dst, $src \t// l2d" %}
15167
15168 ins_encode %{
15169 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15170 %}
15171
15172 ins_pipe(fp_l2d);
15173 %}
15174
15175 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
15176 %{
15177 match(Set dst (RoundD src));
15178 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15179 format %{ "java_round_double $dst,$src"%}
15180 ins_encode %{
15181 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
15182 as_FloatRegister($ftmp$$reg));
15183 %}
15184 ins_pipe(pipe_slow);
15185 %}
15186
15187 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
15188 %{
15189 match(Set dst (RoundF src));
15190 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15191 format %{ "java_round_float $dst,$src"%}
15192 ins_encode %{
15193 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
15194 as_FloatRegister($ftmp$$reg));
15195 %}
15196 ins_pipe(pipe_slow);
15197 %}
15198
15199 // stack <-> reg and reg <-> reg shuffles with no conversion
15200
15201 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15202
15203 match(Set dst (MoveF2I src));
15204
15205 effect(DEF dst, USE src);
15206
15207 ins_cost(4 * INSN_COST);
15208
15209 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15210
15211 ins_encode %{
15212 __ ldrw($dst$$Register, Address(sp, $src$$disp));
15213 %}
15214
15215 ins_pipe(iload_reg_reg);
15216
15217 %}
15218
15219 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15220
15221 match(Set dst (MoveI2F src));
15222
15223 effect(DEF dst, USE src);
15224
15225 ins_cost(4 * INSN_COST);
15226
15227 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15228
15229 ins_encode %{
15230 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15231 %}
15232
15233 ins_pipe(pipe_class_memory);
15234
15235 %}
15236
15237 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15238
15239 match(Set dst (MoveD2L src));
15240
15241 effect(DEF dst, USE src);
15242
15243 ins_cost(4 * INSN_COST);
15244
15245 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15246
15247 ins_encode %{
15248 __ ldr($dst$$Register, Address(sp, $src$$disp));
15249 %}
15250
15251 ins_pipe(iload_reg_reg);
15252
15253 %}
15254
15255 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15256
15257 match(Set dst (MoveL2D src));
15258
15259 effect(DEF dst, USE src);
15260
15261 ins_cost(4 * INSN_COST);
15262
15263 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15264
15265 ins_encode %{
15266 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15267 %}
15268
15269 ins_pipe(pipe_class_memory);
15270
15271 %}
15272
15273 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15274
15275 match(Set dst (MoveF2I src));
15276
15277 effect(DEF dst, USE src);
15278
15279 ins_cost(INSN_COST);
15280
15281 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15282
15283 ins_encode %{
15284 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15285 %}
15286
15287 ins_pipe(pipe_class_memory);
15288
15289 %}
15290
15291 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15292
15293 match(Set dst (MoveI2F src));
15294
15295 effect(DEF dst, USE src);
15296
15297 ins_cost(INSN_COST);
15298
15299 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15300
15301 ins_encode %{
15302 __ strw($src$$Register, Address(sp, $dst$$disp));
15303 %}
15304
15305 ins_pipe(istore_reg_reg);
15306
15307 %}
15308
15309 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15310
15311 match(Set dst (MoveD2L src));
15312
15313 effect(DEF dst, USE src);
15314
15315 ins_cost(INSN_COST);
15316
15317 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15318
15319 ins_encode %{
15320 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15321 %}
15322
15323 ins_pipe(pipe_class_memory);
15324
15325 %}
15326
15327 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15328
15329 match(Set dst (MoveL2D src));
15330
15331 effect(DEF dst, USE src);
15332
15333 ins_cost(INSN_COST);
15334
15335 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15336
15337 ins_encode %{
15338 __ str($src$$Register, Address(sp, $dst$$disp));
15339 %}
15340
15341 ins_pipe(istore_reg_reg);
15342
15343 %}
15344
15345 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15346
15347 match(Set dst (MoveF2I src));
15348
15349 effect(DEF dst, USE src);
15350
15351 ins_cost(INSN_COST);
15352
15353 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15354
15355 ins_encode %{
15356 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15357 %}
15358
15359 ins_pipe(fp_f2i);
15360
15361 %}
15362
15363 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15364
15365 match(Set dst (MoveI2F src));
15366
15367 effect(DEF dst, USE src);
15368
15369 ins_cost(INSN_COST);
15370
15371 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15372
15373 ins_encode %{
15374 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15375 %}
15376
15377 ins_pipe(fp_i2f);
15378
15379 %}
15380
15381 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15382
15383 match(Set dst (MoveD2L src));
15384
15385 effect(DEF dst, USE src);
15386
15387 ins_cost(INSN_COST);
15388
15389 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15390
15391 ins_encode %{
15392 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15393 %}
15394
15395 ins_pipe(fp_d2l);
15396
15397 %}
15398
15399 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15400
15401 match(Set dst (MoveL2D src));
15402
15403 effect(DEF dst, USE src);
15404
15405 ins_cost(INSN_COST);
15406
15407 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15408
15409 ins_encode %{
15410 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15411 %}
15412
15413 ins_pipe(fp_l2d);
15414
15415 %}
15416
15417 // ============================================================================
15418 // clearing of an array
15419
15420 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15421 %{
15422 match(Set dummy (ClearArray cnt base));
15423 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15424
15425 ins_cost(4 * INSN_COST);
15426 format %{ "ClearArray $cnt, $base" %}
15427
15428 ins_encode %{
15429 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15430 if (tpc == NULL) {
15431 ciEnv::current()->record_failure("CodeCache is full");
15432 return;
15433 }
15434 %}
15435
15436 ins_pipe(pipe_class_memory);
15437 %}
15438
15439 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15440 %{
15441 predicate((uint64_t)n->in(2)->get_long()
15442 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15443 match(Set dummy (ClearArray cnt base));
15444 effect(TEMP temp, USE_KILL base, KILL cr);
15445
15446 ins_cost(4 * INSN_COST);
15447 format %{ "ClearArray $cnt, $base" %}
15448
15449 ins_encode %{
15450 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15451 if (tpc == NULL) {
15452 ciEnv::current()->record_failure("CodeCache is full");
15453 return;
15454 }
15455 %}
15456
15457 ins_pipe(pipe_class_memory);
15458 %}
15459
15460 // ============================================================================
15461 // Overflow Math Instructions
15462
15463 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15464 %{
15465 match(Set cr (OverflowAddI op1 op2));
15466
15467 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15468 ins_cost(INSN_COST);
15469 ins_encode %{
15470 __ cmnw($op1$$Register, $op2$$Register);
15471 %}
15472
15473 ins_pipe(icmp_reg_reg);
15474 %}
15475
15476 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15477 %{
15478 match(Set cr (OverflowAddI op1 op2));
15479
15480 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15481 ins_cost(INSN_COST);
15482 ins_encode %{
15483 __ cmnw($op1$$Register, $op2$$constant);
15484 %}
15485
15486 ins_pipe(icmp_reg_imm);
15487 %}
15488
15489 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15490 %{
15491 match(Set cr (OverflowAddL op1 op2));
15492
15493 format %{ "cmn $op1, $op2\t# overflow check long" %}
15494 ins_cost(INSN_COST);
15495 ins_encode %{
15496 __ cmn($op1$$Register, $op2$$Register);
15497 %}
15498
15499 ins_pipe(icmp_reg_reg);
15500 %}
15501
15502 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15503 %{
15504 match(Set cr (OverflowAddL op1 op2));
15505
15506 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15507 ins_cost(INSN_COST);
15508 ins_encode %{
15509 __ adds(zr, $op1$$Register, $op2$$constant);
15510 %}
15511
15512 ins_pipe(icmp_reg_imm);
15513 %}
15514
15515 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15516 %{
15517 match(Set cr (OverflowSubI op1 op2));
15518
15519 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15520 ins_cost(INSN_COST);
15521 ins_encode %{
15522 __ cmpw($op1$$Register, $op2$$Register);
15523 %}
15524
15525 ins_pipe(icmp_reg_reg);
15526 %}
15527
15528 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15529 %{
15530 match(Set cr (OverflowSubI op1 op2));
15531
15532 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15533 ins_cost(INSN_COST);
15534 ins_encode %{
15535 __ cmpw($op1$$Register, $op2$$constant);
15536 %}
15537
15538 ins_pipe(icmp_reg_imm);
15539 %}
15540
15541 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15542 %{
15543 match(Set cr (OverflowSubL op1 op2));
15544
15545 format %{ "cmp $op1, $op2\t# overflow check long" %}
15546 ins_cost(INSN_COST);
15547 ins_encode %{
15548 __ cmp($op1$$Register, $op2$$Register);
15549 %}
15550
15551 ins_pipe(icmp_reg_reg);
15552 %}
15553
15554 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15555 %{
15556 match(Set cr (OverflowSubL op1 op2));
15557
15558 format %{ "cmp $op1, $op2\t# overflow check long" %}
15559 ins_cost(INSN_COST);
15560 ins_encode %{
15561 __ subs(zr, $op1$$Register, $op2$$constant);
15562 %}
15563
15564 ins_pipe(icmp_reg_imm);
15565 %}
15566
15567 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15568 %{
15569 match(Set cr (OverflowSubI zero op1));
15570
15571 format %{ "cmpw zr, $op1\t# overflow check int" %}
15572 ins_cost(INSN_COST);
15573 ins_encode %{
15574 __ cmpw(zr, $op1$$Register);
15575 %}
15576
15577 ins_pipe(icmp_reg_imm);
15578 %}
15579
15580 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15581 %{
15582 match(Set cr (OverflowSubL zero op1));
15583
15584 format %{ "cmp zr, $op1\t# overflow check long" %}
15585 ins_cost(INSN_COST);
15586 ins_encode %{
15587 __ cmp(zr, $op1$$Register);
15588 %}
15589
15590 ins_pipe(icmp_reg_imm);
15591 %}
15592
15593 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15594 %{
15595 match(Set cr (OverflowMulI op1 op2));
15596
15597 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15598 "cmp rscratch1, rscratch1, sxtw\n\t"
15599 "movw rscratch1, #0x80000000\n\t"
15600 "cselw rscratch1, rscratch1, zr, NE\n\t"
15601 "cmpw rscratch1, #1" %}
15602 ins_cost(5 * INSN_COST);
15603 ins_encode %{
15604 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15605 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15606 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15607 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15608 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15609 %}
15610
15611 ins_pipe(pipe_slow);
15612 %}
15613
15614 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15615 %{
15616 match(If cmp (OverflowMulI op1 op2));
15617 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15618 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15619 effect(USE labl, KILL cr);
15620
15621 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15622 "cmp rscratch1, rscratch1, sxtw\n\t"
15623 "b$cmp $labl" %}
15624 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15625 ins_encode %{
15626 Label* L = $labl$$label;
15627 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15628 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15629 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15630 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15631 %}
15632
15633 ins_pipe(pipe_serial);
15634 %}
15635
15636 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15637 %{
15638 match(Set cr (OverflowMulL op1 op2));
15639
15640 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15641 "smulh rscratch2, $op1, $op2\n\t"
15642 "cmp rscratch2, rscratch1, ASR #63\n\t"
15643 "movw rscratch1, #0x80000000\n\t"
15644 "cselw rscratch1, rscratch1, zr, NE\n\t"
15645 "cmpw rscratch1, #1" %}
15646 ins_cost(6 * INSN_COST);
15647 ins_encode %{
15648 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15649 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15650 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15651 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15652 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15653 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15654 %}
15655
15656 ins_pipe(pipe_slow);
15657 %}
15658
15659 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15660 %{
15661 match(If cmp (OverflowMulL op1 op2));
15662 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15663 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15664 effect(USE labl, KILL cr);
15665
15666 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15667 "smulh rscratch2, $op1, $op2\n\t"
15668 "cmp rscratch2, rscratch1, ASR #63\n\t"
15669 "b$cmp $labl" %}
15670 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15671 ins_encode %{
15672 Label* L = $labl$$label;
15673 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15674 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15675 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15676 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15677 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15678 %}
15679
15680 ins_pipe(pipe_serial);
15681 %}
15682
15683 // ============================================================================
15684 // Compare Instructions
15685
15686 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15687 %{
15688 match(Set cr (CmpI op1 op2));
15689
15690 effect(DEF cr, USE op1, USE op2);
15691
15692 ins_cost(INSN_COST);
15693 format %{ "cmpw $op1, $op2" %}
15694
15695 ins_encode(aarch64_enc_cmpw(op1, op2));
15696
15697 ins_pipe(icmp_reg_reg);
15698 %}
15699
15700 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15701 %{
15702 match(Set cr (CmpI op1 zero));
15703
15704 effect(DEF cr, USE op1);
15705
15706 ins_cost(INSN_COST);
15707 format %{ "cmpw $op1, 0" %}
15708
15709 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15710
15711 ins_pipe(icmp_reg_imm);
15712 %}
15713
15714 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15715 %{
15716 match(Set cr (CmpI op1 op2));
15717
15718 effect(DEF cr, USE op1);
15719
15720 ins_cost(INSN_COST);
15721 format %{ "cmpw $op1, $op2" %}
15722
15723 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15724
15725 ins_pipe(icmp_reg_imm);
15726 %}
15727
15728 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15729 %{
15730 match(Set cr (CmpI op1 op2));
15731
15732 effect(DEF cr, USE op1);
15733
15734 ins_cost(INSN_COST * 2);
15735 format %{ "cmpw $op1, $op2" %}
15736
15737 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15738
15739 ins_pipe(icmp_reg_imm);
15740 %}
15741
15742 // Unsigned compare Instructions; really, same as signed compare
15743 // except it should only be used to feed an If or a CMovI which takes a
15744 // cmpOpU.
15745
15746 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15747 %{
15748 match(Set cr (CmpU op1 op2));
15749
15750 effect(DEF cr, USE op1, USE op2);
15751
15752 ins_cost(INSN_COST);
15753 format %{ "cmpw $op1, $op2\t# unsigned" %}
15754
15755 ins_encode(aarch64_enc_cmpw(op1, op2));
15756
15757 ins_pipe(icmp_reg_reg);
15758 %}
15759
15760 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15761 %{
15762 match(Set cr (CmpU op1 zero));
15763
15764 effect(DEF cr, USE op1);
15765
15766 ins_cost(INSN_COST);
15767 format %{ "cmpw $op1, #0\t# unsigned" %}
15768
15769 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15770
15771 ins_pipe(icmp_reg_imm);
15772 %}
15773
15774 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15775 %{
15776 match(Set cr (CmpU op1 op2));
15777
15778 effect(DEF cr, USE op1);
15779
15780 ins_cost(INSN_COST);
15781 format %{ "cmpw $op1, $op2\t# unsigned" %}
15782
15783 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15784
15785 ins_pipe(icmp_reg_imm);
15786 %}
15787
15788 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15789 %{
15790 match(Set cr (CmpU op1 op2));
15791
15792 effect(DEF cr, USE op1);
15793
15794 ins_cost(INSN_COST * 2);
15795 format %{ "cmpw $op1, $op2\t# unsigned" %}
15796
15797 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15798
15799 ins_pipe(icmp_reg_imm);
15800 %}
15801
15802 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15803 %{
15804 match(Set cr (CmpL op1 op2));
15805
15806 effect(DEF cr, USE op1, USE op2);
15807
15808 ins_cost(INSN_COST);
15809 format %{ "cmp $op1, $op2" %}
15810
15811 ins_encode(aarch64_enc_cmp(op1, op2));
15812
15813 ins_pipe(icmp_reg_reg);
15814 %}
15815
15816 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15817 %{
15818 match(Set cr (CmpL op1 zero));
15819
15820 effect(DEF cr, USE op1);
15821
15822 ins_cost(INSN_COST);
15823 format %{ "tst $op1" %}
15824
15825 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15826
15827 ins_pipe(icmp_reg_imm);
15828 %}
15829
15830 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15831 %{
15832 match(Set cr (CmpL op1 op2));
15833
15834 effect(DEF cr, USE op1);
15835
15836 ins_cost(INSN_COST);
15837 format %{ "cmp $op1, $op2" %}
15838
15839 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15840
15841 ins_pipe(icmp_reg_imm);
15842 %}
15843
15844 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15845 %{
15846 match(Set cr (CmpL op1 op2));
15847
15848 effect(DEF cr, USE op1);
15849
15850 ins_cost(INSN_COST * 2);
15851 format %{ "cmp $op1, $op2" %}
15852
15853 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15854
15855 ins_pipe(icmp_reg_imm);
15856 %}
15857
15858 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15859 %{
15860 match(Set cr (CmpUL op1 op2));
15861
15862 effect(DEF cr, USE op1, USE op2);
15863
15864 ins_cost(INSN_COST);
15865 format %{ "cmp $op1, $op2" %}
15866
15867 ins_encode(aarch64_enc_cmp(op1, op2));
15868
15869 ins_pipe(icmp_reg_reg);
15870 %}
15871
15872 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15873 %{
15874 match(Set cr (CmpUL op1 zero));
15875
15876 effect(DEF cr, USE op1);
15877
15878 ins_cost(INSN_COST);
15879 format %{ "tst $op1" %}
15880
15881 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15882
15883 ins_pipe(icmp_reg_imm);
15884 %}
15885
15886 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15887 %{
15888 match(Set cr (CmpUL op1 op2));
15889
15890 effect(DEF cr, USE op1);
15891
15892 ins_cost(INSN_COST);
15893 format %{ "cmp $op1, $op2" %}
15894
15895 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15896
15897 ins_pipe(icmp_reg_imm);
15898 %}
15899
15900 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15901 %{
15902 match(Set cr (CmpUL op1 op2));
15903
15904 effect(DEF cr, USE op1);
15905
15906 ins_cost(INSN_COST * 2);
15907 format %{ "cmp $op1, $op2" %}
15908
15909 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15910
15911 ins_pipe(icmp_reg_imm);
15912 %}
15913
15914 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15915 %{
15916 match(Set cr (CmpP op1 op2));
15917
15918 effect(DEF cr, USE op1, USE op2);
15919
15920 ins_cost(INSN_COST);
15921 format %{ "cmp $op1, $op2\t // ptr" %}
15922
15923 ins_encode(aarch64_enc_cmpp(op1, op2));
15924
15925 ins_pipe(icmp_reg_reg);
15926 %}
15927
15928 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15929 %{
15930 match(Set cr (CmpN op1 op2));
15931
15932 effect(DEF cr, USE op1, USE op2);
15933
15934 ins_cost(INSN_COST);
15935 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15936
15937 ins_encode(aarch64_enc_cmpn(op1, op2));
15938
15939 ins_pipe(icmp_reg_reg);
15940 %}
15941
15942 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15943 %{
15944 match(Set cr (CmpP op1 zero));
15945
15946 effect(DEF cr, USE op1, USE zero);
15947
15948 ins_cost(INSN_COST);
15949 format %{ "cmp $op1, 0\t // ptr" %}
15950
15951 ins_encode(aarch64_enc_testp(op1));
15952
15953 ins_pipe(icmp_reg_imm);
15954 %}
15955
15956 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15957 %{
15958 match(Set cr (CmpN op1 zero));
15959
15960 effect(DEF cr, USE op1, USE zero);
15961
15962 ins_cost(INSN_COST);
15963 format %{ "cmp $op1, 0\t // compressed ptr" %}
15964
15965 ins_encode(aarch64_enc_testn(op1));
15966
15967 ins_pipe(icmp_reg_imm);
15968 %}
15969
15970 // FP comparisons
15971 //
15972 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15973 // using normal cmpOp. See declaration of rFlagsReg for details.
15974
15975 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15976 %{
15977 match(Set cr (CmpF src1 src2));
15978
15979 ins_cost(3 * INSN_COST);
15980 format %{ "fcmps $src1, $src2" %}
15981
15982 ins_encode %{
15983 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15984 %}
15985
15986 ins_pipe(pipe_class_compare);
15987 %}
15988
15989 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15990 %{
15991 match(Set cr (CmpF src1 src2));
15992
15993 ins_cost(3 * INSN_COST);
15994 format %{ "fcmps $src1, 0.0" %}
15995
15996 ins_encode %{
15997 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15998 %}
15999
16000 ins_pipe(pipe_class_compare);
16001 %}
16002 // FROM HERE
16003
16004 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
16005 %{
16006 match(Set cr (CmpD src1 src2));
16007
16008 ins_cost(3 * INSN_COST);
16009 format %{ "fcmpd $src1, $src2" %}
16010
16011 ins_encode %{
16012 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16013 %}
16014
16015 ins_pipe(pipe_class_compare);
16016 %}
16017
16018 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
16019 %{
16020 match(Set cr (CmpD src1 src2));
16021
16022 ins_cost(3 * INSN_COST);
16023 format %{ "fcmpd $src1, 0.0" %}
16024
16025 ins_encode %{
16026 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
16027 %}
16028
16029 ins_pipe(pipe_class_compare);
16030 %}
16031
16032 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
16033 %{
16034 match(Set dst (CmpF3 src1 src2));
16035 effect(KILL cr);
16036
16037 ins_cost(5 * INSN_COST);
16038 format %{ "fcmps $src1, $src2\n\t"
16039 "csinvw($dst, zr, zr, eq\n\t"
16040 "csnegw($dst, $dst, $dst, lt)"
16041 %}
16042
16043 ins_encode %{
16044 Label done;
16045 FloatRegister s1 = as_FloatRegister($src1$$reg);
16046 FloatRegister s2 = as_FloatRegister($src2$$reg);
16047 Register d = as_Register($dst$$reg);
16048 __ fcmps(s1, s2);
16049 // installs 0 if EQ else -1
16050 __ csinvw(d, zr, zr, Assembler::EQ);
16051 // keeps -1 if less or unordered else installs 1
16052 __ csnegw(d, d, d, Assembler::LT);
16053 __ bind(done);
16054 %}
16055
16056 ins_pipe(pipe_class_default);
16057
16058 %}
16059
16060 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
16061 %{
16062 match(Set dst (CmpD3 src1 src2));
16063 effect(KILL cr);
16064
16065 ins_cost(5 * INSN_COST);
16066 format %{ "fcmpd $src1, $src2\n\t"
16067 "csinvw($dst, zr, zr, eq\n\t"
16068 "csnegw($dst, $dst, $dst, lt)"
16069 %}
16070
16071 ins_encode %{
16072 Label done;
16073 FloatRegister s1 = as_FloatRegister($src1$$reg);
16074 FloatRegister s2 = as_FloatRegister($src2$$reg);
16075 Register d = as_Register($dst$$reg);
16076 __ fcmpd(s1, s2);
16077 // installs 0 if EQ else -1
16078 __ csinvw(d, zr, zr, Assembler::EQ);
16079 // keeps -1 if less or unordered else installs 1
16080 __ csnegw(d, d, d, Assembler::LT);
16081 __ bind(done);
16082 %}
16083 ins_pipe(pipe_class_default);
16084
16085 %}
16086
16087 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
16088 %{
16089 match(Set dst (CmpF3 src1 zero));
16090 effect(KILL cr);
16091
16092 ins_cost(5 * INSN_COST);
16093 format %{ "fcmps $src1, 0.0\n\t"
16094 "csinvw($dst, zr, zr, eq\n\t"
16095 "csnegw($dst, $dst, $dst, lt)"
16096 %}
16097
16098 ins_encode %{
16099 Label done;
16100 FloatRegister s1 = as_FloatRegister($src1$$reg);
16101 Register d = as_Register($dst$$reg);
16102 __ fcmps(s1, 0.0);
16103 // installs 0 if EQ else -1
16104 __ csinvw(d, zr, zr, Assembler::EQ);
16105 // keeps -1 if less or unordered else installs 1
16106 __ csnegw(d, d, d, Assembler::LT);
16107 __ bind(done);
16108 %}
16109
16110 ins_pipe(pipe_class_default);
16111
16112 %}
16113
16114 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16115 %{
16116 match(Set dst (CmpD3 src1 zero));
16117 effect(KILL cr);
16118
16119 ins_cost(5 * INSN_COST);
16120 format %{ "fcmpd $src1, 0.0\n\t"
16121 "csinvw($dst, zr, zr, eq\n\t"
16122 "csnegw($dst, $dst, $dst, lt)"
16123 %}
16124
16125 ins_encode %{
16126 Label done;
16127 FloatRegister s1 = as_FloatRegister($src1$$reg);
16128 Register d = as_Register($dst$$reg);
16129 __ fcmpd(s1, 0.0);
16130 // installs 0 if EQ else -1
16131 __ csinvw(d, zr, zr, Assembler::EQ);
16132 // keeps -1 if less or unordered else installs 1
16133 __ csnegw(d, d, d, Assembler::LT);
16134 __ bind(done);
16135 %}
16136 ins_pipe(pipe_class_default);
16137
16138 %}
16139
16140 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16141 %{
16142 match(Set dst (CmpLTMask p q));
16143 effect(KILL cr);
16144
16145 ins_cost(3 * INSN_COST);
16146
16147 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16148 "csetw $dst, lt\n\t"
16149 "subw $dst, zr, $dst"
16150 %}
16151
16152 ins_encode %{
16153 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16154 __ csetw(as_Register($dst$$reg), Assembler::LT);
16155 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16156 %}
16157
16158 ins_pipe(ialu_reg_reg);
16159 %}
16160
16161 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16162 %{
16163 match(Set dst (CmpLTMask src zero));
16164 effect(KILL cr);
16165
16166 ins_cost(INSN_COST);
16167
16168 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16169
16170 ins_encode %{
16171 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16172 %}
16173
16174 ins_pipe(ialu_reg_shift);
16175 %}
16176
16177 // ============================================================================
16178 // Max and Min
16179
16180 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
16181
16182 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
16183 %{
16184 effect(DEF cr, USE src);
16185 ins_cost(INSN_COST);
16186 format %{ "cmpw $src, 0" %}
16187
16188 ins_encode %{
16189 __ cmpw($src$$Register, 0);
16190 %}
16191 ins_pipe(icmp_reg_imm);
16192 %}
16193
16194 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16195 %{
16196 match(Set dst (MinI src1 src2));
16197 ins_cost(INSN_COST * 3);
16198
16199 expand %{
16200 rFlagsReg cr;
16201 compI_reg_reg(cr, src1, src2);
16202 cmovI_reg_reg_lt(dst, src1, src2, cr);
16203 %}
16204 %}
16205
16206 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16207 %{
16208 match(Set dst (MaxI src1 src2));
16209 ins_cost(INSN_COST * 3);
16210
16211 expand %{
16212 rFlagsReg cr;
16213 compI_reg_reg(cr, src1, src2);
16214 cmovI_reg_reg_gt(dst, src1, src2, cr);
16215 %}
16216 %}
16217
16218
16219 // ============================================================================
16220 // Branch Instructions
16221
16222 // Direct Branch.
16223 instruct branch(label lbl)
16224 %{
16225 match(Goto);
16226
16227 effect(USE lbl);
16228
16229 ins_cost(BRANCH_COST);
16230 format %{ "b $lbl" %}
16231
16232 ins_encode(aarch64_enc_b(lbl));
16233
16234 ins_pipe(pipe_branch);
16235 %}
16236
16237 // Conditional Near Branch
16238 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16239 %{
16240 // Same match rule as `branchConFar'.
16241 match(If cmp cr);
16242
16243 effect(USE lbl);
16244
16245 ins_cost(BRANCH_COST);
16246 // If set to 1 this indicates that the current instruction is a
16247 // short variant of a long branch. This avoids using this
16248 // instruction in first-pass matching. It will then only be used in
16249 // the `Shorten_branches' pass.
16250 // ins_short_branch(1);
16251 format %{ "b$cmp $lbl" %}
16252
16253 ins_encode(aarch64_enc_br_con(cmp, lbl));
16254
16255 ins_pipe(pipe_branch_cond);
16256 %}
16257
16258 // Conditional Near Branch Unsigned
16259 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16260 %{
16261 // Same match rule as `branchConFar'.
16262 match(If cmp cr);
16263
16264 effect(USE lbl);
16265
16266 ins_cost(BRANCH_COST);
16267 // If set to 1 this indicates that the current instruction is a
16268 // short variant of a long branch. This avoids using this
16269 // instruction in first-pass matching. It will then only be used in
16270 // the `Shorten_branches' pass.
16271 // ins_short_branch(1);
16272 format %{ "b$cmp $lbl\t# unsigned" %}
16273
16274 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16275
16276 ins_pipe(pipe_branch_cond);
16277 %}
16278
16279 // Make use of CBZ and CBNZ. These instructions, as well as being
16280 // shorter than (cmp; branch), have the additional benefit of not
16281 // killing the flags.
16282
16283 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16284 match(If cmp (CmpI op1 op2));
16285 effect(USE labl);
16286
16287 ins_cost(BRANCH_COST);
16288 format %{ "cbw$cmp $op1, $labl" %}
16289 ins_encode %{
16290 Label* L = $labl$$label;
16291 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16292 if (cond == Assembler::EQ)
16293 __ cbzw($op1$$Register, *L);
16294 else
16295 __ cbnzw($op1$$Register, *L);
16296 %}
16297 ins_pipe(pipe_cmp_branch);
16298 %}
16299
16300 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16301 match(If cmp (CmpL op1 op2));
16302 effect(USE labl);
16303
16304 ins_cost(BRANCH_COST);
16305 format %{ "cb$cmp $op1, $labl" %}
16306 ins_encode %{
16307 Label* L = $labl$$label;
16308 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16309 if (cond == Assembler::EQ)
16310 __ cbz($op1$$Register, *L);
16311 else
16312 __ cbnz($op1$$Register, *L);
16313 %}
16314 ins_pipe(pipe_cmp_branch);
16315 %}
16316
16317 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16318 match(If cmp (CmpP op1 op2));
16319 effect(USE labl);
16320
16321 ins_cost(BRANCH_COST);
16322 format %{ "cb$cmp $op1, $labl" %}
16323 ins_encode %{
16324 Label* L = $labl$$label;
16325 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16326 if (cond == Assembler::EQ)
16327 __ cbz($op1$$Register, *L);
16328 else
16329 __ cbnz($op1$$Register, *L);
16330 %}
16331 ins_pipe(pipe_cmp_branch);
16332 %}
16333
16334 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16335 match(If cmp (CmpN op1 op2));
16336 effect(USE labl);
16337
16338 ins_cost(BRANCH_COST);
16339 format %{ "cbw$cmp $op1, $labl" %}
16340 ins_encode %{
16341 Label* L = $labl$$label;
16342 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16343 if (cond == Assembler::EQ)
16344 __ cbzw($op1$$Register, *L);
16345 else
16346 __ cbnzw($op1$$Register, *L);
16347 %}
16348 ins_pipe(pipe_cmp_branch);
16349 %}
16350
16351 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16352 match(If cmp (CmpP (DecodeN oop) zero));
16353 effect(USE labl);
16354
16355 ins_cost(BRANCH_COST);
16356 format %{ "cb$cmp $oop, $labl" %}
16357 ins_encode %{
16358 Label* L = $labl$$label;
16359 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16360 if (cond == Assembler::EQ)
16361 __ cbzw($oop$$Register, *L);
16362 else
16363 __ cbnzw($oop$$Register, *L);
16364 %}
16365 ins_pipe(pipe_cmp_branch);
16366 %}
16367
16368 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16369 match(If cmp (CmpU op1 op2));
16370 effect(USE labl);
16371
16372 ins_cost(BRANCH_COST);
16373 format %{ "cbw$cmp $op1, $labl" %}
16374 ins_encode %{
16375 Label* L = $labl$$label;
16376 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16377 if (cond == Assembler::EQ || cond == Assembler::LS)
16378 __ cbzw($op1$$Register, *L);
16379 else
16380 __ cbnzw($op1$$Register, *L);
16381 %}
16382 ins_pipe(pipe_cmp_branch);
16383 %}
16384
16385 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16386 match(If cmp (CmpUL op1 op2));
16387 effect(USE labl);
16388
16389 ins_cost(BRANCH_COST);
16390 format %{ "cb$cmp $op1, $labl" %}
16391 ins_encode %{
16392 Label* L = $labl$$label;
16393 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16394 if (cond == Assembler::EQ || cond == Assembler::LS)
16395 __ cbz($op1$$Register, *L);
16396 else
16397 __ cbnz($op1$$Register, *L);
16398 %}
16399 ins_pipe(pipe_cmp_branch);
16400 %}
16401
16402 // Test bit and Branch
16403
16404 // Patterns for short (< 32KiB) variants
16405 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16406 match(If cmp (CmpL op1 op2));
16407 effect(USE labl);
16408
16409 ins_cost(BRANCH_COST);
16410 format %{ "cb$cmp $op1, $labl # long" %}
16411 ins_encode %{
16412 Label* L = $labl$$label;
16413 Assembler::Condition cond =
16414 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16415 __ tbr(cond, $op1$$Register, 63, *L);
16416 %}
16417 ins_pipe(pipe_cmp_branch);
16418 ins_short_branch(1);
16419 %}
16420
16421 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16422 match(If cmp (CmpI op1 op2));
16423 effect(USE labl);
16424
16425 ins_cost(BRANCH_COST);
16426 format %{ "cb$cmp $op1, $labl # int" %}
16427 ins_encode %{
16428 Label* L = $labl$$label;
16429 Assembler::Condition cond =
16430 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16431 __ tbr(cond, $op1$$Register, 31, *L);
16432 %}
16433 ins_pipe(pipe_cmp_branch);
16434 ins_short_branch(1);
16435 %}
16436
16437 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16438 match(If cmp (CmpL (AndL op1 op2) op3));
16439 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16440 effect(USE labl);
16441
16442 ins_cost(BRANCH_COST);
16443 format %{ "tb$cmp $op1, $op2, $labl" %}
16444 ins_encode %{
16445 Label* L = $labl$$label;
16446 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16447 int bit = exact_log2_long($op2$$constant);
16448 __ tbr(cond, $op1$$Register, bit, *L);
16449 %}
16450 ins_pipe(pipe_cmp_branch);
16451 ins_short_branch(1);
16452 %}
16453
16454 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16455 match(If cmp (CmpI (AndI op1 op2) op3));
16456 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16457 effect(USE labl);
16458
16459 ins_cost(BRANCH_COST);
16460 format %{ "tb$cmp $op1, $op2, $labl" %}
16461 ins_encode %{
16462 Label* L = $labl$$label;
16463 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16464 int bit = exact_log2((juint)$op2$$constant);
16465 __ tbr(cond, $op1$$Register, bit, *L);
16466 %}
16467 ins_pipe(pipe_cmp_branch);
16468 ins_short_branch(1);
16469 %}
16470
16471 // And far variants
16472 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16473 match(If cmp (CmpL op1 op2));
16474 effect(USE labl);
16475
16476 ins_cost(BRANCH_COST);
16477 format %{ "cb$cmp $op1, $labl # long" %}
16478 ins_encode %{
16479 Label* L = $labl$$label;
16480 Assembler::Condition cond =
16481 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16482 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16483 %}
16484 ins_pipe(pipe_cmp_branch);
16485 %}
16486
16487 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16488 match(If cmp (CmpI op1 op2));
16489 effect(USE labl);
16490
16491 ins_cost(BRANCH_COST);
16492 format %{ "cb$cmp $op1, $labl # int" %}
16493 ins_encode %{
16494 Label* L = $labl$$label;
16495 Assembler::Condition cond =
16496 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16497 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16498 %}
16499 ins_pipe(pipe_cmp_branch);
16500 %}
16501
16502 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16503 match(If cmp (CmpL (AndL op1 op2) op3));
16504 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16505 effect(USE labl);
16506
16507 ins_cost(BRANCH_COST);
16508 format %{ "tb$cmp $op1, $op2, $labl" %}
16509 ins_encode %{
16510 Label* L = $labl$$label;
16511 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16512 int bit = exact_log2_long($op2$$constant);
16513 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16514 %}
16515 ins_pipe(pipe_cmp_branch);
16516 %}
16517
16518 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16519 match(If cmp (CmpI (AndI op1 op2) op3));
16520 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16521 effect(USE labl);
16522
16523 ins_cost(BRANCH_COST);
16524 format %{ "tb$cmp $op1, $op2, $labl" %}
16525 ins_encode %{
16526 Label* L = $labl$$label;
16527 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16528 int bit = exact_log2((juint)$op2$$constant);
16529 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16530 %}
16531 ins_pipe(pipe_cmp_branch);
16532 %}
16533
16534 // Test bits
16535
16536 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16537 match(Set cr (CmpL (AndL op1 op2) op3));
16538 predicate(Assembler::operand_valid_for_logical_immediate
16539 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16540
16541 ins_cost(INSN_COST);
16542 format %{ "tst $op1, $op2 # long" %}
16543 ins_encode %{
16544 __ tst($op1$$Register, $op2$$constant);
16545 %}
16546 ins_pipe(ialu_reg_reg);
16547 %}
16548
16549 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16550 match(Set cr (CmpI (AndI op1 op2) op3));
16551 predicate(Assembler::operand_valid_for_logical_immediate
16552 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16553
16554 ins_cost(INSN_COST);
16555 format %{ "tst $op1, $op2 # int" %}
16556 ins_encode %{
16557 __ tstw($op1$$Register, $op2$$constant);
16558 %}
16559 ins_pipe(ialu_reg_reg);
16560 %}
16561
16562 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16563 match(Set cr (CmpL (AndL op1 op2) op3));
16564
16565 ins_cost(INSN_COST);
16566 format %{ "tst $op1, $op2 # long" %}
16567 ins_encode %{
16568 __ tst($op1$$Register, $op2$$Register);
16569 %}
16570 ins_pipe(ialu_reg_reg);
16571 %}
16572
16573 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16574 match(Set cr (CmpI (AndI op1 op2) op3));
16575
16576 ins_cost(INSN_COST);
16577 format %{ "tstw $op1, $op2 # int" %}
16578 ins_encode %{
16579 __ tstw($op1$$Register, $op2$$Register);
16580 %}
16581 ins_pipe(ialu_reg_reg);
16582 %}
16583
16584
16585 // Conditional Far Branch
16586 // Conditional Far Branch Unsigned
16587 // TODO: fixme
16588
16589 // counted loop end branch near
16590 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16591 %{
16592 match(CountedLoopEnd cmp cr);
16593
16594 effect(USE lbl);
16595
16596 ins_cost(BRANCH_COST);
16597 // short variant.
16598 // ins_short_branch(1);
16599 format %{ "b$cmp $lbl \t// counted loop end" %}
16600
16601 ins_encode(aarch64_enc_br_con(cmp, lbl));
16602
16603 ins_pipe(pipe_branch);
16604 %}
16605
16606 // counted loop end branch far
16607 // TODO: fixme
16608
16609 // ============================================================================
16610 // inlined locking and unlocking
16611
16612 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16613 %{
16614 match(Set cr (FastLock object box));
16615 effect(TEMP tmp, TEMP tmp2);
16616
16617 // TODO
16618 // identify correct cost
16619 ins_cost(5 * INSN_COST);
16620 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16621
16622 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16623
16624 ins_pipe(pipe_serial);
16625 %}
16626
16627 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16628 %{
16629 match(Set cr (FastUnlock object box));
16630 effect(TEMP tmp, TEMP tmp2);
16631
16632 ins_cost(5 * INSN_COST);
16633 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16634
16635 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16636
16637 ins_pipe(pipe_serial);
16638 %}
16639
16640
16641 // ============================================================================
16642 // Safepoint Instructions
16643
16644 // TODO
16645 // provide a near and far version of this code
16646
16647 instruct safePoint(rFlagsReg cr, iRegP poll)
16648 %{
16649 match(SafePoint poll);
16650 effect(KILL cr);
16651
16652 format %{
16653 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16654 %}
16655 ins_encode %{
16656 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16657 %}
16658 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16659 %}
16660
16661
16662 // ============================================================================
16663 // Procedure Call/Return Instructions
16664
16665 // Call Java Static Instruction
16666
16667 instruct CallStaticJavaDirect(method meth)
16668 %{
16669 match(CallStaticJava);
16670
16671 effect(USE meth);
16672
16673 ins_cost(CALL_COST);
16674
16675 format %{ "call,static $meth \t// ==> " %}
16676
16677 ins_encode(aarch64_enc_java_static_call(meth),
16678 aarch64_enc_call_epilog);
16679
16680 ins_pipe(pipe_class_call);
16681 %}
16682
16683 // TO HERE
16684
16685 // Call Java Dynamic Instruction
16686 instruct CallDynamicJavaDirect(method meth)
16687 %{
16688 match(CallDynamicJava);
16689
16690 effect(USE meth);
16691
16692 ins_cost(CALL_COST);
16693
16694 format %{ "CALL,dynamic $meth \t// ==> " %}
16695
16696 ins_encode(aarch64_enc_java_dynamic_call(meth),
16697 aarch64_enc_call_epilog);
16698
16699 ins_pipe(pipe_class_call);
16700 %}
16701
16702 // Call Runtime Instruction
16703
16704 instruct CallRuntimeDirect(method meth)
16705 %{
16706 match(CallRuntime);
16707
16708 effect(USE meth);
16709
16710 ins_cost(CALL_COST);
16711
16712 format %{ "CALL, runtime $meth" %}
16713
16714 ins_encode( aarch64_enc_java_to_runtime(meth) );
16715
16716 ins_pipe(pipe_class_call);
16717 %}
16718
16719 // Call Runtime Instruction
16720
16721 instruct CallLeafDirect(method meth)
16722 %{
16723 match(CallLeaf);
16724
16725 effect(USE meth);
16726
16727 ins_cost(CALL_COST);
16728
16729 format %{ "CALL, runtime leaf $meth" %}
16730
16731 ins_encode( aarch64_enc_java_to_runtime(meth) );
16732
16733 ins_pipe(pipe_class_call);
16734 %}
16735
16736 // Call Runtime Instruction
16737
16738 instruct CallLeafNoFPDirect(method meth)
16739 %{
16740 match(CallLeafNoFP);
16741
16742 effect(USE meth);
16743
16744 ins_cost(CALL_COST);
16745
16746 format %{ "CALL, runtime leaf nofp $meth" %}
16747
16748 ins_encode( aarch64_enc_java_to_runtime(meth) );
16749
16750 ins_pipe(pipe_class_call);
16751 %}
16752
16753 // Tail Call; Jump from runtime stub to Java code.
16754 // Also known as an 'interprocedural jump'.
16755 // Target of jump will eventually return to caller.
16756 // TailJump below removes the return address.
16757 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16758 %{
16759 match(TailCall jump_target method_ptr);
16760
16761 ins_cost(CALL_COST);
16762
16763 format %{ "br $jump_target\t# $method_ptr holds method" %}
16764
16765 ins_encode(aarch64_enc_tail_call(jump_target));
16766
16767 ins_pipe(pipe_class_call);
16768 %}
16769
16770 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16771 %{
16772 match(TailJump jump_target ex_oop);
16773
16774 ins_cost(CALL_COST);
16775
16776 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16777
16778 ins_encode(aarch64_enc_tail_jmp(jump_target));
16779
16780 ins_pipe(pipe_class_call);
16781 %}
16782
16783 // Create exception oop: created by stack-crawling runtime code.
16784 // Created exception is now available to this handler, and is setup
16785 // just prior to jumping to this handler. No code emitted.
16786 // TODO check
16787 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16788 instruct CreateException(iRegP_R0 ex_oop)
16789 %{
16790 match(Set ex_oop (CreateEx));
16791
16792 format %{ " -- \t// exception oop; no code emitted" %}
16793
16794 size(0);
16795
16796 ins_encode( /*empty*/ );
16797
16798 ins_pipe(pipe_class_empty);
16799 %}
16800
16801 // Rethrow exception: The exception oop will come in the first
16802 // argument position. Then JUMP (not call) to the rethrow stub code.
16803 instruct RethrowException() %{
16804 match(Rethrow);
16805 ins_cost(CALL_COST);
16806
16807 format %{ "b rethrow_stub" %}
16808
16809 ins_encode( aarch64_enc_rethrow() );
16810
16811 ins_pipe(pipe_class_call);
16812 %}
16813
16814
16815 // Return Instruction
16816 // epilog node loads ret address into lr as part of frame pop
16817 instruct Ret()
16818 %{
16819 match(Return);
16820
16821 format %{ "ret\t// return register" %}
16822
16823 ins_encode( aarch64_enc_ret() );
16824
16825 ins_pipe(pipe_branch);
16826 %}
16827
16828 // Die now.
16829 instruct ShouldNotReachHere() %{
16830 match(Halt);
16831
16832 ins_cost(CALL_COST);
16833 format %{ "ShouldNotReachHere" %}
16834
16835 ins_encode %{
16836 if (is_reachable()) {
16837 __ stop(_halt_reason);
16838 }
16839 %}
16840
16841 ins_pipe(pipe_class_default);
16842 %}
16843
16844 // ============================================================================
16845 // Partial Subtype Check
16846 //
16847 // superklass array for an instance of the superklass. Set a hidden
16848 // internal cache on a hit (cache is checked with exposed code in
16849 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16850 // encoding ALSO sets flags.
16851
16852 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16853 %{
16854 match(Set result (PartialSubtypeCheck sub super));
16855 effect(KILL cr, KILL temp);
16856
16857 ins_cost(1100); // slightly larger than the next version
16858 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16859
16860 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16861
16862 opcode(0x1); // Force zero of result reg on hit
16863
16864 ins_pipe(pipe_class_memory);
16865 %}
16866
16867 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16868 %{
16869 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16870 effect(KILL temp, KILL result);
16871
16872 ins_cost(1100); // slightly larger than the next version
16873 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16874
16875 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16876
16877 opcode(0x0); // Don't zero result reg on hit
16878
16879 ins_pipe(pipe_class_memory);
16880 %}
16881
16882 // Intrisics for String.compareTo()
16883
16884 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16885 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16886 %{
16887 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16888 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16889 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16890
16891 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16892 ins_encode %{
16893 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16894 __ string_compare($str1$$Register, $str2$$Register,
16895 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16896 $tmp1$$Register, $tmp2$$Register,
16897 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16898 %}
16899 ins_pipe(pipe_class_memory);
16900 %}
16901
16902 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16903 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16904 %{
16905 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16906 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16907 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16908
16909 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16910 ins_encode %{
16911 __ string_compare($str1$$Register, $str2$$Register,
16912 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16913 $tmp1$$Register, $tmp2$$Register,
16914 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16915 %}
16916 ins_pipe(pipe_class_memory);
16917 %}
16918
16919 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16920 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16921 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16922 %{
16923 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16924 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16925 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16926 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16927
16928 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16929 ins_encode %{
16930 __ string_compare($str1$$Register, $str2$$Register,
16931 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16932 $tmp1$$Register, $tmp2$$Register,
16933 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16934 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16935 %}
16936 ins_pipe(pipe_class_memory);
16937 %}
16938
16939 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16940 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16941 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16942 %{
16943 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16944 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16945 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16946 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16947
16948 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16949 ins_encode %{
16950 __ string_compare($str1$$Register, $str2$$Register,
16951 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16952 $tmp1$$Register, $tmp2$$Register,
16953 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16954 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16955 %}
16956 ins_pipe(pipe_class_memory);
16957 %}
16958
16959 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16960 // these string_compare variants as NEON register type for convenience so that the prototype of
16961 // string_compare can be shared with all variants.
16962
16963 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16964 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16965 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16966 pRegGov_P1 pgtmp2, rFlagsReg cr)
16967 %{
16968 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16969 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16970 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16971 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16972
16973 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16974 ins_encode %{
16975 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16976 __ string_compare($str1$$Register, $str2$$Register,
16977 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16978 $tmp1$$Register, $tmp2$$Register,
16979 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16980 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16981 StrIntrinsicNode::LL);
16982 %}
16983 ins_pipe(pipe_class_memory);
16984 %}
16985
16986 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16987 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16988 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16989 pRegGov_P1 pgtmp2, rFlagsReg cr)
16990 %{
16991 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16992 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16993 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16994 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16995
16996 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16997 ins_encode %{
16998 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16999 __ string_compare($str1$$Register, $str2$$Register,
17000 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17001 $tmp1$$Register, $tmp2$$Register,
17002 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17003 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17004 StrIntrinsicNode::LU);
17005 %}
17006 ins_pipe(pipe_class_memory);
17007 %}
17008
17009 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17010 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17011 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17012 pRegGov_P1 pgtmp2, rFlagsReg cr)
17013 %{
17014 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
17015 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17016 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17017 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17018
17019 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
17020 ins_encode %{
17021 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17022 __ string_compare($str1$$Register, $str2$$Register,
17023 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17024 $tmp1$$Register, $tmp2$$Register,
17025 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17026 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17027 StrIntrinsicNode::UL);
17028 %}
17029 ins_pipe(pipe_class_memory);
17030 %}
17031
17032 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17033 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17034 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17035 pRegGov_P1 pgtmp2, rFlagsReg cr)
17036 %{
17037 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
17038 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17039 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17040 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17041
17042 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
17043 ins_encode %{
17044 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17045 __ string_compare($str1$$Register, $str2$$Register,
17046 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17047 $tmp1$$Register, $tmp2$$Register,
17048 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17049 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17050 StrIntrinsicNode::UU);
17051 %}
17052 ins_pipe(pipe_class_memory);
17053 %}
17054
17055 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17056 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17057 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17058 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17059 %{
17060 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17061 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17062 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17063 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
17064 TEMP vtmp0, TEMP vtmp1, KILL cr);
17065 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
17066 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17067
17068 ins_encode %{
17069 __ string_indexof($str1$$Register, $str2$$Register,
17070 $cnt1$$Register, $cnt2$$Register,
17071 $tmp1$$Register, $tmp2$$Register,
17072 $tmp3$$Register, $tmp4$$Register,
17073 $tmp5$$Register, $tmp6$$Register,
17074 -1, $result$$Register, StrIntrinsicNode::UU);
17075 %}
17076 ins_pipe(pipe_class_memory);
17077 %}
17078
17079 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17080 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17081 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17082 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17083 %{
17084 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17085 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17086 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17087 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
17088 TEMP vtmp0, TEMP vtmp1, KILL cr);
17089 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
17090 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17091
17092 ins_encode %{
17093 __ string_indexof($str1$$Register, $str2$$Register,
17094 $cnt1$$Register, $cnt2$$Register,
17095 $tmp1$$Register, $tmp2$$Register,
17096 $tmp3$$Register, $tmp4$$Register,
17097 $tmp5$$Register, $tmp6$$Register,
17098 -1, $result$$Register, StrIntrinsicNode::LL);
17099 %}
17100 ins_pipe(pipe_class_memory);
17101 %}
17102
17103 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17104 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
17105 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17106 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17107 %{
17108 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17109 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17110 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17111 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
17112 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
17113 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
17114 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17115
17116 ins_encode %{
17117 __ string_indexof($str1$$Register, $str2$$Register,
17118 $cnt1$$Register, $cnt2$$Register,
17119 $tmp1$$Register, $tmp2$$Register,
17120 $tmp3$$Register, $tmp4$$Register,
17121 $tmp5$$Register, $tmp6$$Register,
17122 -1, $result$$Register, StrIntrinsicNode::UL);
17123 %}
17124 ins_pipe(pipe_class_memory);
17125 %}
17126
17127 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17128 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17129 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17130 %{
17131 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17132 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17133 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17134 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17135 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
17136 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17137
17138 ins_encode %{
17139 int icnt2 = (int)$int_cnt2$$constant;
17140 __ string_indexof($str1$$Register, $str2$$Register,
17141 $cnt1$$Register, zr,
17142 $tmp1$$Register, $tmp2$$Register,
17143 $tmp3$$Register, $tmp4$$Register, zr, zr,
17144 icnt2, $result$$Register, StrIntrinsicNode::UU);
17145 %}
17146 ins_pipe(pipe_class_memory);
17147 %}
17148
17149 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17150 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17151 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17152 %{
17153 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17154 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17155 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17156 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17157 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
17158 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17159
17160 ins_encode %{
17161 int icnt2 = (int)$int_cnt2$$constant;
17162 __ string_indexof($str1$$Register, $str2$$Register,
17163 $cnt1$$Register, zr,
17164 $tmp1$$Register, $tmp2$$Register,
17165 $tmp3$$Register, $tmp4$$Register, zr, zr,
17166 icnt2, $result$$Register, StrIntrinsicNode::LL);
17167 %}
17168 ins_pipe(pipe_class_memory);
17169 %}
17170
17171 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17172 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17173 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17174 %{
17175 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17176 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17177 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17178 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17179 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
17180 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17181
17182 ins_encode %{
17183 int icnt2 = (int)$int_cnt2$$constant;
17184 __ string_indexof($str1$$Register, $str2$$Register,
17185 $cnt1$$Register, zr,
17186 $tmp1$$Register, $tmp2$$Register,
17187 $tmp3$$Register, $tmp4$$Register, zr, zr,
17188 icnt2, $result$$Register, StrIntrinsicNode::UL);
17189 %}
17190 ins_pipe(pipe_class_memory);
17191 %}
17192
17193 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17194 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17195 iRegINoSp tmp3, rFlagsReg cr)
17196 %{
17197 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17198 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17199 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17200 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17201
17202 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17203
17204 ins_encode %{
17205 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17206 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17207 $tmp3$$Register);
17208 %}
17209 ins_pipe(pipe_class_memory);
17210 %}
17211
17212 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17213 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17214 iRegINoSp tmp3, rFlagsReg cr)
17215 %{
17216 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17217 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17218 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17219 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17220
17221 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17222
17223 ins_encode %{
17224 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17225 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17226 $tmp3$$Register);
17227 %}
17228 ins_pipe(pipe_class_memory);
17229 %}
17230
17231 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17232 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17233 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17234 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17235 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17236 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17237 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17238 ins_encode %{
17239 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17240 $result$$Register, $ztmp1$$FloatRegister,
17241 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17242 $ptmp$$PRegister, true /* isL */);
17243 %}
17244 ins_pipe(pipe_class_memory);
17245 %}
17246
17247 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17248 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17249 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17250 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17251 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17252 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17253 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17254 ins_encode %{
17255 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17256 $result$$Register, $ztmp1$$FloatRegister,
17257 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17258 $ptmp$$PRegister, false /* isL */);
17259 %}
17260 ins_pipe(pipe_class_memory);
17261 %}
17262
17263 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17264 iRegI_R0 result, rFlagsReg cr)
17265 %{
17266 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17267 match(Set result (StrEquals (Binary str1 str2) cnt));
17268 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17269
17270 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17271 ins_encode %{
17272 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17273 __ string_equals($str1$$Register, $str2$$Register,
17274 $result$$Register, $cnt$$Register, 1);
17275 %}
17276 ins_pipe(pipe_class_memory);
17277 %}
17278
17279 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17280 iRegI_R0 result, rFlagsReg cr)
17281 %{
17282 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17283 match(Set result (StrEquals (Binary str1 str2) cnt));
17284 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17285
17286 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17287 ins_encode %{
17288 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17289 __ string_equals($str1$$Register, $str2$$Register,
17290 $result$$Register, $cnt$$Register, 2);
17291 %}
17292 ins_pipe(pipe_class_memory);
17293 %}
17294
17295 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17296 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17297 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17298 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17299 iRegP_R10 tmp, rFlagsReg cr)
17300 %{
17301 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17302 match(Set result (AryEq ary1 ary2));
17303 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17304 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17305 TEMP vtmp6, TEMP vtmp7, KILL cr);
17306
17307 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17308 ins_encode %{
17309 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17310 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17311 $result$$Register, $tmp$$Register, 1);
17312 if (tpc == NULL) {
17313 ciEnv::current()->record_failure("CodeCache is full");
17314 return;
17315 }
17316 %}
17317 ins_pipe(pipe_class_memory);
17318 %}
17319
17320 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17321 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17322 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17323 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17324 iRegP_R10 tmp, rFlagsReg cr)
17325 %{
17326 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17327 match(Set result (AryEq ary1 ary2));
17328 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17329 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17330 TEMP vtmp6, TEMP vtmp7, KILL cr);
17331
17332 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17333 ins_encode %{
17334 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17335 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17336 $result$$Register, $tmp$$Register, 2);
17337 if (tpc == NULL) {
17338 ciEnv::current()->record_failure("CodeCache is full");
17339 return;
17340 }
17341 %}
17342 ins_pipe(pipe_class_memory);
17343 %}
17344
17345 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17346 %{
17347 match(Set result (CountPositives ary1 len));
17348 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17349 format %{ "count positives byte[] $ary1,$len -> $result" %}
17350 ins_encode %{
17351 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17352 if (tpc == NULL) {
17353 ciEnv::current()->record_failure("CodeCache is full");
17354 return;
17355 }
17356 %}
17357 ins_pipe( pipe_slow );
17358 %}
17359
17360 // fast char[] to byte[] compression
17361 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17362 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17363 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17364 iRegI_R0 result, rFlagsReg cr)
17365 %{
17366 match(Set result (StrCompressedCopy src (Binary dst len)));
17367 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17368 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17369
17370 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17371 ins_encode %{
17372 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17373 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17374 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17375 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17376 %}
17377 ins_pipe(pipe_slow);
17378 %}
17379
17380 // fast byte[] to char[] inflation
17381 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17382 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17383 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17384 %{
17385 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17386 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17387 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17388 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17389
17390 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17391 ins_encode %{
17392 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17393 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17394 $vtmp2$$FloatRegister, $tmp$$Register);
17395 if (tpc == NULL) {
17396 ciEnv::current()->record_failure("CodeCache is full");
17397 return;
17398 }
17399 %}
17400 ins_pipe(pipe_class_memory);
17401 %}
17402
17403 // encode char[] to byte[] in ISO_8859_1
17404 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17405 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17406 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17407 iRegI_R0 result, rFlagsReg cr)
17408 %{
17409 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17410 match(Set result (EncodeISOArray src (Binary dst len)));
17411 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17412 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17413
17414 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17415 ins_encode %{
17416 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17417 $result$$Register, false,
17418 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17419 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17420 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17421 %}
17422 ins_pipe(pipe_class_memory);
17423 %}
17424
17425 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17426 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17427 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17428 iRegI_R0 result, rFlagsReg cr)
17429 %{
17430 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17431 match(Set result (EncodeISOArray src (Binary dst len)));
17432 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17433 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17434
17435 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17436 ins_encode %{
17437 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17438 $result$$Register, true,
17439 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17440 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17441 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17442 %}
17443 ins_pipe(pipe_class_memory);
17444 %}
17445
17446 //----------------------------- CompressBits/ExpandBits ------------------------
17447
17448 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17449 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17450 match(Set dst (CompressBits src mask));
17451 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17452 format %{ "mov $tsrc, $src\n\t"
17453 "mov $tmask, $mask\n\t"
17454 "bext $tdst, $tsrc, $tmask\n\t"
17455 "mov $dst, $tdst"
17456 %}
17457 ins_encode %{
17458 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17459 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17460 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17461 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17462 %}
17463 ins_pipe(pipe_slow);
17464 %}
17465
17466 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17467 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17468 match(Set dst (CompressBits (LoadI mem) mask));
17469 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17470 format %{ "ldrs $tsrc, $mem\n\t"
17471 "ldrs $tmask, $mask\n\t"
17472 "bext $tdst, $tsrc, $tmask\n\t"
17473 "mov $dst, $tdst"
17474 %}
17475 ins_encode %{
17476 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17477 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17478 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17479 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17480 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17481 %}
17482 ins_pipe(pipe_slow);
17483 %}
17484
17485 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17486 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17487 match(Set dst (CompressBits src mask));
17488 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17489 format %{ "mov $tsrc, $src\n\t"
17490 "mov $tmask, $mask\n\t"
17491 "bext $tdst, $tsrc, $tmask\n\t"
17492 "mov $dst, $tdst"
17493 %}
17494 ins_encode %{
17495 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17496 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17497 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17498 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17499 %}
17500 ins_pipe(pipe_slow);
17501 %}
17502
17503 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17504 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17505 match(Set dst (CompressBits (LoadL mem) mask));
17506 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17507 format %{ "ldrd $tsrc, $mem\n\t"
17508 "ldrd $tmask, $mask\n\t"
17509 "bext $tdst, $tsrc, $tmask\n\t"
17510 "mov $dst, $tdst"
17511 %}
17512 ins_encode %{
17513 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17514 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17515 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17516 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17517 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17518 %}
17519 ins_pipe(pipe_slow);
17520 %}
17521
17522 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17523 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17524 match(Set dst (ExpandBits src mask));
17525 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17526 format %{ "mov $tsrc, $src\n\t"
17527 "mov $tmask, $mask\n\t"
17528 "bdep $tdst, $tsrc, $tmask\n\t"
17529 "mov $dst, $tdst"
17530 %}
17531 ins_encode %{
17532 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17533 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17534 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17535 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17536 %}
17537 ins_pipe(pipe_slow);
17538 %}
17539
17540 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17541 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17542 match(Set dst (ExpandBits (LoadI mem) mask));
17543 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17544 format %{ "ldrs $tsrc, $mem\n\t"
17545 "ldrs $tmask, $mask\n\t"
17546 "bdep $tdst, $tsrc, $tmask\n\t"
17547 "mov $dst, $tdst"
17548 %}
17549 ins_encode %{
17550 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17551 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17552 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17553 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17554 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17555 %}
17556 ins_pipe(pipe_slow);
17557 %}
17558
17559 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17560 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17561 match(Set dst (ExpandBits src mask));
17562 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17563 format %{ "mov $tsrc, $src\n\t"
17564 "mov $tmask, $mask\n\t"
17565 "bdep $tdst, $tsrc, $tmask\n\t"
17566 "mov $dst, $tdst"
17567 %}
17568 ins_encode %{
17569 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17570 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17571 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17572 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17573 %}
17574 ins_pipe(pipe_slow);
17575 %}
17576
17577
17578 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17579 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17580 match(Set dst (ExpandBits (LoadL mem) mask));
17581 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17582 format %{ "ldrd $tsrc, $mem\n\t"
17583 "ldrd $tmask, $mask\n\t"
17584 "bdep $tdst, $tsrc, $tmask\n\t"
17585 "mov $dst, $tdst"
17586 %}
17587 ins_encode %{
17588 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17589 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17590 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17591 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17592 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17593 %}
17594 ins_pipe(pipe_slow);
17595 %}
17596
17597 // ============================================================================
17598 // This name is KNOWN by the ADLC and cannot be changed.
17599 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17600 // for this guy.
17601 instruct tlsLoadP(thread_RegP dst)
17602 %{
17603 match(Set dst (ThreadLocal));
17604
17605 ins_cost(0);
17606
17607 format %{ " -- \t// $dst=Thread::current(), empty" %}
17608
17609 size(0);
17610
17611 ins_encode( /*empty*/ );
17612
17613 ins_pipe(pipe_class_empty);
17614 %}
17615
17616 //----------PEEPHOLE RULES-----------------------------------------------------
17617 // These must follow all instruction definitions as they use the names
17618 // defined in the instructions definitions.
17619 //
17620 // peepmatch ( root_instr_name [preceding_instruction]* );
17621 //
17622 // peepconstraint %{
17623 // (instruction_number.operand_name relational_op instruction_number.operand_name
17624 // [, ...] );
17625 // // instruction numbers are zero-based using left to right order in peepmatch
17626 //
17627 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17628 // // provide an instruction_number.operand_name for each operand that appears
17629 // // in the replacement instruction's match rule
17630 //
17631 // ---------VM FLAGS---------------------------------------------------------
17632 //
17633 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17634 //
17635 // Each peephole rule is given an identifying number starting with zero and
17636 // increasing by one in the order seen by the parser. An individual peephole
17637 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17638 // on the command-line.
17639 //
17640 // ---------CURRENT LIMITATIONS----------------------------------------------
17641 //
17642 // Only match adjacent instructions in same basic block
17643 // Only equality constraints
17644 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17645 // Only one replacement instruction
17646 //
17647 // ---------EXAMPLE----------------------------------------------------------
17648 //
17649 // // pertinent parts of existing instructions in architecture description
17650 // instruct movI(iRegINoSp dst, iRegI src)
17651 // %{
17652 // match(Set dst (CopyI src));
17653 // %}
17654 //
17655 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17656 // %{
17657 // match(Set dst (AddI dst src));
17658 // effect(KILL cr);
17659 // %}
17660 //
17661 // // Change (inc mov) to lea
17662 // peephole %{
17663 // // increment preceded by register-register move
17664 // peepmatch ( incI_iReg movI );
17665 // // require that the destination register of the increment
17666 // // match the destination register of the move
17667 // peepconstraint ( 0.dst == 1.dst );
17668 // // construct a replacement instruction that sets
17669 // // the destination to ( move's source register + one )
17670 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17671 // %}
17672 //
17673
17674 // Implementation no longer uses movX instructions since
17675 // machine-independent system no longer uses CopyX nodes.
17676 //
17677 // peephole
17678 // %{
17679 // peepmatch (incI_iReg movI);
17680 // peepconstraint (0.dst == 1.dst);
17681 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17682 // %}
17683
17684 // peephole
17685 // %{
17686 // peepmatch (decI_iReg movI);
17687 // peepconstraint (0.dst == 1.dst);
17688 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17689 // %}
17690
17691 // peephole
17692 // %{
17693 // peepmatch (addI_iReg_imm movI);
17694 // peepconstraint (0.dst == 1.dst);
17695 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17696 // %}
17697
17698 // peephole
17699 // %{
17700 // peepmatch (incL_iReg movL);
17701 // peepconstraint (0.dst == 1.dst);
17702 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17703 // %}
17704
17705 // peephole
17706 // %{
17707 // peepmatch (decL_iReg movL);
17708 // peepconstraint (0.dst == 1.dst);
17709 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17710 // %}
17711
17712 // peephole
17713 // %{
17714 // peepmatch (addL_iReg_imm movL);
17715 // peepconstraint (0.dst == 1.dst);
17716 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17717 // %}
17718
17719 // peephole
17720 // %{
17721 // peepmatch (addP_iReg_imm movP);
17722 // peepconstraint (0.dst == 1.dst);
17723 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17724 // %}
17725
17726 // // Change load of spilled value to only a spill
17727 // instruct storeI(memory mem, iRegI src)
17728 // %{
17729 // match(Set mem (StoreI mem src));
17730 // %}
17731 //
17732 // instruct loadI(iRegINoSp dst, memory mem)
17733 // %{
17734 // match(Set dst (LoadI mem));
17735 // %}
17736 //
17737
17738 //----------SMARTSPILL RULES---------------------------------------------------
17739 // These must follow all instruction definitions as they use the names
17740 // defined in the instructions definitions.
17741
17742 // Local Variables:
17743 // mode: c++
17744 // End: