1 //
2 // Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for heapbase register
668 reg_class heapbase_reg(
669 R27, R27_H
670 );
671
672 // Class for thread register
673 reg_class thread_reg(
674 R28, R28_H
675 );
676
677 // Class for frame pointer register
678 reg_class fp_reg(
679 R29, R29_H
680 );
681
682 // Class for link register
683 reg_class lr_reg(
684 R30, R30_H
685 );
686
687 // Class for long sp register
688 reg_class sp_reg(
689 R31, R31_H
690 );
691
692 // Class for all pointer registers
693 reg_class ptr_reg %{
694 return _PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers
698 reg_class no_special_ptr_reg %{
699 return _NO_SPECIAL_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133
1134 class CallStubImpl {
1135
1136 //--------------------------------------------------------------
1137 //---< Used for optimization in Compile::shorten_branches >---
1138 //--------------------------------------------------------------
1139
1140 public:
1141 // Size of call trampoline stub.
1142 static uint size_call_trampoline() {
1143 return 0; // no call trampolines on this platform
1144 }
1145
1146 // number of relocations needed by a call trampoline stub
1147 static uint reloc_call_trampoline() {
1148 return 0; // no call trampolines on this platform
1149 }
1150 };
1151
1152 class HandlerImpl {
1153
1154 public:
1155
1156 static int emit_exception_handler(CodeBuffer &cbuf);
1157 static int emit_deopt_handler(CodeBuffer& cbuf);
1158
1159 static uint size_exception_handler() {
1160 return MacroAssembler::far_codestub_branch_size();
1161 }
1162
1163 static uint size_deopt_handler() {
1164 // count one adr and one far branch instruction
1165 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1166 }
1167 };
1168
1169 class Node::PD {
1170 public:
1171 enum NodeFlags {
1172 _last_flag = Node::_last_flag
1173 };
1174 };
1175
1176 bool is_CAS(int opcode, bool maybe_volatile);
1177
1178 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1179
1180 bool unnecessary_acquire(const Node *barrier);
1181 bool needs_acquiring_load(const Node *load);
1182
1183 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1184
1185 bool unnecessary_release(const Node *barrier);
1186 bool unnecessary_volatile(const Node *barrier);
1187 bool needs_releasing_store(const Node *store);
1188
1189 // predicate controlling translation of CompareAndSwapX
1190 bool needs_acquiring_load_exclusive(const Node *load);
1191
1192 // predicate controlling addressing modes
1193 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1194 %}
1195
1196 source %{
1197
1198 // Derived RegMask with conditionally allocatable registers
1199
1200 void PhaseOutput::pd_perform_mach_node_analysis() {
1201 }
1202
1203 int MachNode::pd_alignment_required() const {
1204 return 1;
1205 }
1206
1207 int MachNode::compute_padding(int current_offset) const {
1208 return 0;
1209 }
1210
1211 RegMask _ANY_REG32_mask;
1212 RegMask _ANY_REG_mask;
1213 RegMask _PTR_REG_mask;
1214 RegMask _NO_SPECIAL_REG32_mask;
1215 RegMask _NO_SPECIAL_REG_mask;
1216 RegMask _NO_SPECIAL_PTR_REG_mask;
1217
1218 void reg_mask_init() {
1219 // We derive below RegMask(s) from the ones which are auto-generated from
1220 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1221 // registers conditionally reserved.
1222
1223 _ANY_REG32_mask = _ALL_REG32_mask;
1224 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1225
1226 _ANY_REG_mask = _ALL_REG_mask;
1227
1228 _PTR_REG_mask = _ALL_REG_mask;
1229
1230 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1231 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1232
1233 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1234 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1235
1236 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1237 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1238
1239 // r27 is not allocatable when compressed oops is on and heapbase is not
1240 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1241 if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
1242 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1243 _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1244 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1245 }
1246
1247 // r29 is not allocatable when PreserveFramePointer is on
1248 if (PreserveFramePointer) {
1249 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1250 _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
1251 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
1252 }
1253 }
1254
1255 // Optimizaton of volatile gets and puts
1256 // -------------------------------------
1257 //
1258 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1259 // use to implement volatile reads and writes. For a volatile read
1260 // we simply need
1261 //
1262 // ldar<x>
1263 //
1264 // and for a volatile write we need
1265 //
1266 // stlr<x>
1267 //
1268 // Alternatively, we can implement them by pairing a normal
1269 // load/store with a memory barrier. For a volatile read we need
1270 //
1271 // ldr<x>
1272 // dmb ishld
1273 //
1274 // for a volatile write
1275 //
1276 // dmb ish
1277 // str<x>
1278 // dmb ish
1279 //
1280 // We can also use ldaxr and stlxr to implement compare and swap CAS
1281 // sequences. These are normally translated to an instruction
1282 // sequence like the following
1283 //
1284 // dmb ish
1285 // retry:
1286 // ldxr<x> rval raddr
1287 // cmp rval rold
1288 // b.ne done
1289 // stlxr<x> rval, rnew, rold
1290 // cbnz rval retry
1291 // done:
1292 // cset r0, eq
1293 // dmb ishld
1294 //
1295 // Note that the exclusive store is already using an stlxr
1296 // instruction. That is required to ensure visibility to other
1297 // threads of the exclusive write (assuming it succeeds) before that
1298 // of any subsequent writes.
1299 //
1300 // The following instruction sequence is an improvement on the above
1301 //
1302 // retry:
1303 // ldaxr<x> rval raddr
1304 // cmp rval rold
1305 // b.ne done
1306 // stlxr<x> rval, rnew, rold
1307 // cbnz rval retry
1308 // done:
1309 // cset r0, eq
1310 //
1311 // We don't need the leading dmb ish since the stlxr guarantees
1312 // visibility of prior writes in the case that the swap is
1313 // successful. Crucially we don't have to worry about the case where
1314 // the swap is not successful since no valid program should be
1315 // relying on visibility of prior changes by the attempting thread
1316 // in the case where the CAS fails.
1317 //
1318 // Similarly, we don't need the trailing dmb ishld if we substitute
1319 // an ldaxr instruction since that will provide all the guarantees we
1320 // require regarding observation of changes made by other threads
1321 // before any change to the CAS address observed by the load.
1322 //
1323 // In order to generate the desired instruction sequence we need to
1324 // be able to identify specific 'signature' ideal graph node
1325 // sequences which i) occur as a translation of a volatile reads or
1326 // writes or CAS operations and ii) do not occur through any other
1327 // translation or graph transformation. We can then provide
1328 // alternative aldc matching rules which translate these node
1329 // sequences to the desired machine code sequences. Selection of the
1330 // alternative rules can be implemented by predicates which identify
1331 // the relevant node sequences.
1332 //
1333 // The ideal graph generator translates a volatile read to the node
1334 // sequence
1335 //
1336 // LoadX[mo_acquire]
1337 // MemBarAcquire
1338 //
1339 // As a special case when using the compressed oops optimization we
1340 // may also see this variant
1341 //
1342 // LoadN[mo_acquire]
1343 // DecodeN
1344 // MemBarAcquire
1345 //
1346 // A volatile write is translated to the node sequence
1347 //
1348 // MemBarRelease
1349 // StoreX[mo_release] {CardMark}-optional
1350 // MemBarVolatile
1351 //
1352 // n.b. the above node patterns are generated with a strict
1353 // 'signature' configuration of input and output dependencies (see
1354 // the predicates below for exact details). The card mark may be as
1355 // simple as a few extra nodes or, in a few GC configurations, may
1356 // include more complex control flow between the leading and
1357 // trailing memory barriers. However, whatever the card mark
1358 // configuration these signatures are unique to translated volatile
1359 // reads/stores -- they will not appear as a result of any other
1360 // bytecode translation or inlining nor as a consequence of
1361 // optimizing transforms.
1362 //
1363 // We also want to catch inlined unsafe volatile gets and puts and
1364 // be able to implement them using either ldar<x>/stlr<x> or some
1365 // combination of ldr<x>/stlr<x> and dmb instructions.
1366 //
1367 // Inlined unsafe volatiles puts manifest as a minor variant of the
1368 // normal volatile put node sequence containing an extra cpuorder
1369 // membar
1370 //
1371 // MemBarRelease
1372 // MemBarCPUOrder
1373 // StoreX[mo_release] {CardMark}-optional
1374 // MemBarCPUOrder
1375 // MemBarVolatile
1376 //
1377 // n.b. as an aside, a cpuorder membar is not itself subject to
1378 // matching and translation by adlc rules. However, the rule
1379 // predicates need to detect its presence in order to correctly
1380 // select the desired adlc rules.
1381 //
1382 // Inlined unsafe volatile gets manifest as a slightly different
1383 // node sequence to a normal volatile get because of the
1384 // introduction of some CPUOrder memory barriers to bracket the
1385 // Load. However, but the same basic skeleton of a LoadX feeding a
1386 // MemBarAcquire, possibly through an optional DecodeN, is still
1387 // present
1388 //
1389 // MemBarCPUOrder
1390 // || \\
1391 // MemBarCPUOrder LoadX[mo_acquire]
1392 // || |
1393 // || {DecodeN} optional
1394 // || /
1395 // MemBarAcquire
1396 //
1397 // In this case the acquire membar does not directly depend on the
1398 // load. However, we can be sure that the load is generated from an
1399 // inlined unsafe volatile get if we see it dependent on this unique
1400 // sequence of membar nodes. Similarly, given an acquire membar we
1401 // can know that it was added because of an inlined unsafe volatile
1402 // get if it is fed and feeds a cpuorder membar and if its feed
1403 // membar also feeds an acquiring load.
1404 //
1405 // Finally an inlined (Unsafe) CAS operation is translated to the
1406 // following ideal graph
1407 //
1408 // MemBarRelease
1409 // MemBarCPUOrder
1410 // CompareAndSwapX {CardMark}-optional
1411 // MemBarCPUOrder
1412 // MemBarAcquire
1413 //
1414 // So, where we can identify these volatile read and write
1415 // signatures we can choose to plant either of the above two code
1416 // sequences. For a volatile read we can simply plant a normal
1417 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1418 // also choose to inhibit translation of the MemBarAcquire and
1419 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1420 //
1421 // When we recognise a volatile store signature we can choose to
1422 // plant at a dmb ish as a translation for the MemBarRelease, a
1423 // normal str<x> and then a dmb ish for the MemBarVolatile.
1424 // Alternatively, we can inhibit translation of the MemBarRelease
1425 // and MemBarVolatile and instead plant a simple stlr<x>
1426 // instruction.
1427 //
1428 // when we recognise a CAS signature we can choose to plant a dmb
1429 // ish as a translation for the MemBarRelease, the conventional
1430 // macro-instruction sequence for the CompareAndSwap node (which
1431 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1432 // Alternatively, we can elide generation of the dmb instructions
1433 // and plant the alternative CompareAndSwap macro-instruction
1434 // sequence (which uses ldaxr<x>).
1435 //
1436 // Of course, the above only applies when we see these signature
1437 // configurations. We still want to plant dmb instructions in any
1438 // other cases where we may see a MemBarAcquire, MemBarRelease or
1439 // MemBarVolatile. For example, at the end of a constructor which
1440 // writes final/volatile fields we will see a MemBarRelease
1441 // instruction and this needs a 'dmb ish' lest we risk the
1442 // constructed object being visible without making the
1443 // final/volatile field writes visible.
1444 //
1445 // n.b. the translation rules below which rely on detection of the
1446 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1447 // If we see anything other than the signature configurations we
1448 // always just translate the loads and stores to ldr<x> and str<x>
1449 // and translate acquire, release and volatile membars to the
1450 // relevant dmb instructions.
1451 //
1452
1453 // is_CAS(int opcode, bool maybe_volatile)
1454 //
1455 // return true if opcode is one of the possible CompareAndSwapX
1456 // values otherwise false.
1457
1458 bool is_CAS(int opcode, bool maybe_volatile)
1459 {
1460 switch(opcode) {
1461 // We handle these
1462 case Op_CompareAndSwapI:
1463 case Op_CompareAndSwapL:
1464 case Op_CompareAndSwapP:
1465 case Op_CompareAndSwapN:
1466 case Op_ShenandoahCompareAndSwapP:
1467 case Op_ShenandoahCompareAndSwapN:
1468 case Op_CompareAndSwapB:
1469 case Op_CompareAndSwapS:
1470 case Op_GetAndSetI:
1471 case Op_GetAndSetL:
1472 case Op_GetAndSetP:
1473 case Op_GetAndSetN:
1474 case Op_GetAndAddI:
1475 case Op_GetAndAddL:
1476 return true;
1477 case Op_CompareAndExchangeI:
1478 case Op_CompareAndExchangeN:
1479 case Op_CompareAndExchangeB:
1480 case Op_CompareAndExchangeS:
1481 case Op_CompareAndExchangeL:
1482 case Op_CompareAndExchangeP:
1483 case Op_WeakCompareAndSwapB:
1484 case Op_WeakCompareAndSwapS:
1485 case Op_WeakCompareAndSwapI:
1486 case Op_WeakCompareAndSwapL:
1487 case Op_WeakCompareAndSwapP:
1488 case Op_WeakCompareAndSwapN:
1489 case Op_ShenandoahWeakCompareAndSwapP:
1490 case Op_ShenandoahWeakCompareAndSwapN:
1491 case Op_ShenandoahCompareAndExchangeP:
1492 case Op_ShenandoahCompareAndExchangeN:
1493 return maybe_volatile;
1494 default:
1495 return false;
1496 }
1497 }
1498
1499 // helper to determine the maximum number of Phi nodes we may need to
1500 // traverse when searching from a card mark membar for the merge mem
1501 // feeding a trailing membar or vice versa
1502
1503 // predicates controlling emit of ldr<x>/ldar<x>
1504
1505 bool unnecessary_acquire(const Node *barrier)
1506 {
1507 assert(barrier->is_MemBar(), "expecting a membar");
1508
1509 MemBarNode* mb = barrier->as_MemBar();
1510
1511 if (mb->trailing_load()) {
1512 return true;
1513 }
1514
1515 if (mb->trailing_load_store()) {
1516 Node* load_store = mb->in(MemBarNode::Precedent);
1517 assert(load_store->is_LoadStore(), "unexpected graph shape");
1518 return is_CAS(load_store->Opcode(), true);
1519 }
1520
1521 return false;
1522 }
1523
1524 bool needs_acquiring_load(const Node *n)
1525 {
1526 assert(n->is_Load(), "expecting a load");
1527 LoadNode *ld = n->as_Load();
1528 return ld->is_acquire();
1529 }
1530
1531 bool unnecessary_release(const Node *n)
1532 {
1533 assert((n->is_MemBar() &&
1534 n->Opcode() == Op_MemBarRelease),
1535 "expecting a release membar");
1536
1537 MemBarNode *barrier = n->as_MemBar();
1538 if (!barrier->leading()) {
1539 return false;
1540 } else {
1541 Node* trailing = barrier->trailing_membar();
1542 MemBarNode* trailing_mb = trailing->as_MemBar();
1543 assert(trailing_mb->trailing(), "Not a trailing membar?");
1544 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1545
1546 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1547 if (mem->is_Store()) {
1548 assert(mem->as_Store()->is_release(), "");
1549 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1550 return true;
1551 } else {
1552 assert(mem->is_LoadStore(), "");
1553 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1554 return is_CAS(mem->Opcode(), true);
1555 }
1556 }
1557 return false;
1558 }
1559
1560 bool unnecessary_volatile(const Node *n)
1561 {
1562 // assert n->is_MemBar();
1563 MemBarNode *mbvol = n->as_MemBar();
1564
1565 bool release = mbvol->trailing_store();
1566 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1567 #ifdef ASSERT
1568 if (release) {
1569 Node* leading = mbvol->leading_membar();
1570 assert(leading->Opcode() == Op_MemBarRelease, "");
1571 assert(leading->as_MemBar()->leading_store(), "");
1572 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1573 }
1574 #endif
1575
1576 return release;
1577 }
1578
1579 // predicates controlling emit of str<x>/stlr<x>
1580
1581 bool needs_releasing_store(const Node *n)
1582 {
1583 // assert n->is_Store();
1584 StoreNode *st = n->as_Store();
1585 return st->trailing_membar() != NULL;
1586 }
1587
1588 // predicate controlling translation of CAS
1589 //
1590 // returns true if CAS needs to use an acquiring load otherwise false
1591
1592 bool needs_acquiring_load_exclusive(const Node *n)
1593 {
1594 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1595 LoadStoreNode* ldst = n->as_LoadStore();
1596 if (is_CAS(n->Opcode(), false)) {
1597 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1598 } else {
1599 return ldst->trailing_membar() != NULL;
1600 }
1601
1602 // so we can just return true here
1603 return true;
1604 }
1605
1606 #define __ _masm.
1607
1608 // advance declarations for helper functions to convert register
1609 // indices to register objects
1610
1611 // the ad file has to provide implementations of certain methods
1612 // expected by the generic code
1613 //
1614 // REQUIRED FUNCTIONALITY
1615
1616 //=============================================================================
1617
1618 // !!!!! Special hack to get all types of calls to specify the byte offset
1619 // from the start of the call to the point where the return address
1620 // will point.
1621
1622 int MachCallStaticJavaNode::ret_addr_offset()
1623 {
1624 // call should be a simple bl
1625 int off = 4;
1626 return off;
1627 }
1628
1629 int MachCallDynamicJavaNode::ret_addr_offset()
1630 {
1631 return 16; // movz, movk, movk, bl
1632 }
1633
1634 int MachCallRuntimeNode::ret_addr_offset() {
1635 // for generated stubs the call will be
1636 // bl(addr)
1637 // or with far branches
1638 // bl(trampoline_stub)
1639 // for real runtime callouts it will be six instructions
1640 // see aarch64_enc_java_to_runtime
1641 // adr(rscratch2, retaddr)
1642 // lea(rscratch1, RuntimeAddress(addr)
1643 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1644 // blr(rscratch1)
1645 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1646 if (cb) {
1647 return 1 * NativeInstruction::instruction_size;
1648 } else if (_entry_point == NULL) {
1649 // See CallLeafNoFPIndirect
1650 return 1 * NativeInstruction::instruction_size;
1651 } else {
1652 return 6 * NativeInstruction::instruction_size;
1653 }
1654 }
1655
1656 //=============================================================================
1657
1658 #ifndef PRODUCT
1659 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1660 st->print("BREAKPOINT");
1661 }
1662 #endif
1663
1664 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1665 C2_MacroAssembler _masm(&cbuf);
1666 __ brk(0);
1667 }
1668
1669 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1670 return MachNode::size(ra_);
1671 }
1672
1673 //=============================================================================
1674
1675 #ifndef PRODUCT
1676 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1677 st->print("nop \t# %d bytes pad for loops and calls", _count);
1678 }
1679 #endif
1680
1681 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1682 C2_MacroAssembler _masm(&cbuf);
1683 for (int i = 0; i < _count; i++) {
1684 __ nop();
1685 }
1686 }
1687
1688 uint MachNopNode::size(PhaseRegAlloc*) const {
1689 return _count * NativeInstruction::instruction_size;
1690 }
1691
1692 //=============================================================================
1693 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1694
1695 int ConstantTable::calculate_table_base_offset() const {
1696 return 0; // absolute addressing, no offset
1697 }
1698
1699 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1700 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1701 ShouldNotReachHere();
1702 }
1703
1704 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1705 // Empty encoding
1706 }
1707
1708 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1709 return 0;
1710 }
1711
1712 #ifndef PRODUCT
1713 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1714 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1715 }
1716 #endif
1717
1718 #ifndef PRODUCT
1719 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1720 Compile* C = ra_->C;
1721
1722 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1723
1724 if (C->output()->need_stack_bang(framesize))
1725 st->print("# stack bang size=%d\n\t", framesize);
1726
1727 if (VM_Version::use_rop_protection()) {
1728 st->print("ldr zr, [lr]\n\t");
1729 st->print("pacia lr, rfp\n\t");
1730 }
1731 if (framesize < ((1 << 9) + 2 * wordSize)) {
1732 st->print("sub sp, sp, #%d\n\t", framesize);
1733 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1734 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1735 } else {
1736 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1737 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1738 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1739 st->print("sub sp, sp, rscratch1");
1740 }
1741 if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1742 st->print("\n\t");
1743 st->print("ldr rscratch1, [guard]\n\t");
1744 st->print("dmb ishld\n\t");
1745 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1746 st->print("cmp rscratch1, rscratch2\n\t");
1747 st->print("b.eq skip");
1748 st->print("\n\t");
1749 st->print("blr #nmethod_entry_barrier_stub\n\t");
1750 st->print("b skip\n\t");
1751 st->print("guard: int\n\t");
1752 st->print("\n\t");
1753 st->print("skip:\n\t");
1754 }
1755 }
1756 #endif
1757
1758 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1759 Compile* C = ra_->C;
1760 C2_MacroAssembler _masm(&cbuf);
1761
1762 // insert a nop at the start of the prolog so we can patch in a
1763 // branch if we need to invalidate the method later
1764 __ nop();
1765
1766 __ verified_entry(C, 0);
1767
1768 if (C->stub_function() == NULL) {
1769 __ entry_barrier();
1770 }
1771
1772 if (!Compile::current()->output()->in_scratch_emit_size()) {
1773 __ bind(*_verified_entry);
1774 }
1775
1776 if (VerifyStackAtCalls) {
1777 Unimplemented();
1778 }
1779
1780 C->output()->set_frame_complete(cbuf.insts_size());
1781
1782 if (C->has_mach_constant_base_node()) {
1783 // NOTE: We set the table base offset here because users might be
1784 // emitted before MachConstantBaseNode.
1785 ConstantTable& constant_table = C->output()->constant_table();
1786 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1787 }
1788 }
1789
1790 int MachPrologNode::reloc() const
1791 {
1792 return 0;
1793 }
1794
1795 //=============================================================================
1796
1797 #ifndef PRODUCT
1798 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1799 Compile* C = ra_->C;
1800 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1801
1802 st->print("# pop frame %d\n\t",framesize);
1803
1804 if (framesize == 0) {
1805 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1806 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1807 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1808 st->print("add sp, sp, #%d\n\t", framesize);
1809 } else {
1810 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1811 st->print("add sp, sp, rscratch1\n\t");
1812 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1813 }
1814 if (VM_Version::use_rop_protection()) {
1815 st->print("autia lr, rfp\n\t");
1816 st->print("ldr zr, [lr]\n\t");
1817 }
1818
1819 if (do_polling() && C->is_method_compilation()) {
1820 st->print("# test polling word\n\t");
1821 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1822 st->print("cmp sp, rscratch1\n\t");
1823 st->print("bhi #slow_path");
1824 }
1825 }
1826 #endif
1827
1828 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1829 Compile* C = ra_->C;
1830 C2_MacroAssembler _masm(&cbuf);
1831 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1832
1833 __ remove_frame(framesize, C->needs_stack_repair());
1834
1835 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1836 __ reserved_stack_check();
1837 }
1838
1839 if (do_polling() && C->is_method_compilation()) {
1840 Label dummy_label;
1841 Label* code_stub = &dummy_label;
1842 if (!C->output()->in_scratch_emit_size()) {
1843 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1844 C->output()->add_stub(stub);
1845 code_stub = &stub->entry();
1846 }
1847 __ relocate(relocInfo::poll_return_type);
1848 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1849 }
1850 }
1851
1852 int MachEpilogNode::reloc() const {
1853 // Return number of relocatable values contained in this instruction.
1854 return 1; // 1 for polling page.
1855 }
1856
1857 const Pipeline * MachEpilogNode::pipeline() const {
1858 return MachNode::pipeline_class();
1859 }
1860
1861 //=============================================================================
1862
1863 // Figure out which register class each belongs in: rc_int, rc_float or
1864 // rc_stack.
1865 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
1866
1867 static enum RC rc_class(OptoReg::Name reg) {
1868
1869 if (reg == OptoReg::Bad) {
1870 return rc_bad;
1871 }
1872
1873 // we have 32 int registers * 2 halves
1874 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1875
1876 if (reg < slots_of_int_registers) {
1877 return rc_int;
1878 }
1879
1880 // we have 32 float register * 8 halves
1881 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1882 if (reg < slots_of_int_registers + slots_of_float_registers) {
1883 return rc_float;
1884 }
1885
1886 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1887 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1888 return rc_predicate;
1889 }
1890
1891 // Between predicate regs & stack is the flags.
1892 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1893
1894 return rc_stack;
1895 }
1896
1897 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1898 Compile* C = ra_->C;
1899
1900 // Get registers to move.
1901 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1902 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1903 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1904 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1905
1906 enum RC src_hi_rc = rc_class(src_hi);
1907 enum RC src_lo_rc = rc_class(src_lo);
1908 enum RC dst_hi_rc = rc_class(dst_hi);
1909 enum RC dst_lo_rc = rc_class(dst_lo);
1910
1911 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1912
1913 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1914 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1915 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1916 "expected aligned-adjacent pairs");
1917 }
1918
1919 if (src_lo == dst_lo && src_hi == dst_hi) {
1920 return 0; // Self copy, no move.
1921 }
1922
1923 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1924 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1925 int src_offset = ra_->reg2offset(src_lo);
1926 int dst_offset = ra_->reg2offset(dst_lo);
1927
1928 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1929 uint ireg = ideal_reg();
1930 if (ireg == Op_VecA && cbuf) {
1931 C2_MacroAssembler _masm(cbuf);
1932 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1933 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1934 // stack->stack
1935 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1936 sve_vector_reg_size_in_bytes);
1937 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1938 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1939 sve_vector_reg_size_in_bytes);
1940 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1941 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1942 sve_vector_reg_size_in_bytes);
1943 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1944 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1945 as_FloatRegister(Matcher::_regEncode[src_lo]),
1946 as_FloatRegister(Matcher::_regEncode[src_lo]));
1947 } else {
1948 ShouldNotReachHere();
1949 }
1950 } else if (cbuf) {
1951 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1952 C2_MacroAssembler _masm(cbuf);
1953 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1954 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1955 // stack->stack
1956 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1957 if (ireg == Op_VecD) {
1958 __ unspill(rscratch1, true, src_offset);
1959 __ spill(rscratch1, true, dst_offset);
1960 } else {
1961 __ spill_copy128(src_offset, dst_offset);
1962 }
1963 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1964 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1965 ireg == Op_VecD ? __ T8B : __ T16B,
1966 as_FloatRegister(Matcher::_regEncode[src_lo]));
1967 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1968 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1969 ireg == Op_VecD ? __ D : __ Q,
1970 ra_->reg2offset(dst_lo));
1971 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1972 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1973 ireg == Op_VecD ? __ D : __ Q,
1974 ra_->reg2offset(src_lo));
1975 } else {
1976 ShouldNotReachHere();
1977 }
1978 }
1979 } else if (cbuf) {
1980 C2_MacroAssembler _masm(cbuf);
1981 switch (src_lo_rc) {
1982 case rc_int:
1983 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
1984 if (is64) {
1985 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1986 as_Register(Matcher::_regEncode[src_lo]));
1987 } else {
1988 C2_MacroAssembler _masm(cbuf);
1989 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1990 as_Register(Matcher::_regEncode[src_lo]));
1991 }
1992 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1993 if (is64) {
1994 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1995 as_Register(Matcher::_regEncode[src_lo]));
1996 } else {
1997 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1998 as_Register(Matcher::_regEncode[src_lo]));
1999 }
2000 } else { // gpr --> stack spill
2001 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2002 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2003 }
2004 break;
2005 case rc_float:
2006 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2007 if (is64) {
2008 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2009 as_FloatRegister(Matcher::_regEncode[src_lo]));
2010 } else {
2011 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2012 as_FloatRegister(Matcher::_regEncode[src_lo]));
2013 }
2014 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2015 if (is64) {
2016 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 as_FloatRegister(Matcher::_regEncode[src_lo]));
2018 } else {
2019 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2020 as_FloatRegister(Matcher::_regEncode[src_lo]));
2021 }
2022 } else { // fpr --> stack spill
2023 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2024 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2025 is64 ? __ D : __ S, dst_offset);
2026 }
2027 break;
2028 case rc_stack:
2029 if (dst_lo_rc == rc_int) { // stack --> gpr load
2030 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2031 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2032 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2033 is64 ? __ D : __ S, src_offset);
2034 } else if (dst_lo_rc == rc_predicate) {
2035 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2036 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2037 } else { // stack --> stack copy
2038 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2039 if (ideal_reg() == Op_RegVectMask) {
2040 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2041 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2042 } else {
2043 __ unspill(rscratch1, is64, src_offset);
2044 __ spill(rscratch1, is64, dst_offset);
2045 }
2046 }
2047 break;
2048 case rc_predicate:
2049 if (dst_lo_rc == rc_predicate) {
2050 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2051 } else if (dst_lo_rc == rc_stack) {
2052 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2053 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2054 } else {
2055 assert(false, "bad src and dst rc_class combination.");
2056 ShouldNotReachHere();
2057 }
2058 break;
2059 default:
2060 assert(false, "bad rc_class for spill");
2061 ShouldNotReachHere();
2062 }
2063 }
2064
2065 if (st) {
2066 st->print("spill ");
2067 if (src_lo_rc == rc_stack) {
2068 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2069 } else {
2070 st->print("%s -> ", Matcher::regName[src_lo]);
2071 }
2072 if (dst_lo_rc == rc_stack) {
2073 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2074 } else {
2075 st->print("%s", Matcher::regName[dst_lo]);
2076 }
2077 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2078 int vsize = 0;
2079 switch (ideal_reg()) {
2080 case Op_VecD:
2081 vsize = 64;
2082 break;
2083 case Op_VecX:
2084 vsize = 128;
2085 break;
2086 case Op_VecA:
2087 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2088 break;
2089 default:
2090 assert(false, "bad register type for spill");
2091 ShouldNotReachHere();
2092 }
2093 st->print("\t# vector spill size = %d", vsize);
2094 } else if (ideal_reg() == Op_RegVectMask) {
2095 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2096 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2097 st->print("\t# predicate spill size = %d", vsize);
2098 } else {
2099 st->print("\t# spill size = %d", is64 ? 64 : 32);
2100 }
2101 }
2102
2103 return 0;
2104
2105 }
2106
2107 #ifndef PRODUCT
2108 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2109 if (!ra_)
2110 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2111 else
2112 implementation(NULL, ra_, false, st);
2113 }
2114 #endif
2115
2116 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2117 implementation(&cbuf, ra_, false, NULL);
2118 }
2119
2120 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2121 return MachNode::size(ra_);
2122 }
2123
2124 //=============================================================================
2125
2126 #ifndef PRODUCT
2127 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2128 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2129 int reg = ra_->get_reg_first(this);
2130 st->print("add %s, rsp, #%d]\t# box lock",
2131 Matcher::regName[reg], offset);
2132 }
2133 #endif
2134
2135 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2136 C2_MacroAssembler _masm(&cbuf);
2137
2138 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2139 int reg = ra_->get_encode(this);
2140
2141 // This add will handle any 24-bit signed offset. 24 bits allows an
2142 // 8 megabyte stack frame.
2143 __ add(as_Register(reg), sp, offset);
2144 }
2145
2146 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2147 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2148 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2149
2150 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2151 return NativeInstruction::instruction_size;
2152 } else {
2153 return 2 * NativeInstruction::instruction_size;
2154 }
2155 }
2156
2157 ///=============================================================================
2158 #ifndef PRODUCT
2159 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2160 {
2161 st->print_cr("# MachVEPNode");
2162 if (!_verified) {
2163 st->print_cr("\t load_class");
2164 } else {
2165 st->print_cr("\t unpack_inline_arg");
2166 }
2167 }
2168 #endif
2169
2170 void MachVEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2171 {
2172 C2_MacroAssembler _masm(&cbuf);
2173
2174 if (!_verified) {
2175 Label skip;
2176 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2177 __ br(Assembler::EQ, skip);
2178 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2179 __ bind(skip);
2180
2181 } else {
2182 // insert a nop at the start of the prolog so we can patch in a
2183 // branch if we need to invalidate the method later
2184 __ nop();
2185
2186 // TODO 8284443 Avoid creation of temporary frame
2187 if (ra_->C->stub_function() == NULL) {
2188 __ verified_entry(ra_->C, 0);
2189 __ entry_barrier();
2190 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2191 __ remove_frame(framesize, false);
2192 }
2193 // Unpack inline type args passed as oop and then jump to
2194 // the verified entry point (skipping the unverified entry).
2195 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2196 // Emit code for verified entry and save increment for stack repair on return
2197 __ verified_entry(ra_->C, sp_inc);
2198 if (Compile::current()->output()->in_scratch_emit_size()) {
2199 Label dummy_verified_entry;
2200 __ b(dummy_verified_entry);
2201 } else {
2202 __ b(*_verified_entry);
2203 }
2204 }
2205 }
2206
2207 //=============================================================================
2208 #ifndef PRODUCT
2209 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2210 {
2211 st->print_cr("# MachUEPNode");
2212 if (UseCompressedClassPointers) {
2213 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2214 if (CompressedKlassPointers::shift() != 0) {
2215 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2216 }
2217 } else {
2218 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2219 }
2220 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2221 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2222 }
2223 #endif
2224
2225 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2226 {
2227 // This is the unverified entry point.
2228 C2_MacroAssembler _masm(&cbuf);
2229 Label skip;
2230
2231 // UseCompressedClassPointers logic are inside cmp_klass
2232 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2233
2234 // TODO
2235 // can we avoid this skip and still use a reloc?
2236 __ br(Assembler::EQ, skip);
2237 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2238 __ bind(skip);
2239 }
2240
2241 // REQUIRED EMIT CODE
2242
2243 //=============================================================================
2244
2245 // Emit exception handler code.
2246 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2247 {
2248 // mov rscratch1 #exception_blob_entry_point
2249 // br rscratch1
2250 // Note that the code buffer's insts_mark is always relative to insts.
2251 // That's why we must use the macroassembler to generate a handler.
2252 C2_MacroAssembler _masm(&cbuf);
2253 address base = __ start_a_stub(size_exception_handler());
2254 if (base == NULL) {
2255 ciEnv::current()->record_failure("CodeCache is full");
2256 return 0; // CodeBuffer::expand failed
2257 }
2258 int offset = __ offset();
2259 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2260 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2261 __ end_a_stub();
2262 return offset;
2263 }
2264
2265 // Emit deopt handler code.
2266 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2267 {
2268 // Note that the code buffer's insts_mark is always relative to insts.
2269 // That's why we must use the macroassembler to generate a handler.
2270 C2_MacroAssembler _masm(&cbuf);
2271 address base = __ start_a_stub(size_deopt_handler());
2272 if (base == NULL) {
2273 ciEnv::current()->record_failure("CodeCache is full");
2274 return 0; // CodeBuffer::expand failed
2275 }
2276 int offset = __ offset();
2277
2278 __ adr(lr, __ pc());
2279 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2280
2281 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2282 __ end_a_stub();
2283 return offset;
2284 }
2285
2286 // REQUIRED MATCHER CODE
2287
2288 //=============================================================================
2289
2290 const bool Matcher::match_rule_supported(int opcode) {
2291 if (!has_match_rule(opcode))
2292 return false;
2293
2294 bool ret_value = true;
2295 switch (opcode) {
2296 case Op_OnSpinWait:
2297 return VM_Version::supports_on_spin_wait();
2298 case Op_CacheWB:
2299 case Op_CacheWBPreSync:
2300 case Op_CacheWBPostSync:
2301 if (!VM_Version::supports_data_cache_line_flush()) {
2302 ret_value = false;
2303 }
2304 break;
2305 }
2306
2307 return ret_value; // Per default match rules are supported.
2308 }
2309
2310 const RegMask* Matcher::predicate_reg_mask(void) {
2311 return &_PR_REG_mask;
2312 }
2313
2314 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2315 return new TypeVectMask(elemTy, length);
2316 }
2317
2318 // Vector calling convention not yet implemented.
2319 const bool Matcher::supports_vector_calling_convention(void) {
2320 return false;
2321 }
2322
2323 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2324 Unimplemented();
2325 return OptoRegPair(0, 0);
2326 }
2327
2328 // Is this branch offset short enough that a short branch can be used?
2329 //
2330 // NOTE: If the platform does not provide any short branch variants, then
2331 // this method should return false for offset 0.
2332 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2333 // The passed offset is relative to address of the branch.
2334
2335 return (-32768 <= offset && offset < 32768);
2336 }
2337
2338 // Vector width in bytes.
2339 const int Matcher::vector_width_in_bytes(BasicType bt) {
2340 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2341 int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
2342 // Minimum 2 values in vector
2343 if (size < 2*type2aelembytes(bt)) size = 0;
2344 // But never < 4
2345 if (size < 4) size = 0;
2346 return size;
2347 }
2348
2349 // Limits on vector size (number of elements) loaded into vector.
2350 const int Matcher::max_vector_size(const BasicType bt) {
2351 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2352 }
2353
2354 const int Matcher::min_vector_size(const BasicType bt) {
2355 int max_size = max_vector_size(bt);
2356 // Limit the min vector size to 8 bytes.
2357 int size = 8 / type2aelembytes(bt);
2358 if (bt == T_BYTE) {
2359 // To support vector api shuffle/rearrange.
2360 size = 4;
2361 } else if (bt == T_BOOLEAN) {
2362 // To support vector api load/store mask.
2363 size = 2;
2364 }
2365 if (size < 2) size = 2;
2366 return MIN2(size, max_size);
2367 }
2368
2369 // Actual max scalable vector register length.
2370 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
2371 return Matcher::max_vector_size(bt);
2372 }
2373
2374 // Vector ideal reg.
2375 const uint Matcher::vector_ideal_reg(int len) {
2376 if (UseSVE > 0 && 16 < len && len <= 256) {
2377 return Op_VecA;
2378 }
2379 switch(len) {
2380 // For 16-bit/32-bit mask vector, reuse VecD.
2381 case 2:
2382 case 4:
2383 case 8: return Op_VecD;
2384 case 16: return Op_VecX;
2385 }
2386 ShouldNotReachHere();
2387 return 0;
2388 }
2389
2390 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2391 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2392 switch (ideal_reg) {
2393 case Op_VecA: return new vecAOper();
2394 case Op_VecD: return new vecDOper();
2395 case Op_VecX: return new vecXOper();
2396 }
2397 ShouldNotReachHere();
2398 return NULL;
2399 }
2400
2401 bool Matcher::is_reg2reg_move(MachNode* m) {
2402 return false;
2403 }
2404
2405 bool Matcher::is_generic_vector(MachOper* opnd) {
2406 return opnd->opcode() == VREG;
2407 }
2408
2409 // Return whether or not this register is ever used as an argument.
2410 // This function is used on startup to build the trampoline stubs in
2411 // generateOptoStub. Registers not mentioned will be killed by the VM
2412 // call in the trampoline, and arguments in those registers not be
2413 // available to the callee.
2414 bool Matcher::can_be_java_arg(int reg)
2415 {
2416 return
2417 reg == R0_num || reg == R0_H_num ||
2418 reg == R1_num || reg == R1_H_num ||
2419 reg == R2_num || reg == R2_H_num ||
2420 reg == R3_num || reg == R3_H_num ||
2421 reg == R4_num || reg == R4_H_num ||
2422 reg == R5_num || reg == R5_H_num ||
2423 reg == R6_num || reg == R6_H_num ||
2424 reg == R7_num || reg == R7_H_num ||
2425 reg == V0_num || reg == V0_H_num ||
2426 reg == V1_num || reg == V1_H_num ||
2427 reg == V2_num || reg == V2_H_num ||
2428 reg == V3_num || reg == V3_H_num ||
2429 reg == V4_num || reg == V4_H_num ||
2430 reg == V5_num || reg == V5_H_num ||
2431 reg == V6_num || reg == V6_H_num ||
2432 reg == V7_num || reg == V7_H_num;
2433 }
2434
2435 bool Matcher::is_spillable_arg(int reg)
2436 {
2437 return can_be_java_arg(reg);
2438 }
2439
2440 uint Matcher::int_pressure_limit()
2441 {
2442 // JDK-8183543: When taking the number of available registers as int
2443 // register pressure threshold, the jtreg test:
2444 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2445 // failed due to C2 compilation failure with
2446 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2447 //
2448 // A derived pointer is live at CallNode and then is flagged by RA
2449 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2450 // derived pointers and lastly fail to spill after reaching maximum
2451 // number of iterations. Lowering the default pressure threshold to
2452 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2453 // a high register pressure area of the code so that split_DEF can
2454 // generate DefinitionSpillCopy for the derived pointer.
2455 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2456 if (!PreserveFramePointer) {
2457 // When PreserveFramePointer is off, frame pointer is allocatable,
2458 // but different from other SOC registers, it is excluded from
2459 // fatproj's mask because its save type is No-Save. Decrease 1 to
2460 // ensure high pressure at fatproj when PreserveFramePointer is off.
2461 // See check_pressure_at_fatproj().
2462 default_int_pressure_threshold--;
2463 }
2464 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2465 }
2466
2467 uint Matcher::float_pressure_limit()
2468 {
2469 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2470 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2471 }
2472
2473 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2474 return false;
2475 }
2476
2477 RegMask Matcher::divI_proj_mask() {
2478 ShouldNotReachHere();
2479 return RegMask();
2480 }
2481
2482 // Register for MODI projection of divmodI.
2483 RegMask Matcher::modI_proj_mask() {
2484 ShouldNotReachHere();
2485 return RegMask();
2486 }
2487
2488 // Register for DIVL projection of divmodL.
2489 RegMask Matcher::divL_proj_mask() {
2490 ShouldNotReachHere();
2491 return RegMask();
2492 }
2493
2494 // Register for MODL projection of divmodL.
2495 RegMask Matcher::modL_proj_mask() {
2496 ShouldNotReachHere();
2497 return RegMask();
2498 }
2499
2500 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2501 return FP_REG_mask();
2502 }
2503
2504 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2505 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2506 Node* u = addp->fast_out(i);
2507 if (u->is_LoadStore()) {
2508 // On AArch64, LoadStoreNodes (i.e. compare and swap
2509 // instructions) only take register indirect as an operand, so
2510 // any attempt to use an AddPNode as an input to a LoadStoreNode
2511 // must fail.
2512 return false;
2513 }
2514 if (u->is_Mem()) {
2515 int opsize = u->as_Mem()->memory_size();
2516 assert(opsize > 0, "unexpected memory operand size");
2517 if (u->as_Mem()->memory_size() != (1<<shift)) {
2518 return false;
2519 }
2520 }
2521 }
2522 return true;
2523 }
2524
2525 // Binary src (Replicate con)
2526 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2527 if (n == NULL || m == NULL) {
2528 return false;
2529 }
2530
2531 if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
2532 return false;
2533 }
2534
2535 Node* imm_node = m->in(1);
2536 if (!imm_node->is_Con()) {
2537 return false;
2538 }
2539
2540 const Type* t = imm_node->bottom_type();
2541 if (!(t->isa_int() || t->isa_long())) {
2542 return false;
2543 }
2544
2545 switch (n->Opcode()) {
2546 case Op_AndV:
2547 case Op_OrV:
2548 case Op_XorV: {
2549 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2550 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2551 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2552 }
2553 case Op_AddVB:
2554 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2555 case Op_AddVS:
2556 case Op_AddVI:
2557 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2558 case Op_AddVL:
2559 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2560 default:
2561 return false;
2562 }
2563 }
2564
2565 // (XorV src (Replicate m1))
2566 // (XorVMask src (MaskAll m1))
2567 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2568 if (n != NULL && m != NULL) {
2569 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2570 VectorNode::is_all_ones_vector(m);
2571 }
2572 return false;
2573 }
2574
2575 // Should the matcher clone input 'm' of node 'n'?
2576 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2577 if (is_vshift_con_pattern(n, m) ||
2578 is_vector_bitwise_not_pattern(n, m) ||
2579 is_valid_sve_arith_imm_pattern(n, m)) {
2580 mstack.push(m, Visit);
2581 return true;
2582 }
2583 return false;
2584 }
2585
2586 // Should the Matcher clone shifts on addressing modes, expecting them
2587 // to be subsumed into complex addressing expressions or compute them
2588 // into registers?
2589 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2590 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2591 return true;
2592 }
2593
2594 Node *off = m->in(AddPNode::Offset);
2595 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2596 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2597 // Are there other uses besides address expressions?
2598 !is_visited(off)) {
2599 address_visited.set(off->_idx); // Flag as address_visited
2600 mstack.push(off->in(2), Visit);
2601 Node *conv = off->in(1);
2602 if (conv->Opcode() == Op_ConvI2L &&
2603 // Are there other uses besides address expressions?
2604 !is_visited(conv)) {
2605 address_visited.set(conv->_idx); // Flag as address_visited
2606 mstack.push(conv->in(1), Pre_Visit);
2607 } else {
2608 mstack.push(conv, Pre_Visit);
2609 }
2610 address_visited.test_set(m->_idx); // Flag as address_visited
2611 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2612 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2613 return true;
2614 } else if (off->Opcode() == Op_ConvI2L &&
2615 // Are there other uses besides address expressions?
2616 !is_visited(off)) {
2617 address_visited.test_set(m->_idx); // Flag as address_visited
2618 address_visited.set(off->_idx); // Flag as address_visited
2619 mstack.push(off->in(1), Pre_Visit);
2620 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2621 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2622 return true;
2623 }
2624 return false;
2625 }
2626
2627 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2628 C2_MacroAssembler _masm(&cbuf); \
2629 { \
2630 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2631 guarantee(DISP == 0, "mode not permitted for volatile"); \
2632 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2633 __ INSN(REG, as_Register(BASE)); \
2634 }
2635
2636
2637 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2638 {
2639 Address::extend scale;
2640
2641 // Hooboy, this is fugly. We need a way to communicate to the
2642 // encoder that the index needs to be sign extended, so we have to
2643 // enumerate all the cases.
2644 switch (opcode) {
2645 case INDINDEXSCALEDI2L:
2646 case INDINDEXSCALEDI2LN:
2647 case INDINDEXI2L:
2648 case INDINDEXI2LN:
2649 scale = Address::sxtw(size);
2650 break;
2651 default:
2652 scale = Address::lsl(size);
2653 }
2654
2655 if (index == -1) {
2656 return Address(base, disp);
2657 } else {
2658 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2659 return Address(base, as_Register(index), scale);
2660 }
2661 }
2662
2663
2664 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2665 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2666 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2667 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2668 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2669
2670 // Used for all non-volatile memory accesses. The use of
2671 // $mem->opcode() to discover whether this pattern uses sign-extended
2672 // offsets is something of a kludge.
2673 static void loadStore(C2_MacroAssembler masm, mem_insn insn,
2674 Register reg, int opcode,
2675 Register base, int index, int scale, int disp,
2676 int size_in_memory)
2677 {
2678 Address addr = mem2address(opcode, base, index, scale, disp);
2679 if (addr.getMode() == Address::base_plus_offset) {
2680 /* If we get an out-of-range offset it is a bug in the compiler,
2681 so we assert here. */
2682 assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
2683 "c2 compiler bug");
2684 /* Fix up any out-of-range offsets. */
2685 assert_different_registers(rscratch1, base);
2686 assert_different_registers(rscratch1, reg);
2687 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2688 }
2689 (masm.*insn)(reg, addr);
2690 }
2691
2692 static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
2693 FloatRegister reg, int opcode,
2694 Register base, int index, int size, int disp,
2695 int size_in_memory)
2696 {
2697 Address::extend scale;
2698
2699 switch (opcode) {
2700 case INDINDEXSCALEDI2L:
2701 case INDINDEXSCALEDI2LN:
2702 scale = Address::sxtw(size);
2703 break;
2704 default:
2705 scale = Address::lsl(size);
2706 }
2707
2708 if (index == -1) {
2709 /* If we get an out-of-range offset it is a bug in the compiler,
2710 so we assert here. */
2711 assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2712 /* Fix up any out-of-range offsets. */
2713 assert_different_registers(rscratch1, base);
2714 Address addr = Address(base, disp);
2715 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2716 (masm.*insn)(reg, addr);
2717 } else {
2718 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2719 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2720 }
2721 }
2722
2723 static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
2724 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2725 int opcode, Register base, int index, int size, int disp)
2726 {
2727 if (index == -1) {
2728 (masm.*insn)(reg, T, Address(base, disp));
2729 } else {
2730 assert(disp == 0, "unsupported address mode");
2731 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2732 }
2733 }
2734
2735 %}
2736
2737
2738
2739 //----------ENCODING BLOCK-----------------------------------------------------
2740 // This block specifies the encoding classes used by the compiler to
2741 // output byte streams. Encoding classes are parameterized macros
2742 // used by Machine Instruction Nodes in order to generate the bit
2743 // encoding of the instruction. Operands specify their base encoding
2744 // interface with the interface keyword. There are currently
2745 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2746 // COND_INTER. REG_INTER causes an operand to generate a function
2747 // which returns its register number when queried. CONST_INTER causes
2748 // an operand to generate a function which returns the value of the
2749 // constant when queried. MEMORY_INTER causes an operand to generate
2750 // four functions which return the Base Register, the Index Register,
2751 // the Scale Value, and the Offset Value of the operand when queried.
2752 // COND_INTER causes an operand to generate six functions which return
2753 // the encoding code (ie - encoding bits for the instruction)
2754 // associated with each basic boolean condition for a conditional
2755 // instruction.
2756 //
2757 // Instructions specify two basic values for encoding. Again, a
2758 // function is available to check if the constant displacement is an
2759 // oop. They use the ins_encode keyword to specify their encoding
2760 // classes (which must be a sequence of enc_class names, and their
2761 // parameters, specified in the encoding block), and they use the
2762 // opcode keyword to specify, in order, their primary, secondary, and
2763 // tertiary opcode. Only the opcode sections which a particular
2764 // instruction needs for encoding need to be specified.
2765 encode %{
2766 // Build emit functions for each basic byte or larger field in the
2767 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2768 // from C++ code in the enc_class source block. Emit functions will
2769 // live in the main source block for now. In future, we can
2770 // generalize this by adding a syntax that specifies the sizes of
2771 // fields in an order, so that the adlc can build the emit functions
2772 // automagically
2773
2774 // catch all for unimplemented encodings
2775 enc_class enc_unimplemented %{
2776 C2_MacroAssembler _masm(&cbuf);
2777 __ unimplemented("C2 catch all");
2778 %}
2779
2780 // BEGIN Non-volatile memory access
2781
2782 // This encoding class is generated automatically from ad_encode.m4.
2783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2784 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2785 Register dst_reg = as_Register($dst$$reg);
2786 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2787 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2788 %}
2789
2790 // This encoding class is generated automatically from ad_encode.m4.
2791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2792 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2793 Register dst_reg = as_Register($dst$$reg);
2794 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2795 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2796 %}
2797
2798 // This encoding class is generated automatically from ad_encode.m4.
2799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2800 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2801 Register dst_reg = as_Register($dst$$reg);
2802 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2803 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2804 %}
2805
2806 // This encoding class is generated automatically from ad_encode.m4.
2807 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2808 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2809 Register dst_reg = as_Register($dst$$reg);
2810 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2811 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2812 %}
2813
2814 // This encoding class is generated automatically from ad_encode.m4.
2815 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2816 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2817 Register dst_reg = as_Register($dst$$reg);
2818 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2819 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2820 %}
2821
2822 // This encoding class is generated automatically from ad_encode.m4.
2823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2824 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2825 Register dst_reg = as_Register($dst$$reg);
2826 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2827 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2828 %}
2829
2830 // This encoding class is generated automatically from ad_encode.m4.
2831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2832 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2833 Register dst_reg = as_Register($dst$$reg);
2834 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2835 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2836 %}
2837
2838 // This encoding class is generated automatically from ad_encode.m4.
2839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2840 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2841 Register dst_reg = as_Register($dst$$reg);
2842 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2843 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2844 %}
2845
2846 // This encoding class is generated automatically from ad_encode.m4.
2847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2848 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2849 Register dst_reg = as_Register($dst$$reg);
2850 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2851 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2852 %}
2853
2854 // This encoding class is generated automatically from ad_encode.m4.
2855 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2856 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2857 Register dst_reg = as_Register($dst$$reg);
2858 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2859 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2860 %}
2861
2862 // This encoding class is generated automatically from ad_encode.m4.
2863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2864 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2865 Register dst_reg = as_Register($dst$$reg);
2866 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2867 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2868 %}
2869
2870 // This encoding class is generated automatically from ad_encode.m4.
2871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2872 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2873 Register dst_reg = as_Register($dst$$reg);
2874 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2875 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2876 %}
2877
2878 // This encoding class is generated automatically from ad_encode.m4.
2879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2880 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2881 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2882 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2883 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2884 %}
2885
2886 // This encoding class is generated automatically from ad_encode.m4.
2887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2888 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2889 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2890 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2891 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2892 %}
2893
2894 // This encoding class is generated automatically from ad_encode.m4.
2895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2896 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2897 Register src_reg = as_Register($src$$reg);
2898 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2899 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2900 %}
2901
2902 // This encoding class is generated automatically from ad_encode.m4.
2903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2904 enc_class aarch64_enc_strb0(memory1 mem) %{
2905 C2_MacroAssembler _masm(&cbuf);
2906 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2907 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2908 %}
2909
2910 // This encoding class is generated automatically from ad_encode.m4.
2911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2912 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2913 Register src_reg = as_Register($src$$reg);
2914 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2915 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2916 %}
2917
2918 // This encoding class is generated automatically from ad_encode.m4.
2919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2920 enc_class aarch64_enc_strh0(memory2 mem) %{
2921 C2_MacroAssembler _masm(&cbuf);
2922 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2923 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2924 %}
2925
2926 // This encoding class is generated automatically from ad_encode.m4.
2927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2928 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2929 Register src_reg = as_Register($src$$reg);
2930 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2931 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2932 %}
2933
2934 // This encoding class is generated automatically from ad_encode.m4.
2935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2936 enc_class aarch64_enc_strw0(memory4 mem) %{
2937 C2_MacroAssembler _masm(&cbuf);
2938 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2939 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2940 %}
2941
2942 // This encoding class is generated automatically from ad_encode.m4.
2943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2944 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2945 Register src_reg = as_Register($src$$reg);
2946 // we sometimes get asked to store the stack pointer into the
2947 // current thread -- we cannot do that directly on AArch64
2948 if (src_reg == r31_sp) {
2949 C2_MacroAssembler _masm(&cbuf);
2950 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2951 __ mov(rscratch2, sp);
2952 src_reg = rscratch2;
2953 }
2954 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
2955 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2956 %}
2957
2958 // This encoding class is generated automatically from ad_encode.m4.
2959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2960 enc_class aarch64_enc_str0(memory8 mem) %{
2961 C2_MacroAssembler _masm(&cbuf);
2962 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
2963 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2964 %}
2965
2966 // This encoding class is generated automatically from ad_encode.m4.
2967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2968 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
2969 FloatRegister src_reg = as_FloatRegister($src$$reg);
2970 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
2971 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2972 %}
2973
2974 // This encoding class is generated automatically from ad_encode.m4.
2975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2976 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
2977 FloatRegister src_reg = as_FloatRegister($src$$reg);
2978 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
2979 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2980 %}
2981
2982 // This encoding class is generated automatically from ad_encode.m4.
2983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2984 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
2985 C2_MacroAssembler _masm(&cbuf);
2986 __ membar(Assembler::StoreStore);
2987 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2988 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2989 %}
2990
2991 // END Non-volatile memory access
2992
2993 // Vector loads and stores
2994 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
2995 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2996 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
2997 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2998 %}
2999
3000 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3001 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3002 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3003 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3004 %}
3005
3006 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3007 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3008 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3009 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3010 %}
3011
3012 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3013 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3014 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3015 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3016 %}
3017
3018 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3019 FloatRegister src_reg = as_FloatRegister($src$$reg);
3020 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
3021 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3022 %}
3023
3024 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3025 FloatRegister src_reg = as_FloatRegister($src$$reg);
3026 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
3027 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3028 %}
3029
3030 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3031 FloatRegister src_reg = as_FloatRegister($src$$reg);
3032 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
3033 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3034 %}
3035
3036 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3037 FloatRegister src_reg = as_FloatRegister($src$$reg);
3038 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
3039 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3040 %}
3041
3042 // volatile loads and stores
3043
3044 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3045 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3046 rscratch1, stlrb);
3047 %}
3048
3049 enc_class aarch64_enc_stlrb0(memory mem) %{
3050 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3051 rscratch1, stlrb);
3052 %}
3053
3054 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3055 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3056 rscratch1, stlrh);
3057 %}
3058
3059 enc_class aarch64_enc_stlrh0(memory mem) %{
3060 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3061 rscratch1, stlrh);
3062 %}
3063
3064 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3065 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3066 rscratch1, stlrw);
3067 %}
3068
3069 enc_class aarch64_enc_stlrw0(memory mem) %{
3070 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3071 rscratch1, stlrw);
3072 %}
3073
3074 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3075 Register dst_reg = as_Register($dst$$reg);
3076 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3077 rscratch1, ldarb);
3078 __ sxtbw(dst_reg, dst_reg);
3079 %}
3080
3081 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3082 Register dst_reg = as_Register($dst$$reg);
3083 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3084 rscratch1, ldarb);
3085 __ sxtb(dst_reg, dst_reg);
3086 %}
3087
3088 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3089 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3090 rscratch1, ldarb);
3091 %}
3092
3093 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3094 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3095 rscratch1, ldarb);
3096 %}
3097
3098 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3099 Register dst_reg = as_Register($dst$$reg);
3100 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3101 rscratch1, ldarh);
3102 __ sxthw(dst_reg, dst_reg);
3103 %}
3104
3105 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3106 Register dst_reg = as_Register($dst$$reg);
3107 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3108 rscratch1, ldarh);
3109 __ sxth(dst_reg, dst_reg);
3110 %}
3111
3112 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3113 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3114 rscratch1, ldarh);
3115 %}
3116
3117 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3118 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3119 rscratch1, ldarh);
3120 %}
3121
3122 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3123 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3124 rscratch1, ldarw);
3125 %}
3126
3127 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3128 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3129 rscratch1, ldarw);
3130 %}
3131
3132 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3133 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3134 rscratch1, ldar);
3135 %}
3136
3137 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3138 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3139 rscratch1, ldarw);
3140 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3141 %}
3142
3143 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3144 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3145 rscratch1, ldar);
3146 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3147 %}
3148
3149 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3150 Register src_reg = as_Register($src$$reg);
3151 // we sometimes get asked to store the stack pointer into the
3152 // current thread -- we cannot do that directly on AArch64
3153 if (src_reg == r31_sp) {
3154 C2_MacroAssembler _masm(&cbuf);
3155 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3156 __ mov(rscratch2, sp);
3157 src_reg = rscratch2;
3158 }
3159 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, stlr);
3161 %}
3162
3163 enc_class aarch64_enc_stlr0(memory mem) %{
3164 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3165 rscratch1, stlr);
3166 %}
3167
3168 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3169 {
3170 C2_MacroAssembler _masm(&cbuf);
3171 FloatRegister src_reg = as_FloatRegister($src$$reg);
3172 __ fmovs(rscratch2, src_reg);
3173 }
3174 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3175 rscratch1, stlrw);
3176 %}
3177
3178 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3179 {
3180 C2_MacroAssembler _masm(&cbuf);
3181 FloatRegister src_reg = as_FloatRegister($src$$reg);
3182 __ fmovd(rscratch2, src_reg);
3183 }
3184 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3185 rscratch1, stlr);
3186 %}
3187
3188 // synchronized read/update encodings
3189
3190 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3191 C2_MacroAssembler _masm(&cbuf);
3192 Register dst_reg = as_Register($dst$$reg);
3193 Register base = as_Register($mem$$base);
3194 int index = $mem$$index;
3195 int scale = $mem$$scale;
3196 int disp = $mem$$disp;
3197 if (index == -1) {
3198 if (disp != 0) {
3199 __ lea(rscratch1, Address(base, disp));
3200 __ ldaxr(dst_reg, rscratch1);
3201 } else {
3202 // TODO
3203 // should we ever get anything other than this case?
3204 __ ldaxr(dst_reg, base);
3205 }
3206 } else {
3207 Register index_reg = as_Register(index);
3208 if (disp == 0) {
3209 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3210 __ ldaxr(dst_reg, rscratch1);
3211 } else {
3212 __ lea(rscratch1, Address(base, disp));
3213 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3214 __ ldaxr(dst_reg, rscratch1);
3215 }
3216 }
3217 %}
3218
3219 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3220 C2_MacroAssembler _masm(&cbuf);
3221 Register src_reg = as_Register($src$$reg);
3222 Register base = as_Register($mem$$base);
3223 int index = $mem$$index;
3224 int scale = $mem$$scale;
3225 int disp = $mem$$disp;
3226 if (index == -1) {
3227 if (disp != 0) {
3228 __ lea(rscratch2, Address(base, disp));
3229 __ stlxr(rscratch1, src_reg, rscratch2);
3230 } else {
3231 // TODO
3232 // should we ever get anything other than this case?
3233 __ stlxr(rscratch1, src_reg, base);
3234 }
3235 } else {
3236 Register index_reg = as_Register(index);
3237 if (disp == 0) {
3238 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3239 __ stlxr(rscratch1, src_reg, rscratch2);
3240 } else {
3241 __ lea(rscratch2, Address(base, disp));
3242 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3243 __ stlxr(rscratch1, src_reg, rscratch2);
3244 }
3245 }
3246 __ cmpw(rscratch1, zr);
3247 %}
3248
3249 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3250 C2_MacroAssembler _masm(&cbuf);
3251 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3252 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3253 Assembler::xword, /*acquire*/ false, /*release*/ true,
3254 /*weak*/ false, noreg);
3255 %}
3256
3257 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3258 C2_MacroAssembler _masm(&cbuf);
3259 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3260 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3261 Assembler::word, /*acquire*/ false, /*release*/ true,
3262 /*weak*/ false, noreg);
3263 %}
3264
3265 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3266 C2_MacroAssembler _masm(&cbuf);
3267 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3268 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3269 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3270 /*weak*/ false, noreg);
3271 %}
3272
3273 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3274 C2_MacroAssembler _masm(&cbuf);
3275 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3276 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3277 Assembler::byte, /*acquire*/ false, /*release*/ true,
3278 /*weak*/ false, noreg);
3279 %}
3280
3281
3282 // The only difference between aarch64_enc_cmpxchg and
3283 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3284 // CompareAndSwap sequence to serve as a barrier on acquiring a
3285 // lock.
3286 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3287 C2_MacroAssembler _masm(&cbuf);
3288 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3289 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3290 Assembler::xword, /*acquire*/ true, /*release*/ true,
3291 /*weak*/ false, noreg);
3292 %}
3293
3294 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3295 C2_MacroAssembler _masm(&cbuf);
3296 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3297 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3298 Assembler::word, /*acquire*/ true, /*release*/ true,
3299 /*weak*/ false, noreg);
3300 %}
3301
3302 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3303 C2_MacroAssembler _masm(&cbuf);
3304 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3305 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3306 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3307 /*weak*/ false, noreg);
3308 %}
3309
3310 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3311 C2_MacroAssembler _masm(&cbuf);
3312 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3313 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3314 Assembler::byte, /*acquire*/ true, /*release*/ true,
3315 /*weak*/ false, noreg);
3316 %}
3317
3318 // auxiliary used for CompareAndSwapX to set result register
3319 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3320 C2_MacroAssembler _masm(&cbuf);
3321 Register res_reg = as_Register($res$$reg);
3322 __ cset(res_reg, Assembler::EQ);
3323 %}
3324
3325 // prefetch encodings
3326
3327 enc_class aarch64_enc_prefetchw(memory mem) %{
3328 C2_MacroAssembler _masm(&cbuf);
3329 Register base = as_Register($mem$$base);
3330 int index = $mem$$index;
3331 int scale = $mem$$scale;
3332 int disp = $mem$$disp;
3333 if (index == -1) {
3334 __ prfm(Address(base, disp), PSTL1KEEP);
3335 } else {
3336 Register index_reg = as_Register(index);
3337 if (disp == 0) {
3338 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3339 } else {
3340 __ lea(rscratch1, Address(base, disp));
3341 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3342 }
3343 }
3344 %}
3345
3346 /// mov envcodings
3347
3348 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3349 C2_MacroAssembler _masm(&cbuf);
3350 uint32_t con = (uint32_t)$src$$constant;
3351 Register dst_reg = as_Register($dst$$reg);
3352 if (con == 0) {
3353 __ movw(dst_reg, zr);
3354 } else {
3355 __ movw(dst_reg, con);
3356 }
3357 %}
3358
3359 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3360 C2_MacroAssembler _masm(&cbuf);
3361 Register dst_reg = as_Register($dst$$reg);
3362 uint64_t con = (uint64_t)$src$$constant;
3363 if (con == 0) {
3364 __ mov(dst_reg, zr);
3365 } else {
3366 __ mov(dst_reg, con);
3367 }
3368 %}
3369
3370 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3371 C2_MacroAssembler _masm(&cbuf);
3372 Register dst_reg = as_Register($dst$$reg);
3373 address con = (address)$src$$constant;
3374 if (con == NULL || con == (address)1) {
3375 ShouldNotReachHere();
3376 } else {
3377 relocInfo::relocType rtype = $src->constant_reloc();
3378 if (rtype == relocInfo::oop_type) {
3379 __ movoop(dst_reg, (jobject)con);
3380 } else if (rtype == relocInfo::metadata_type) {
3381 __ mov_metadata(dst_reg, (Metadata*)con);
3382 } else {
3383 assert(rtype == relocInfo::none, "unexpected reloc type");
3384 if (! __ is_valid_AArch64_address(con) ||
3385 con < (address)(uintptr_t)os::vm_page_size()) {
3386 __ mov(dst_reg, con);
3387 } else {
3388 uint64_t offset;
3389 __ adrp(dst_reg, con, offset);
3390 __ add(dst_reg, dst_reg, offset);
3391 }
3392 }
3393 }
3394 %}
3395
3396 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3397 C2_MacroAssembler _masm(&cbuf);
3398 Register dst_reg = as_Register($dst$$reg);
3399 __ mov(dst_reg, zr);
3400 %}
3401
3402 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3403 C2_MacroAssembler _masm(&cbuf);
3404 Register dst_reg = as_Register($dst$$reg);
3405 __ mov(dst_reg, (uint64_t)1);
3406 %}
3407
3408 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3409 C2_MacroAssembler _masm(&cbuf);
3410 __ load_byte_map_base($dst$$Register);
3411 %}
3412
3413 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3414 C2_MacroAssembler _masm(&cbuf);
3415 Register dst_reg = as_Register($dst$$reg);
3416 address con = (address)$src$$constant;
3417 if (con == NULL) {
3418 ShouldNotReachHere();
3419 } else {
3420 relocInfo::relocType rtype = $src->constant_reloc();
3421 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3422 __ set_narrow_oop(dst_reg, (jobject)con);
3423 }
3424 %}
3425
3426 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3427 C2_MacroAssembler _masm(&cbuf);
3428 Register dst_reg = as_Register($dst$$reg);
3429 __ mov(dst_reg, zr);
3430 %}
3431
3432 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3433 C2_MacroAssembler _masm(&cbuf);
3434 Register dst_reg = as_Register($dst$$reg);
3435 address con = (address)$src$$constant;
3436 if (con == NULL) {
3437 ShouldNotReachHere();
3438 } else {
3439 relocInfo::relocType rtype = $src->constant_reloc();
3440 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3441 __ set_narrow_klass(dst_reg, (Klass *)con);
3442 }
3443 %}
3444
3445 // arithmetic encodings
3446
3447 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3448 C2_MacroAssembler _masm(&cbuf);
3449 Register dst_reg = as_Register($dst$$reg);
3450 Register src_reg = as_Register($src1$$reg);
3451 int32_t con = (int32_t)$src2$$constant;
3452 // add has primary == 0, subtract has primary == 1
3453 if ($primary) { con = -con; }
3454 if (con < 0) {
3455 __ subw(dst_reg, src_reg, -con);
3456 } else {
3457 __ addw(dst_reg, src_reg, con);
3458 }
3459 %}
3460
3461 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3462 C2_MacroAssembler _masm(&cbuf);
3463 Register dst_reg = as_Register($dst$$reg);
3464 Register src_reg = as_Register($src1$$reg);
3465 int32_t con = (int32_t)$src2$$constant;
3466 // add has primary == 0, subtract has primary == 1
3467 if ($primary) { con = -con; }
3468 if (con < 0) {
3469 __ sub(dst_reg, src_reg, -con);
3470 } else {
3471 __ add(dst_reg, src_reg, con);
3472 }
3473 %}
3474
3475 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3476 C2_MacroAssembler _masm(&cbuf);
3477 Register dst_reg = as_Register($dst$$reg);
3478 Register src1_reg = as_Register($src1$$reg);
3479 Register src2_reg = as_Register($src2$$reg);
3480 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3481 %}
3482
3483 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3484 C2_MacroAssembler _masm(&cbuf);
3485 Register dst_reg = as_Register($dst$$reg);
3486 Register src1_reg = as_Register($src1$$reg);
3487 Register src2_reg = as_Register($src2$$reg);
3488 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3489 %}
3490
3491 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3492 C2_MacroAssembler _masm(&cbuf);
3493 Register dst_reg = as_Register($dst$$reg);
3494 Register src1_reg = as_Register($src1$$reg);
3495 Register src2_reg = as_Register($src2$$reg);
3496 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3497 %}
3498
3499 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3500 C2_MacroAssembler _masm(&cbuf);
3501 Register dst_reg = as_Register($dst$$reg);
3502 Register src1_reg = as_Register($src1$$reg);
3503 Register src2_reg = as_Register($src2$$reg);
3504 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3505 %}
3506
3507 // compare instruction encodings
3508
3509 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3510 C2_MacroAssembler _masm(&cbuf);
3511 Register reg1 = as_Register($src1$$reg);
3512 Register reg2 = as_Register($src2$$reg);
3513 __ cmpw(reg1, reg2);
3514 %}
3515
3516 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3517 C2_MacroAssembler _masm(&cbuf);
3518 Register reg = as_Register($src1$$reg);
3519 int32_t val = $src2$$constant;
3520 if (val >= 0) {
3521 __ subsw(zr, reg, val);
3522 } else {
3523 __ addsw(zr, reg, -val);
3524 }
3525 %}
3526
3527 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3528 C2_MacroAssembler _masm(&cbuf);
3529 Register reg1 = as_Register($src1$$reg);
3530 uint32_t val = (uint32_t)$src2$$constant;
3531 __ movw(rscratch1, val);
3532 __ cmpw(reg1, rscratch1);
3533 %}
3534
3535 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3536 C2_MacroAssembler _masm(&cbuf);
3537 Register reg1 = as_Register($src1$$reg);
3538 Register reg2 = as_Register($src2$$reg);
3539 __ cmp(reg1, reg2);
3540 %}
3541
3542 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3543 C2_MacroAssembler _masm(&cbuf);
3544 Register reg = as_Register($src1$$reg);
3545 int64_t val = $src2$$constant;
3546 if (val >= 0) {
3547 __ subs(zr, reg, val);
3548 } else if (val != -val) {
3549 __ adds(zr, reg, -val);
3550 } else {
3551 // aargh, Long.MIN_VALUE is a special case
3552 __ orr(rscratch1, zr, (uint64_t)val);
3553 __ subs(zr, reg, rscratch1);
3554 }
3555 %}
3556
3557 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3558 C2_MacroAssembler _masm(&cbuf);
3559 Register reg1 = as_Register($src1$$reg);
3560 uint64_t val = (uint64_t)$src2$$constant;
3561 __ mov(rscratch1, val);
3562 __ cmp(reg1, rscratch1);
3563 %}
3564
3565 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3566 C2_MacroAssembler _masm(&cbuf);
3567 Register reg1 = as_Register($src1$$reg);
3568 Register reg2 = as_Register($src2$$reg);
3569 __ cmp(reg1, reg2);
3570 %}
3571
3572 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3573 C2_MacroAssembler _masm(&cbuf);
3574 Register reg1 = as_Register($src1$$reg);
3575 Register reg2 = as_Register($src2$$reg);
3576 __ cmpw(reg1, reg2);
3577 %}
3578
3579 enc_class aarch64_enc_testp(iRegP src) %{
3580 C2_MacroAssembler _masm(&cbuf);
3581 Register reg = as_Register($src$$reg);
3582 __ cmp(reg, zr);
3583 %}
3584
3585 enc_class aarch64_enc_testn(iRegN src) %{
3586 C2_MacroAssembler _masm(&cbuf);
3587 Register reg = as_Register($src$$reg);
3588 __ cmpw(reg, zr);
3589 %}
3590
3591 enc_class aarch64_enc_b(label lbl) %{
3592 C2_MacroAssembler _masm(&cbuf);
3593 Label *L = $lbl$$label;
3594 __ b(*L);
3595 %}
3596
3597 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3598 C2_MacroAssembler _masm(&cbuf);
3599 Label *L = $lbl$$label;
3600 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3601 %}
3602
3603 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3604 C2_MacroAssembler _masm(&cbuf);
3605 Label *L = $lbl$$label;
3606 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3607 %}
3608
3609 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3610 %{
3611 Register sub_reg = as_Register($sub$$reg);
3612 Register super_reg = as_Register($super$$reg);
3613 Register temp_reg = as_Register($temp$$reg);
3614 Register result_reg = as_Register($result$$reg);
3615
3616 Label miss;
3617 C2_MacroAssembler _masm(&cbuf);
3618 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3619 NULL, &miss,
3620 /*set_cond_codes:*/ true);
3621 if ($primary) {
3622 __ mov(result_reg, zr);
3623 }
3624 __ bind(miss);
3625 %}
3626
3627 enc_class aarch64_enc_java_static_call(method meth) %{
3628 C2_MacroAssembler _masm(&cbuf);
3629
3630 address addr = (address)$meth$$method;
3631 address call;
3632 if (!_method) {
3633 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3634 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3635 if (call == NULL) {
3636 ciEnv::current()->record_failure("CodeCache is full");
3637 return;
3638 }
3639 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3640 // The NOP here is purely to ensure that eliding a call to
3641 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3642 __ nop();
3643 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3644 } else {
3645 int method_index = resolved_method_index(cbuf);
3646 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3647 : static_call_Relocation::spec(method_index);
3648 call = __ trampoline_call(Address(addr, rspec));
3649 if (call == NULL) {
3650 ciEnv::current()->record_failure("CodeCache is full");
3651 return;
3652 }
3653 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3654 // Calls of the same statically bound method can share
3655 // a stub to the interpreter.
3656 cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
3657 } else {
3658 // Emit stub for static call
3659 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
3660 if (stub == NULL) {
3661 ciEnv::current()->record_failure("CodeCache is full");
3662 return;
3663 }
3664 }
3665 }
3666
3667 __ post_call_nop();
3668
3669 // Only non uncommon_trap calls need to reinitialize ptrue.
3670 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3671 __ reinitialize_ptrue();
3672 }
3673 %}
3674
3675 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3676 C2_MacroAssembler _masm(&cbuf);
3677 int method_index = resolved_method_index(cbuf);
3678 address call = __ ic_call((address)$meth$$method, method_index);
3679 if (call == NULL) {
3680 ciEnv::current()->record_failure("CodeCache is full");
3681 return;
3682 }
3683 __ post_call_nop();
3684 if (Compile::current()->max_vector_size() > 0) {
3685 __ reinitialize_ptrue();
3686 }
3687 %}
3688
3689 enc_class aarch64_enc_call_epilog() %{
3690 C2_MacroAssembler _masm(&cbuf);
3691 if (VerifyStackAtCalls) {
3692 // Check that stack depth is unchanged: find majik cookie on stack
3693 __ call_Unimplemented();
3694 }
3695 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
3696 if (!_method->signature()->returns_null_free_inline_type()) {
3697 // The last return value is not set by the callee but used to pass IsInit information to compiled code.
3698 // Search for the corresponding projection, get the register and emit code that initialized it.
3699 uint con = (tf()->range_cc()->cnt() - 1);
3700 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3701 ProjNode* proj = fast_out(i)->as_Proj();
3702 if (proj->_con == con) {
3703 // Set IsInit if r0 is non-null (a non-null value is returned buffered or scalarized)
3704 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3705 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3706 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3707 __ cmp(r0, zr);
3708 __ cset(toReg, Assembler::NE);
3709 if (reg->is_stack()) {
3710 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3711 __ str(toReg, Address(sp, st_off));
3712 }
3713 break;
3714 }
3715 }
3716 }
3717 if (return_value_is_used()) {
3718 // An inline type is returned as fields in multiple registers.
3719 // R0 either contains an oop if the inline type is buffered or a pointer
3720 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3721 // if the lowest bit is set to allow C2 to use the oop after null checking.
3722 // r0 &= (r0 & 1) - 1
3723 __ andr(rscratch1, r0, 0x1);
3724 __ sub(rscratch1, rscratch1, 0x1);
3725 __ andr(r0, r0, rscratch1);
3726 }
3727 }
3728 %}
3729
3730 enc_class aarch64_enc_java_to_runtime(method meth) %{
3731 C2_MacroAssembler _masm(&cbuf);
3732
3733 // some calls to generated routines (arraycopy code) are scheduled
3734 // by C2 as runtime calls. if so we can call them using a br (they
3735 // will be in a reachable segment) otherwise we have to use a blr
3736 // which loads the absolute address into a register.
3737 address entry = (address)$meth$$method;
3738 CodeBlob *cb = CodeCache::find_blob(entry);
3739 if (cb) {
3740 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3741 if (call == NULL) {
3742 ciEnv::current()->record_failure("CodeCache is full");
3743 return;
3744 }
3745 __ post_call_nop();
3746 } else {
3747 Label retaddr;
3748 __ adr(rscratch2, retaddr);
3749 __ lea(rscratch1, RuntimeAddress(entry));
3750 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3751 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3752 __ blr(rscratch1);
3753 __ bind(retaddr);
3754 __ post_call_nop();
3755 __ add(sp, sp, 2 * wordSize);
3756 }
3757 if (Compile::current()->max_vector_size() > 0) {
3758 __ reinitialize_ptrue();
3759 }
3760 %}
3761
3762 enc_class aarch64_enc_rethrow() %{
3763 C2_MacroAssembler _masm(&cbuf);
3764 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3765 %}
3766
3767 enc_class aarch64_enc_ret() %{
3768 C2_MacroAssembler _masm(&cbuf);
3769 #ifdef ASSERT
3770 if (Compile::current()->max_vector_size() > 0) {
3771 __ verify_ptrue();
3772 }
3773 #endif
3774 __ ret(lr);
3775 %}
3776
3777 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3778 C2_MacroAssembler _masm(&cbuf);
3779 Register target_reg = as_Register($jump_target$$reg);
3780 __ br(target_reg);
3781 %}
3782
3783 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3784 C2_MacroAssembler _masm(&cbuf);
3785 Register target_reg = as_Register($jump_target$$reg);
3786 // exception oop should be in r0
3787 // ret addr has been popped into lr
3788 // callee expects it in r3
3789 __ mov(r3, lr);
3790 __ br(target_reg);
3791 %}
3792
3793 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3794 C2_MacroAssembler _masm(&cbuf);
3795 Register oop = as_Register($object$$reg);
3796 Register box = as_Register($box$$reg);
3797 Register disp_hdr = as_Register($tmp$$reg);
3798 Register tmp = as_Register($tmp2$$reg);
3799 Label cont;
3800 Label object_has_monitor;
3801 Label no_count;
3802
3803 assert_different_registers(oop, box, tmp, disp_hdr);
3804
3805 // Load markWord from object into displaced_header.
3806 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3807
3808 if (DiagnoseSyncOnValueBasedClasses != 0) {
3809 __ load_klass(tmp, oop);
3810 __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
3811 __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
3812 __ br(Assembler::NE, cont);
3813 }
3814
3815 // Check for existing monitor
3816 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3817
3818 if (!UseHeavyMonitors) {
3819 // Set tmp to be (markWord of object | UNLOCK_VALUE).
3820 __ orr(tmp, disp_hdr, markWord::unlocked_value);
3821
3822 if (EnableValhalla) {
3823 // Mask inline_type bit such that we go to the slow path if object is an inline type
3824 __ andr(tmp, tmp, ~((int) markWord::inline_type_bit_in_place));
3825 }
3826
3827 // Initialize the box. (Must happen before we update the object mark!)
3828 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3829
3830 // Compare object markWord with an unlocked value (tmp) and if
3831 // equal exchange the stack address of our box with object markWord.
3832 // On failure disp_hdr contains the possibly locked markWord.
3833 __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3834 /*release*/ true, /*weak*/ false, disp_hdr);
3835 __ br(Assembler::EQ, cont);
3836
3837 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3838
3839 // If the compare-and-exchange succeeded, then we found an unlocked
3840 // object, will have now locked it will continue at label cont
3841
3842 // Check if the owner is self by comparing the value in the
3843 // markWord of object (disp_hdr) with the stack pointer.
3844 __ mov(rscratch1, sp);
3845 __ sub(disp_hdr, disp_hdr, rscratch1);
3846 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
3847 // If condition is true we are cont and hence we can store 0 as the
3848 // displaced header in the box, which indicates that it is a recursive lock.
3849 __ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
3850 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3851 } else {
3852 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3853 }
3854 __ b(cont);
3855
3856 // Handle existing monitor.
3857 __ bind(object_has_monitor);
3858
3859 // The object's monitor m is unlocked iff m->owner == NULL,
3860 // otherwise m->owner may contain a thread or a stack address.
3861 //
3862 // Try to CAS m->owner from NULL to current thread.
3863 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
3864 __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3865 /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
3866
3867 // Store a non-null value into the box to avoid looking like a re-entrant
3868 // lock. The fast-path monitor unlock code checks for
3869 // markWord::monitor_value so use markWord::unused_mark which has the
3870 // relevant bit set, and also matches ObjectSynchronizer::enter.
3871 __ mov(tmp, (address)markWord::unused_mark().value());
3872 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3873
3874 __ br(Assembler::EQ, cont); // CAS success means locking succeeded
3875
3876 __ cmp(rscratch1, rthread);
3877 __ br(Assembler::NE, cont); // Check for recursive locking
3878
3879 // Recursive lock case
3880 __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
3881 // flag == EQ still from the cmp above, checking if this is a reentrant lock
3882
3883 __ bind(cont);
3884 // flag == EQ indicates success
3885 // flag == NE indicates failure
3886 __ br(Assembler::NE, no_count);
3887
3888 __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
3889
3890 __ bind(no_count);
3891 %}
3892
3893 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3894 C2_MacroAssembler _masm(&cbuf);
3895 Register oop = as_Register($object$$reg);
3896 Register box = as_Register($box$$reg);
3897 Register disp_hdr = as_Register($tmp$$reg);
3898 Register tmp = as_Register($tmp2$$reg);
3899 Label cont;
3900 Label object_has_monitor;
3901 Label no_count;
3902
3903 assert_different_registers(oop, box, tmp, disp_hdr);
3904
3905 if (!UseHeavyMonitors) {
3906 // Find the lock address and load the displaced header from the stack.
3907 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3908
3909 // If the displaced header is 0, we have a recursive unlock.
3910 __ cmp(disp_hdr, zr);
3911 __ br(Assembler::EQ, cont);
3912 }
3913
3914 // Handle existing monitor.
3915 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3916 __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
3917
3918 if (!UseHeavyMonitors) {
3919 // Check if it is still a light weight lock, this is is true if we
3920 // see the stack address of the basicLock in the markWord of the
3921 // object.
3922
3923 __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
3924 /*release*/ true, /*weak*/ false, tmp);
3925 } else {
3926 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3927 }
3928 __ b(cont);
3929
3930 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3931
3932 // Handle existing monitor.
3933 __ bind(object_has_monitor);
3934 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
3935 __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
3936 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3937
3938 Label notRecursive;
3939 __ cbz(disp_hdr, notRecursive);
3940
3941 // Recursive lock
3942 __ sub(disp_hdr, disp_hdr, 1u);
3943 __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3944 __ cmp(disp_hdr, disp_hdr); // Sets flags for result
3945 __ b(cont);
3946
3947 __ bind(notRecursive);
3948 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3949 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3950 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3951 __ cmp(rscratch1, zr); // Sets flags for result
3952 __ cbnz(rscratch1, cont);
3953 // need a release store here
3954 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3955 __ stlr(zr, tmp); // set unowned
3956
3957 __ bind(cont);
3958 // flag == EQ indicates success
3959 // flag == NE indicates failure
3960 __ br(Assembler::NE, no_count);
3961
3962 __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
3963
3964 __ bind(no_count);
3965 %}
3966
3967 %}
3968
3969 //----------FRAME--------------------------------------------------------------
3970 // Definition of frame structure and management information.
3971 //
3972 // S T A C K L A Y O U T Allocators stack-slot number
3973 // | (to get allocators register number
3974 // G Owned by | | v add OptoReg::stack0())
3975 // r CALLER | |
3976 // o | +--------+ pad to even-align allocators stack-slot
3977 // w V | pad0 | numbers; owned by CALLER
3978 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3979 // h ^ | in | 5
3980 // | | args | 4 Holes in incoming args owned by SELF
3981 // | | | | 3
3982 // | | +--------+
3983 // V | | old out| Empty on Intel, window on Sparc
3984 // | old |preserve| Must be even aligned.
3985 // | SP-+--------+----> Matcher::_old_SP, even aligned
3986 // | | in | 3 area for Intel ret address
3987 // Owned by |preserve| Empty on Sparc.
3988 // SELF +--------+
3989 // | | pad2 | 2 pad to align old SP
3990 // | +--------+ 1
3991 // | | locks | 0
3992 // | +--------+----> OptoReg::stack0(), even aligned
3993 // | | pad1 | 11 pad to align new SP
3994 // | +--------+
3995 // | | | 10
3996 // | | spills | 9 spills
3997 // V | | 8 (pad0 slot for callee)
3998 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3999 // ^ | out | 7
4000 // | | args | 6 Holes in outgoing args owned by CALLEE
4001 // Owned by +--------+
4002 // CALLEE | new out| 6 Empty on Intel, window on Sparc
4003 // | new |preserve| Must be even-aligned.
4004 // | SP-+--------+----> Matcher::_new_SP, even aligned
4005 // | | |
4006 //
4007 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
4008 // known from SELF's arguments and the Java calling convention.
4009 // Region 6-7 is determined per call site.
4010 // Note 2: If the calling convention leaves holes in the incoming argument
4011 // area, those holes are owned by SELF. Holes in the outgoing area
4012 // are owned by the CALLEE. Holes should not be necessary in the
4013 // incoming area, as the Java calling convention is completely under
4014 // the control of the AD file. Doubles can be sorted and packed to
4015 // avoid holes. Holes in the outgoing arguments may be necessary for
4016 // varargs C calling conventions.
4017 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
4018 // even aligned with pad0 as needed.
4019 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
4020 // (the latter is true on Intel but is it false on AArch64?)
4021 // region 6-11 is even aligned; it may be padded out more so that
4022 // the region from SP to FP meets the minimum stack alignment.
4023 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
4024 // alignment. Region 11, pad1, may be dynamically extended so that
4025 // SP meets the minimum alignment.
4026
4027 frame %{
4028 // These three registers define part of the calling convention
4029 // between compiled code and the interpreter.
4030
4031 // Inline Cache Register or Method for I2C.
4032 inline_cache_reg(R12);
4033
4034 // Number of stack slots consumed by locking an object
4035 sync_stack_slots(2);
4036
4037 // Compiled code's Frame Pointer
4038 frame_pointer(R31);
4039
4040 // Interpreter stores its frame pointer in a register which is
4041 // stored to the stack by I2CAdaptors.
4042 // I2CAdaptors convert from interpreted java to compiled java.
4043 interpreter_frame_pointer(R29);
4044
4045 // Stack alignment requirement
4046 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4047
4048 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4049 // for calls to C. Supports the var-args backing area for register parms.
4050 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4051
4052 // The after-PROLOG location of the return address. Location of
4053 // return address specifies a type (REG or STACK) and a number
4054 // representing the register number (i.e. - use a register name) or
4055 // stack slot.
4056 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4057 // Otherwise, it is above the locks and verification slot and alignment word
4058 // TODO this may well be correct but need to check why that - 2 is there
4059 // ppc port uses 0 but we definitely need to allow for fixed_slots
4060 // which folds in the space used for monitors
4061 return_addr(STACK - 2 +
4062 align_up((Compile::current()->in_preserve_stack_slots() +
4063 Compile::current()->fixed_slots()),
4064 stack_alignment_in_slots()));
4065
4066 // Location of compiled Java return values. Same as C for now.
4067 return_value
4068 %{
4069 // TODO do we allow ideal_reg == Op_RegN???
4070 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4071 "only return normal values");
4072
4073 static const int lo[Op_RegL + 1] = { // enum name
4074 0, // Op_Node
4075 0, // Op_Set
4076 R0_num, // Op_RegN
4077 R0_num, // Op_RegI
4078 R0_num, // Op_RegP
4079 V0_num, // Op_RegF
4080 V0_num, // Op_RegD
4081 R0_num // Op_RegL
4082 };
4083
4084 static const int hi[Op_RegL + 1] = { // enum name
4085 0, // Op_Node
4086 0, // Op_Set
4087 OptoReg::Bad, // Op_RegN
4088 OptoReg::Bad, // Op_RegI
4089 R0_H_num, // Op_RegP
4090 OptoReg::Bad, // Op_RegF
4091 V0_H_num, // Op_RegD
4092 R0_H_num // Op_RegL
4093 };
4094
4095 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4096 %}
4097 %}
4098
4099 //----------ATTRIBUTES---------------------------------------------------------
4100 //----------Operand Attributes-------------------------------------------------
4101 op_attrib op_cost(1); // Required cost attribute
4102
4103 //----------Instruction Attributes---------------------------------------------
4104 ins_attrib ins_cost(INSN_COST); // Required cost attribute
4105 ins_attrib ins_size(32); // Required size attribute (in bits)
4106 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4107 // a non-matching short branch variant
4108 // of some long branch?
4109 ins_attrib ins_alignment(4); // Required alignment attribute (must
4110 // be a power of 2) specifies the
4111 // alignment that some part of the
4112 // instruction (not necessarily the
4113 // start) requires. If > 1, a
4114 // compute_padding() function must be
4115 // provided for the instruction
4116
4117 //----------OPERANDS-----------------------------------------------------------
4118 // Operand definitions must precede instruction definitions for correct parsing
4119 // in the ADLC because operands constitute user defined types which are used in
4120 // instruction definitions.
4121
4122 //----------Simple Operands----------------------------------------------------
4123
4124 // Integer operands 32 bit
4125 // 32 bit immediate
4126 operand immI()
4127 %{
4128 match(ConI);
4129
4130 op_cost(0);
4131 format %{ %}
4132 interface(CONST_INTER);
4133 %}
4134
4135 // 32 bit zero
4136 operand immI0()
4137 %{
4138 predicate(n->get_int() == 0);
4139 match(ConI);
4140
4141 op_cost(0);
4142 format %{ %}
4143 interface(CONST_INTER);
4144 %}
4145
4146 // 32 bit unit increment
4147 operand immI_1()
4148 %{
4149 predicate(n->get_int() == 1);
4150 match(ConI);
4151
4152 op_cost(0);
4153 format %{ %}
4154 interface(CONST_INTER);
4155 %}
4156
4157 // 32 bit unit decrement
4158 operand immI_M1()
4159 %{
4160 predicate(n->get_int() == -1);
4161 match(ConI);
4162
4163 op_cost(0);
4164 format %{ %}
4165 interface(CONST_INTER);
4166 %}
4167
4168 // Shift values for add/sub extension shift
4169 operand immIExt()
4170 %{
4171 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4172 match(ConI);
4173
4174 op_cost(0);
4175 format %{ %}
4176 interface(CONST_INTER);
4177 %}
4178
4179 operand immI_gt_1()
4180 %{
4181 predicate(n->get_int() > 1);
4182 match(ConI);
4183
4184 op_cost(0);
4185 format %{ %}
4186 interface(CONST_INTER);
4187 %}
4188
4189 operand immI_le_4()
4190 %{
4191 predicate(n->get_int() <= 4);
4192 match(ConI);
4193
4194 op_cost(0);
4195 format %{ %}
4196 interface(CONST_INTER);
4197 %}
4198
4199 operand immI_16()
4200 %{
4201 predicate(n->get_int() == 16);
4202 match(ConI);
4203
4204 op_cost(0);
4205 format %{ %}
4206 interface(CONST_INTER);
4207 %}
4208
4209 operand immI_24()
4210 %{
4211 predicate(n->get_int() == 24);
4212 match(ConI);
4213
4214 op_cost(0);
4215 format %{ %}
4216 interface(CONST_INTER);
4217 %}
4218
4219 operand immI_32()
4220 %{
4221 predicate(n->get_int() == 32);
4222 match(ConI);
4223
4224 op_cost(0);
4225 format %{ %}
4226 interface(CONST_INTER);
4227 %}
4228
4229 operand immI_48()
4230 %{
4231 predicate(n->get_int() == 48);
4232 match(ConI);
4233
4234 op_cost(0);
4235 format %{ %}
4236 interface(CONST_INTER);
4237 %}
4238
4239 operand immI_56()
4240 %{
4241 predicate(n->get_int() == 56);
4242 match(ConI);
4243
4244 op_cost(0);
4245 format %{ %}
4246 interface(CONST_INTER);
4247 %}
4248
4249 operand immI_63()
4250 %{
4251 predicate(n->get_int() == 63);
4252 match(ConI);
4253
4254 op_cost(0);
4255 format %{ %}
4256 interface(CONST_INTER);
4257 %}
4258
4259 operand immI_64()
4260 %{
4261 predicate(n->get_int() == 64);
4262 match(ConI);
4263
4264 op_cost(0);
4265 format %{ %}
4266 interface(CONST_INTER);
4267 %}
4268
4269 operand immI_255()
4270 %{
4271 predicate(n->get_int() == 255);
4272 match(ConI);
4273
4274 op_cost(0);
4275 format %{ %}
4276 interface(CONST_INTER);
4277 %}
4278
4279 operand immI_65535()
4280 %{
4281 predicate(n->get_int() == 65535);
4282 match(ConI);
4283
4284 op_cost(0);
4285 format %{ %}
4286 interface(CONST_INTER);
4287 %}
4288
4289 operand immI_positive()
4290 %{
4291 predicate(n->get_int() > 0);
4292 match(ConI);
4293
4294 op_cost(0);
4295 format %{ %}
4296 interface(CONST_INTER);
4297 %}
4298
4299 operand immL_255()
4300 %{
4301 predicate(n->get_long() == 255L);
4302 match(ConL);
4303
4304 op_cost(0);
4305 format %{ %}
4306 interface(CONST_INTER);
4307 %}
4308
4309 operand immL_65535()
4310 %{
4311 predicate(n->get_long() == 65535L);
4312 match(ConL);
4313
4314 op_cost(0);
4315 format %{ %}
4316 interface(CONST_INTER);
4317 %}
4318
4319 operand immL_4294967295()
4320 %{
4321 predicate(n->get_long() == 4294967295L);
4322 match(ConL);
4323
4324 op_cost(0);
4325 format %{ %}
4326 interface(CONST_INTER);
4327 %}
4328
4329 operand immL_bitmask()
4330 %{
4331 predicate((n->get_long() != 0)
4332 && ((n->get_long() & 0xc000000000000000l) == 0)
4333 && is_power_of_2(n->get_long() + 1));
4334 match(ConL);
4335
4336 op_cost(0);
4337 format %{ %}
4338 interface(CONST_INTER);
4339 %}
4340
4341 operand immI_bitmask()
4342 %{
4343 predicate((n->get_int() != 0)
4344 && ((n->get_int() & 0xc0000000) == 0)
4345 && is_power_of_2(n->get_int() + 1));
4346 match(ConI);
4347
4348 op_cost(0);
4349 format %{ %}
4350 interface(CONST_INTER);
4351 %}
4352
4353 operand immL_positive_bitmaskI()
4354 %{
4355 predicate((n->get_long() != 0)
4356 && ((julong)n->get_long() < 0x80000000ULL)
4357 && is_power_of_2(n->get_long() + 1));
4358 match(ConL);
4359
4360 op_cost(0);
4361 format %{ %}
4362 interface(CONST_INTER);
4363 %}
4364
4365 // Scale values for scaled offset addressing modes (up to long but not quad)
4366 operand immIScale()
4367 %{
4368 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4369 match(ConI);
4370
4371 op_cost(0);
4372 format %{ %}
4373 interface(CONST_INTER);
4374 %}
4375
4376 // 26 bit signed offset -- for pc-relative branches
4377 operand immI26()
4378 %{
4379 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4380 match(ConI);
4381
4382 op_cost(0);
4383 format %{ %}
4384 interface(CONST_INTER);
4385 %}
4386
4387 // 19 bit signed offset -- for pc-relative loads
4388 operand immI19()
4389 %{
4390 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4391 match(ConI);
4392
4393 op_cost(0);
4394 format %{ %}
4395 interface(CONST_INTER);
4396 %}
4397
4398 // 12 bit unsigned offset -- for base plus immediate loads
4399 operand immIU12()
4400 %{
4401 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4402 match(ConI);
4403
4404 op_cost(0);
4405 format %{ %}
4406 interface(CONST_INTER);
4407 %}
4408
4409 operand immLU12()
4410 %{
4411 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4412 match(ConL);
4413
4414 op_cost(0);
4415 format %{ %}
4416 interface(CONST_INTER);
4417 %}
4418
4419 // Offset for scaled or unscaled immediate loads and stores
4420 operand immIOffset()
4421 %{
4422 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4423 match(ConI);
4424
4425 op_cost(0);
4426 format %{ %}
4427 interface(CONST_INTER);
4428 %}
4429
4430 operand immIOffset1()
4431 %{
4432 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4433 match(ConI);
4434
4435 op_cost(0);
4436 format %{ %}
4437 interface(CONST_INTER);
4438 %}
4439
4440 operand immIOffset2()
4441 %{
4442 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4443 match(ConI);
4444
4445 op_cost(0);
4446 format %{ %}
4447 interface(CONST_INTER);
4448 %}
4449
4450 operand immIOffset4()
4451 %{
4452 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4453 match(ConI);
4454
4455 op_cost(0);
4456 format %{ %}
4457 interface(CONST_INTER);
4458 %}
4459
4460 operand immIOffset8()
4461 %{
4462 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4463 match(ConI);
4464
4465 op_cost(0);
4466 format %{ %}
4467 interface(CONST_INTER);
4468 %}
4469
4470 operand immIOffset16()
4471 %{
4472 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4473 match(ConI);
4474
4475 op_cost(0);
4476 format %{ %}
4477 interface(CONST_INTER);
4478 %}
4479
4480 operand immLoffset()
4481 %{
4482 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4483 match(ConL);
4484
4485 op_cost(0);
4486 format %{ %}
4487 interface(CONST_INTER);
4488 %}
4489
4490 operand immLoffset1()
4491 %{
4492 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4493 match(ConL);
4494
4495 op_cost(0);
4496 format %{ %}
4497 interface(CONST_INTER);
4498 %}
4499
4500 operand immLoffset2()
4501 %{
4502 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4503 match(ConL);
4504
4505 op_cost(0);
4506 format %{ %}
4507 interface(CONST_INTER);
4508 %}
4509
4510 operand immLoffset4()
4511 %{
4512 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4513 match(ConL);
4514
4515 op_cost(0);
4516 format %{ %}
4517 interface(CONST_INTER);
4518 %}
4519
4520 operand immLoffset8()
4521 %{
4522 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4523 match(ConL);
4524
4525 op_cost(0);
4526 format %{ %}
4527 interface(CONST_INTER);
4528 %}
4529
4530 operand immLoffset16()
4531 %{
4532 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4533 match(ConL);
4534
4535 op_cost(0);
4536 format %{ %}
4537 interface(CONST_INTER);
4538 %}
4539
4540 // 8 bit signed value.
4541 operand immI8()
4542 %{
4543 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4544 match(ConI);
4545
4546 op_cost(0);
4547 format %{ %}
4548 interface(CONST_INTER);
4549 %}
4550
4551 // 8 bit signed value (simm8), or #simm8 LSL 8.
4552 operand immI8_shift8()
4553 %{
4554 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4555 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4556 match(ConI);
4557
4558 op_cost(0);
4559 format %{ %}
4560 interface(CONST_INTER);
4561 %}
4562
4563 // 8 bit signed value (simm8), or #simm8 LSL 8.
4564 operand immL8_shift8()
4565 %{
4566 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4567 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4568 match(ConL);
4569
4570 op_cost(0);
4571 format %{ %}
4572 interface(CONST_INTER);
4573 %}
4574
4575 // 8 bit integer valid for vector add sub immediate
4576 operand immBAddSubV()
4577 %{
4578 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4579 match(ConI);
4580
4581 op_cost(0);
4582 format %{ %}
4583 interface(CONST_INTER);
4584 %}
4585
4586 // 32 bit integer valid for add sub immediate
4587 operand immIAddSub()
4588 %{
4589 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4590 match(ConI);
4591 op_cost(0);
4592 format %{ %}
4593 interface(CONST_INTER);
4594 %}
4595
4596 // 32 bit integer valid for vector add sub immediate
4597 operand immIAddSubV()
4598 %{
4599 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4600 match(ConI);
4601
4602 op_cost(0);
4603 format %{ %}
4604 interface(CONST_INTER);
4605 %}
4606
4607 // 32 bit unsigned integer valid for logical immediate
4608
4609 operand immBLog()
4610 %{
4611 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4612 match(ConI);
4613
4614 op_cost(0);
4615 format %{ %}
4616 interface(CONST_INTER);
4617 %}
4618
4619 operand immSLog()
4620 %{
4621 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4622 match(ConI);
4623
4624 op_cost(0);
4625 format %{ %}
4626 interface(CONST_INTER);
4627 %}
4628
4629 operand immILog()
4630 %{
4631 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4632 match(ConI);
4633
4634 op_cost(0);
4635 format %{ %}
4636 interface(CONST_INTER);
4637 %}
4638
4639 // Integer operands 64 bit
4640 // 64 bit immediate
4641 operand immL()
4642 %{
4643 match(ConL);
4644
4645 op_cost(0);
4646 format %{ %}
4647 interface(CONST_INTER);
4648 %}
4649
4650 // 64 bit zero
4651 operand immL0()
4652 %{
4653 predicate(n->get_long() == 0);
4654 match(ConL);
4655
4656 op_cost(0);
4657 format %{ %}
4658 interface(CONST_INTER);
4659 %}
4660
4661 // 64 bit unit increment
4662 operand immL_1()
4663 %{
4664 predicate(n->get_long() == 1);
4665 match(ConL);
4666
4667 op_cost(0);
4668 format %{ %}
4669 interface(CONST_INTER);
4670 %}
4671
4672 // 64 bit unit decrement
4673 operand immL_M1()
4674 %{
4675 predicate(n->get_long() == -1);
4676 match(ConL);
4677
4678 op_cost(0);
4679 format %{ %}
4680 interface(CONST_INTER);
4681 %}
4682
4683 // 32 bit offset of pc in thread anchor
4684
4685 operand immL_pc_off()
4686 %{
4687 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4688 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4689 match(ConL);
4690
4691 op_cost(0);
4692 format %{ %}
4693 interface(CONST_INTER);
4694 %}
4695
4696 // 64 bit integer valid for add sub immediate
4697 operand immLAddSub()
4698 %{
4699 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4700 match(ConL);
4701 op_cost(0);
4702 format %{ %}
4703 interface(CONST_INTER);
4704 %}
4705
4706 // 64 bit integer valid for addv subv immediate
4707 operand immLAddSubV()
4708 %{
4709 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4710 match(ConL);
4711
4712 op_cost(0);
4713 format %{ %}
4714 interface(CONST_INTER);
4715 %}
4716
4717 // 64 bit integer valid for logical immediate
4718 operand immLLog()
4719 %{
4720 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4721 match(ConL);
4722 op_cost(0);
4723 format %{ %}
4724 interface(CONST_INTER);
4725 %}
4726
4727 // Long Immediate: low 32-bit mask
4728 operand immL_32bits()
4729 %{
4730 predicate(n->get_long() == 0xFFFFFFFFL);
4731 match(ConL);
4732 op_cost(0);
4733 format %{ %}
4734 interface(CONST_INTER);
4735 %}
4736
4737 // Pointer operands
4738 // Pointer Immediate
4739 operand immP()
4740 %{
4741 match(ConP);
4742
4743 op_cost(0);
4744 format %{ %}
4745 interface(CONST_INTER);
4746 %}
4747
4748 // NULL Pointer Immediate
4749 operand immP0()
4750 %{
4751 predicate(n->get_ptr() == 0);
4752 match(ConP);
4753
4754 op_cost(0);
4755 format %{ %}
4756 interface(CONST_INTER);
4757 %}
4758
4759 // Pointer Immediate One
4760 // this is used in object initialization (initial object header)
4761 operand immP_1()
4762 %{
4763 predicate(n->get_ptr() == 1);
4764 match(ConP);
4765
4766 op_cost(0);
4767 format %{ %}
4768 interface(CONST_INTER);
4769 %}
4770
4771 // Card Table Byte Map Base
4772 operand immByteMapBase()
4773 %{
4774 // Get base of card map
4775 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4776 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4777 match(ConP);
4778
4779 op_cost(0);
4780 format %{ %}
4781 interface(CONST_INTER);
4782 %}
4783
4784 // Pointer Immediate Minus One
4785 // this is used when we want to write the current PC to the thread anchor
4786 operand immP_M1()
4787 %{
4788 predicate(n->get_ptr() == -1);
4789 match(ConP);
4790
4791 op_cost(0);
4792 format %{ %}
4793 interface(CONST_INTER);
4794 %}
4795
4796 // Pointer Immediate Minus Two
4797 // this is used when we want to write the current PC to the thread anchor
4798 operand immP_M2()
4799 %{
4800 predicate(n->get_ptr() == -2);
4801 match(ConP);
4802
4803 op_cost(0);
4804 format %{ %}
4805 interface(CONST_INTER);
4806 %}
4807
4808 // Float and Double operands
4809 // Double Immediate
4810 operand immD()
4811 %{
4812 match(ConD);
4813 op_cost(0);
4814 format %{ %}
4815 interface(CONST_INTER);
4816 %}
4817
4818 // Double Immediate: +0.0d
4819 operand immD0()
4820 %{
4821 predicate(jlong_cast(n->getd()) == 0);
4822 match(ConD);
4823
4824 op_cost(0);
4825 format %{ %}
4826 interface(CONST_INTER);
4827 %}
4828
4829 // constant 'double +0.0'.
4830 operand immDPacked()
4831 %{
4832 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4833 match(ConD);
4834 op_cost(0);
4835 format %{ %}
4836 interface(CONST_INTER);
4837 %}
4838
4839 // Float Immediate
4840 operand immF()
4841 %{
4842 match(ConF);
4843 op_cost(0);
4844 format %{ %}
4845 interface(CONST_INTER);
4846 %}
4847
4848 // Float Immediate: +0.0f.
4849 operand immF0()
4850 %{
4851 predicate(jint_cast(n->getf()) == 0);
4852 match(ConF);
4853
4854 op_cost(0);
4855 format %{ %}
4856 interface(CONST_INTER);
4857 %}
4858
4859 //
4860 operand immFPacked()
4861 %{
4862 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4863 match(ConF);
4864 op_cost(0);
4865 format %{ %}
4866 interface(CONST_INTER);
4867 %}
4868
4869 // Narrow pointer operands
4870 // Narrow Pointer Immediate
4871 operand immN()
4872 %{
4873 match(ConN);
4874
4875 op_cost(0);
4876 format %{ %}
4877 interface(CONST_INTER);
4878 %}
4879
4880 // Narrow NULL Pointer Immediate
4881 operand immN0()
4882 %{
4883 predicate(n->get_narrowcon() == 0);
4884 match(ConN);
4885
4886 op_cost(0);
4887 format %{ %}
4888 interface(CONST_INTER);
4889 %}
4890
4891 operand immNKlass()
4892 %{
4893 match(ConNKlass);
4894
4895 op_cost(0);
4896 format %{ %}
4897 interface(CONST_INTER);
4898 %}
4899
4900 // Integer 32 bit Register Operands
4901 // Integer 32 bitRegister (excludes SP)
4902 operand iRegI()
4903 %{
4904 constraint(ALLOC_IN_RC(any_reg32));
4905 match(RegI);
4906 match(iRegINoSp);
4907 op_cost(0);
4908 format %{ %}
4909 interface(REG_INTER);
4910 %}
4911
4912 // Integer 32 bit Register not Special
4913 operand iRegINoSp()
4914 %{
4915 constraint(ALLOC_IN_RC(no_special_reg32));
4916 match(RegI);
4917 op_cost(0);
4918 format %{ %}
4919 interface(REG_INTER);
4920 %}
4921
4922 // Integer 64 bit Register Operands
4923 // Integer 64 bit Register (includes SP)
4924 operand iRegL()
4925 %{
4926 constraint(ALLOC_IN_RC(any_reg));
4927 match(RegL);
4928 match(iRegLNoSp);
4929 op_cost(0);
4930 format %{ %}
4931 interface(REG_INTER);
4932 %}
4933
4934 // Integer 64 bit Register not Special
4935 operand iRegLNoSp()
4936 %{
4937 constraint(ALLOC_IN_RC(no_special_reg));
4938 match(RegL);
4939 match(iRegL_R0);
4940 format %{ %}
4941 interface(REG_INTER);
4942 %}
4943
4944 // Pointer Register Operands
4945 // Pointer Register
4946 operand iRegP()
4947 %{
4948 constraint(ALLOC_IN_RC(ptr_reg));
4949 match(RegP);
4950 match(iRegPNoSp);
4951 match(iRegP_R0);
4952 //match(iRegP_R2);
4953 //match(iRegP_R4);
4954 match(iRegP_R5);
4955 match(thread_RegP);
4956 op_cost(0);
4957 format %{ %}
4958 interface(REG_INTER);
4959 %}
4960
4961 // Pointer 64 bit Register not Special
4962 operand iRegPNoSp()
4963 %{
4964 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4965 match(RegP);
4966 // match(iRegP);
4967 // match(iRegP_R0);
4968 // match(iRegP_R2);
4969 // match(iRegP_R4);
4970 // match(iRegP_R5);
4971 // match(thread_RegP);
4972 op_cost(0);
4973 format %{ %}
4974 interface(REG_INTER);
4975 %}
4976
4977 // Pointer 64 bit Register R0 only
4978 operand iRegP_R0()
4979 %{
4980 constraint(ALLOC_IN_RC(r0_reg));
4981 match(RegP);
4982 // match(iRegP);
4983 match(iRegPNoSp);
4984 op_cost(0);
4985 format %{ %}
4986 interface(REG_INTER);
4987 %}
4988
4989 // Pointer 64 bit Register R1 only
4990 operand iRegP_R1()
4991 %{
4992 constraint(ALLOC_IN_RC(r1_reg));
4993 match(RegP);
4994 // match(iRegP);
4995 match(iRegPNoSp);
4996 op_cost(0);
4997 format %{ %}
4998 interface(REG_INTER);
4999 %}
5000
5001 // Pointer 64 bit Register R2 only
5002 operand iRegP_R2()
5003 %{
5004 constraint(ALLOC_IN_RC(r2_reg));
5005 match(RegP);
5006 // match(iRegP);
5007 match(iRegPNoSp);
5008 op_cost(0);
5009 format %{ %}
5010 interface(REG_INTER);
5011 %}
5012
5013 // Pointer 64 bit Register R3 only
5014 operand iRegP_R3()
5015 %{
5016 constraint(ALLOC_IN_RC(r3_reg));
5017 match(RegP);
5018 // match(iRegP);
5019 match(iRegPNoSp);
5020 op_cost(0);
5021 format %{ %}
5022 interface(REG_INTER);
5023 %}
5024
5025 // Pointer 64 bit Register R4 only
5026 operand iRegP_R4()
5027 %{
5028 constraint(ALLOC_IN_RC(r4_reg));
5029 match(RegP);
5030 // match(iRegP);
5031 match(iRegPNoSp);
5032 op_cost(0);
5033 format %{ %}
5034 interface(REG_INTER);
5035 %}
5036
5037 // Pointer 64 bit Register R5 only
5038 operand iRegP_R5()
5039 %{
5040 constraint(ALLOC_IN_RC(r5_reg));
5041 match(RegP);
5042 // match(iRegP);
5043 match(iRegPNoSp);
5044 op_cost(0);
5045 format %{ %}
5046 interface(REG_INTER);
5047 %}
5048
5049 // Pointer 64 bit Register R10 only
5050 operand iRegP_R10()
5051 %{
5052 constraint(ALLOC_IN_RC(r10_reg));
5053 match(RegP);
5054 // match(iRegP);
5055 match(iRegPNoSp);
5056 op_cost(0);
5057 format %{ %}
5058 interface(REG_INTER);
5059 %}
5060
5061 // Long 64 bit Register R0 only
5062 operand iRegL_R0()
5063 %{
5064 constraint(ALLOC_IN_RC(r0_reg));
5065 match(RegL);
5066 match(iRegLNoSp);
5067 op_cost(0);
5068 format %{ %}
5069 interface(REG_INTER);
5070 %}
5071
5072 // Long 64 bit Register R2 only
5073 operand iRegL_R2()
5074 %{
5075 constraint(ALLOC_IN_RC(r2_reg));
5076 match(RegL);
5077 match(iRegLNoSp);
5078 op_cost(0);
5079 format %{ %}
5080 interface(REG_INTER);
5081 %}
5082
5083 // Long 64 bit Register R3 only
5084 operand iRegL_R3()
5085 %{
5086 constraint(ALLOC_IN_RC(r3_reg));
5087 match(RegL);
5088 match(iRegLNoSp);
5089 op_cost(0);
5090 format %{ %}
5091 interface(REG_INTER);
5092 %}
5093
5094 // Long 64 bit Register R11 only
5095 operand iRegL_R11()
5096 %{
5097 constraint(ALLOC_IN_RC(r11_reg));
5098 match(RegL);
5099 match(iRegLNoSp);
5100 op_cost(0);
5101 format %{ %}
5102 interface(REG_INTER);
5103 %}
5104
5105 // Pointer 64 bit Register FP only
5106 operand iRegP_FP()
5107 %{
5108 constraint(ALLOC_IN_RC(fp_reg));
5109 match(RegP);
5110 // match(iRegP);
5111 op_cost(0);
5112 format %{ %}
5113 interface(REG_INTER);
5114 %}
5115
5116 // Register R0 only
5117 operand iRegI_R0()
5118 %{
5119 constraint(ALLOC_IN_RC(int_r0_reg));
5120 match(RegI);
5121 match(iRegINoSp);
5122 op_cost(0);
5123 format %{ %}
5124 interface(REG_INTER);
5125 %}
5126
5127 // Register R2 only
5128 operand iRegI_R2()
5129 %{
5130 constraint(ALLOC_IN_RC(int_r2_reg));
5131 match(RegI);
5132 match(iRegINoSp);
5133 op_cost(0);
5134 format %{ %}
5135 interface(REG_INTER);
5136 %}
5137
5138 // Register R3 only
5139 operand iRegI_R3()
5140 %{
5141 constraint(ALLOC_IN_RC(int_r3_reg));
5142 match(RegI);
5143 match(iRegINoSp);
5144 op_cost(0);
5145 format %{ %}
5146 interface(REG_INTER);
5147 %}
5148
5149
5150 // Register R4 only
5151 operand iRegI_R4()
5152 %{
5153 constraint(ALLOC_IN_RC(int_r4_reg));
5154 match(RegI);
5155 match(iRegINoSp);
5156 op_cost(0);
5157 format %{ %}
5158 interface(REG_INTER);
5159 %}
5160
5161
5162 // Pointer Register Operands
5163 // Narrow Pointer Register
5164 operand iRegN()
5165 %{
5166 constraint(ALLOC_IN_RC(any_reg32));
5167 match(RegN);
5168 match(iRegNNoSp);
5169 op_cost(0);
5170 format %{ %}
5171 interface(REG_INTER);
5172 %}
5173
5174 operand iRegN_R0()
5175 %{
5176 constraint(ALLOC_IN_RC(r0_reg));
5177 match(iRegN);
5178 op_cost(0);
5179 format %{ %}
5180 interface(REG_INTER);
5181 %}
5182
5183 operand iRegN_R2()
5184 %{
5185 constraint(ALLOC_IN_RC(r2_reg));
5186 match(iRegN);
5187 op_cost(0);
5188 format %{ %}
5189 interface(REG_INTER);
5190 %}
5191
5192 operand iRegN_R3()
5193 %{
5194 constraint(ALLOC_IN_RC(r3_reg));
5195 match(iRegN);
5196 op_cost(0);
5197 format %{ %}
5198 interface(REG_INTER);
5199 %}
5200
5201 // Integer 64 bit Register not Special
5202 operand iRegNNoSp()
5203 %{
5204 constraint(ALLOC_IN_RC(no_special_reg32));
5205 match(RegN);
5206 op_cost(0);
5207 format %{ %}
5208 interface(REG_INTER);
5209 %}
5210
5211 // heap base register -- used for encoding immN0
5212
5213 operand iRegIHeapbase()
5214 %{
5215 constraint(ALLOC_IN_RC(heapbase_reg));
5216 match(RegI);
5217 op_cost(0);
5218 format %{ %}
5219 interface(REG_INTER);
5220 %}
5221
5222 // Float Register
5223 // Float register operands
5224 operand vRegF()
5225 %{
5226 constraint(ALLOC_IN_RC(float_reg));
5227 match(RegF);
5228
5229 op_cost(0);
5230 format %{ %}
5231 interface(REG_INTER);
5232 %}
5233
5234 // Double Register
5235 // Double register operands
5236 operand vRegD()
5237 %{
5238 constraint(ALLOC_IN_RC(double_reg));
5239 match(RegD);
5240
5241 op_cost(0);
5242 format %{ %}
5243 interface(REG_INTER);
5244 %}
5245
5246 // Generic vector class. This will be used for
5247 // all vector operands, including NEON and SVE.
5248 operand vReg()
5249 %{
5250 constraint(ALLOC_IN_RC(dynamic));
5251 match(VecA);
5252 match(VecD);
5253 match(VecX);
5254
5255 op_cost(0);
5256 format %{ %}
5257 interface(REG_INTER);
5258 %}
5259
5260 operand vecA()
5261 %{
5262 constraint(ALLOC_IN_RC(vectora_reg));
5263 match(VecA);
5264
5265 op_cost(0);
5266 format %{ %}
5267 interface(REG_INTER);
5268 %}
5269
5270 operand vecD()
5271 %{
5272 constraint(ALLOC_IN_RC(vectord_reg));
5273 match(VecD);
5274
5275 op_cost(0);
5276 format %{ %}
5277 interface(REG_INTER);
5278 %}
5279
5280 operand vecX()
5281 %{
5282 constraint(ALLOC_IN_RC(vectorx_reg));
5283 match(VecX);
5284
5285 op_cost(0);
5286 format %{ %}
5287 interface(REG_INTER);
5288 %}
5289
5290 operand vRegD_V0()
5291 %{
5292 constraint(ALLOC_IN_RC(v0_reg));
5293 match(RegD);
5294 op_cost(0);
5295 format %{ %}
5296 interface(REG_INTER);
5297 %}
5298
5299 operand vRegD_V1()
5300 %{
5301 constraint(ALLOC_IN_RC(v1_reg));
5302 match(RegD);
5303 op_cost(0);
5304 format %{ %}
5305 interface(REG_INTER);
5306 %}
5307
5308 operand vRegD_V2()
5309 %{
5310 constraint(ALLOC_IN_RC(v2_reg));
5311 match(RegD);
5312 op_cost(0);
5313 format %{ %}
5314 interface(REG_INTER);
5315 %}
5316
5317 operand vRegD_V3()
5318 %{
5319 constraint(ALLOC_IN_RC(v3_reg));
5320 match(RegD);
5321 op_cost(0);
5322 format %{ %}
5323 interface(REG_INTER);
5324 %}
5325
5326 operand vRegD_V4()
5327 %{
5328 constraint(ALLOC_IN_RC(v4_reg));
5329 match(RegD);
5330 op_cost(0);
5331 format %{ %}
5332 interface(REG_INTER);
5333 %}
5334
5335 operand vRegD_V5()
5336 %{
5337 constraint(ALLOC_IN_RC(v5_reg));
5338 match(RegD);
5339 op_cost(0);
5340 format %{ %}
5341 interface(REG_INTER);
5342 %}
5343
5344 operand vRegD_V6()
5345 %{
5346 constraint(ALLOC_IN_RC(v6_reg));
5347 match(RegD);
5348 op_cost(0);
5349 format %{ %}
5350 interface(REG_INTER);
5351 %}
5352
5353 operand vRegD_V7()
5354 %{
5355 constraint(ALLOC_IN_RC(v7_reg));
5356 match(RegD);
5357 op_cost(0);
5358 format %{ %}
5359 interface(REG_INTER);
5360 %}
5361
5362 operand vRegD_V8()
5363 %{
5364 constraint(ALLOC_IN_RC(v8_reg));
5365 match(RegD);
5366 op_cost(0);
5367 format %{ %}
5368 interface(REG_INTER);
5369 %}
5370
5371 operand vRegD_V9()
5372 %{
5373 constraint(ALLOC_IN_RC(v9_reg));
5374 match(RegD);
5375 op_cost(0);
5376 format %{ %}
5377 interface(REG_INTER);
5378 %}
5379
5380 operand vRegD_V10()
5381 %{
5382 constraint(ALLOC_IN_RC(v10_reg));
5383 match(RegD);
5384 op_cost(0);
5385 format %{ %}
5386 interface(REG_INTER);
5387 %}
5388
5389 operand vRegD_V11()
5390 %{
5391 constraint(ALLOC_IN_RC(v11_reg));
5392 match(RegD);
5393 op_cost(0);
5394 format %{ %}
5395 interface(REG_INTER);
5396 %}
5397
5398 operand vRegD_V12()
5399 %{
5400 constraint(ALLOC_IN_RC(v12_reg));
5401 match(RegD);
5402 op_cost(0);
5403 format %{ %}
5404 interface(REG_INTER);
5405 %}
5406
5407 operand vRegD_V13()
5408 %{
5409 constraint(ALLOC_IN_RC(v13_reg));
5410 match(RegD);
5411 op_cost(0);
5412 format %{ %}
5413 interface(REG_INTER);
5414 %}
5415
5416 operand vRegD_V14()
5417 %{
5418 constraint(ALLOC_IN_RC(v14_reg));
5419 match(RegD);
5420 op_cost(0);
5421 format %{ %}
5422 interface(REG_INTER);
5423 %}
5424
5425 operand vRegD_V15()
5426 %{
5427 constraint(ALLOC_IN_RC(v15_reg));
5428 match(RegD);
5429 op_cost(0);
5430 format %{ %}
5431 interface(REG_INTER);
5432 %}
5433
5434 operand vRegD_V16()
5435 %{
5436 constraint(ALLOC_IN_RC(v16_reg));
5437 match(RegD);
5438 op_cost(0);
5439 format %{ %}
5440 interface(REG_INTER);
5441 %}
5442
5443 operand vRegD_V17()
5444 %{
5445 constraint(ALLOC_IN_RC(v17_reg));
5446 match(RegD);
5447 op_cost(0);
5448 format %{ %}
5449 interface(REG_INTER);
5450 %}
5451
5452 operand vRegD_V18()
5453 %{
5454 constraint(ALLOC_IN_RC(v18_reg));
5455 match(RegD);
5456 op_cost(0);
5457 format %{ %}
5458 interface(REG_INTER);
5459 %}
5460
5461 operand vRegD_V19()
5462 %{
5463 constraint(ALLOC_IN_RC(v19_reg));
5464 match(RegD);
5465 op_cost(0);
5466 format %{ %}
5467 interface(REG_INTER);
5468 %}
5469
5470 operand vRegD_V20()
5471 %{
5472 constraint(ALLOC_IN_RC(v20_reg));
5473 match(RegD);
5474 op_cost(0);
5475 format %{ %}
5476 interface(REG_INTER);
5477 %}
5478
5479 operand vRegD_V21()
5480 %{
5481 constraint(ALLOC_IN_RC(v21_reg));
5482 match(RegD);
5483 op_cost(0);
5484 format %{ %}
5485 interface(REG_INTER);
5486 %}
5487
5488 operand vRegD_V22()
5489 %{
5490 constraint(ALLOC_IN_RC(v22_reg));
5491 match(RegD);
5492 op_cost(0);
5493 format %{ %}
5494 interface(REG_INTER);
5495 %}
5496
5497 operand vRegD_V23()
5498 %{
5499 constraint(ALLOC_IN_RC(v23_reg));
5500 match(RegD);
5501 op_cost(0);
5502 format %{ %}
5503 interface(REG_INTER);
5504 %}
5505
5506 operand vRegD_V24()
5507 %{
5508 constraint(ALLOC_IN_RC(v24_reg));
5509 match(RegD);
5510 op_cost(0);
5511 format %{ %}
5512 interface(REG_INTER);
5513 %}
5514
5515 operand vRegD_V25()
5516 %{
5517 constraint(ALLOC_IN_RC(v25_reg));
5518 match(RegD);
5519 op_cost(0);
5520 format %{ %}
5521 interface(REG_INTER);
5522 %}
5523
5524 operand vRegD_V26()
5525 %{
5526 constraint(ALLOC_IN_RC(v26_reg));
5527 match(RegD);
5528 op_cost(0);
5529 format %{ %}
5530 interface(REG_INTER);
5531 %}
5532
5533 operand vRegD_V27()
5534 %{
5535 constraint(ALLOC_IN_RC(v27_reg));
5536 match(RegD);
5537 op_cost(0);
5538 format %{ %}
5539 interface(REG_INTER);
5540 %}
5541
5542 operand vRegD_V28()
5543 %{
5544 constraint(ALLOC_IN_RC(v28_reg));
5545 match(RegD);
5546 op_cost(0);
5547 format %{ %}
5548 interface(REG_INTER);
5549 %}
5550
5551 operand vRegD_V29()
5552 %{
5553 constraint(ALLOC_IN_RC(v29_reg));
5554 match(RegD);
5555 op_cost(0);
5556 format %{ %}
5557 interface(REG_INTER);
5558 %}
5559
5560 operand vRegD_V30()
5561 %{
5562 constraint(ALLOC_IN_RC(v30_reg));
5563 match(RegD);
5564 op_cost(0);
5565 format %{ %}
5566 interface(REG_INTER);
5567 %}
5568
5569 operand vRegD_V31()
5570 %{
5571 constraint(ALLOC_IN_RC(v31_reg));
5572 match(RegD);
5573 op_cost(0);
5574 format %{ %}
5575 interface(REG_INTER);
5576 %}
5577
5578 operand pReg()
5579 %{
5580 constraint(ALLOC_IN_RC(pr_reg));
5581 match(RegVectMask);
5582 match(pRegGov);
5583 op_cost(0);
5584 format %{ %}
5585 interface(REG_INTER);
5586 %}
5587
5588 operand pRegGov()
5589 %{
5590 constraint(ALLOC_IN_RC(gov_pr));
5591 match(RegVectMask);
5592 match(pReg);
5593 op_cost(0);
5594 format %{ %}
5595 interface(REG_INTER);
5596 %}
5597
5598 operand pRegGov_P0()
5599 %{
5600 constraint(ALLOC_IN_RC(p0_reg));
5601 match(RegVectMask);
5602 op_cost(0);
5603 format %{ %}
5604 interface(REG_INTER);
5605 %}
5606
5607 operand pRegGov_P1()
5608 %{
5609 constraint(ALLOC_IN_RC(p1_reg));
5610 match(RegVectMask);
5611 op_cost(0);
5612 format %{ %}
5613 interface(REG_INTER);
5614 %}
5615
5616 // Flags register, used as output of signed compare instructions
5617
5618 // note that on AArch64 we also use this register as the output for
5619 // for floating point compare instructions (CmpF CmpD). this ensures
5620 // that ordered inequality tests use GT, GE, LT or LE none of which
5621 // pass through cases where the result is unordered i.e. one or both
5622 // inputs to the compare is a NaN. this means that the ideal code can
5623 // replace e.g. a GT with an LE and not end up capturing the NaN case
5624 // (where the comparison should always fail). EQ and NE tests are
5625 // always generated in ideal code so that unordered folds into the NE
5626 // case, matching the behaviour of AArch64 NE.
5627 //
5628 // This differs from x86 where the outputs of FP compares use a
5629 // special FP flags registers and where compares based on this
5630 // register are distinguished into ordered inequalities (cmpOpUCF) and
5631 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5632 // to explicitly handle the unordered case in branches. x86 also has
5633 // to include extra CMoveX rules to accept a cmpOpUCF input.
5634
5635 operand rFlagsReg()
5636 %{
5637 constraint(ALLOC_IN_RC(int_flags));
5638 match(RegFlags);
5639
5640 op_cost(0);
5641 format %{ "RFLAGS" %}
5642 interface(REG_INTER);
5643 %}
5644
5645 // Flags register, used as output of unsigned compare instructions
5646 operand rFlagsRegU()
5647 %{
5648 constraint(ALLOC_IN_RC(int_flags));
5649 match(RegFlags);
5650
5651 op_cost(0);
5652 format %{ "RFLAGSU" %}
5653 interface(REG_INTER);
5654 %}
5655
5656 // Special Registers
5657
5658 // Method Register
5659 operand inline_cache_RegP(iRegP reg)
5660 %{
5661 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5662 match(reg);
5663 match(iRegPNoSp);
5664 op_cost(0);
5665 format %{ %}
5666 interface(REG_INTER);
5667 %}
5668
5669 // Thread Register
5670 operand thread_RegP(iRegP reg)
5671 %{
5672 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5673 match(reg);
5674 op_cost(0);
5675 format %{ %}
5676 interface(REG_INTER);
5677 %}
5678
5679 operand lr_RegP(iRegP reg)
5680 %{
5681 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5682 match(reg);
5683 op_cost(0);
5684 format %{ %}
5685 interface(REG_INTER);
5686 %}
5687
5688 //----------Memory Operands----------------------------------------------------
5689
5690 operand indirect(iRegP reg)
5691 %{
5692 constraint(ALLOC_IN_RC(ptr_reg));
5693 match(reg);
5694 op_cost(0);
5695 format %{ "[$reg]" %}
5696 interface(MEMORY_INTER) %{
5697 base($reg);
5698 index(0xffffffff);
5699 scale(0x0);
5700 disp(0x0);
5701 %}
5702 %}
5703
5704 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5705 %{
5706 constraint(ALLOC_IN_RC(ptr_reg));
5707 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5708 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5709 op_cost(0);
5710 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5711 interface(MEMORY_INTER) %{
5712 base($reg);
5713 index($ireg);
5714 scale($scale);
5715 disp(0x0);
5716 %}
5717 %}
5718
5719 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5720 %{
5721 constraint(ALLOC_IN_RC(ptr_reg));
5722 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5723 match(AddP reg (LShiftL lreg scale));
5724 op_cost(0);
5725 format %{ "$reg, $lreg lsl($scale)" %}
5726 interface(MEMORY_INTER) %{
5727 base($reg);
5728 index($lreg);
5729 scale($scale);
5730 disp(0x0);
5731 %}
5732 %}
5733
5734 operand indIndexI2L(iRegP reg, iRegI ireg)
5735 %{
5736 constraint(ALLOC_IN_RC(ptr_reg));
5737 match(AddP reg (ConvI2L ireg));
5738 op_cost(0);
5739 format %{ "$reg, $ireg, 0, I2L" %}
5740 interface(MEMORY_INTER) %{
5741 base($reg);
5742 index($ireg);
5743 scale(0x0);
5744 disp(0x0);
5745 %}
5746 %}
5747
5748 operand indIndex(iRegP reg, iRegL lreg)
5749 %{
5750 constraint(ALLOC_IN_RC(ptr_reg));
5751 match(AddP reg lreg);
5752 op_cost(0);
5753 format %{ "$reg, $lreg" %}
5754 interface(MEMORY_INTER) %{
5755 base($reg);
5756 index($lreg);
5757 scale(0x0);
5758 disp(0x0);
5759 %}
5760 %}
5761
5762 operand indOffI(iRegP reg, immIOffset off)
5763 %{
5764 constraint(ALLOC_IN_RC(ptr_reg));
5765 match(AddP reg off);
5766 op_cost(0);
5767 format %{ "[$reg, $off]" %}
5768 interface(MEMORY_INTER) %{
5769 base($reg);
5770 index(0xffffffff);
5771 scale(0x0);
5772 disp($off);
5773 %}
5774 %}
5775
5776 operand indOffI1(iRegP reg, immIOffset1 off)
5777 %{
5778 constraint(ALLOC_IN_RC(ptr_reg));
5779 match(AddP reg off);
5780 op_cost(0);
5781 format %{ "[$reg, $off]" %}
5782 interface(MEMORY_INTER) %{
5783 base($reg);
5784 index(0xffffffff);
5785 scale(0x0);
5786 disp($off);
5787 %}
5788 %}
5789
5790 operand indOffI2(iRegP reg, immIOffset2 off)
5791 %{
5792 constraint(ALLOC_IN_RC(ptr_reg));
5793 match(AddP reg off);
5794 op_cost(0);
5795 format %{ "[$reg, $off]" %}
5796 interface(MEMORY_INTER) %{
5797 base($reg);
5798 index(0xffffffff);
5799 scale(0x0);
5800 disp($off);
5801 %}
5802 %}
5803
5804 operand indOffI4(iRegP reg, immIOffset4 off)
5805 %{
5806 constraint(ALLOC_IN_RC(ptr_reg));
5807 match(AddP reg off);
5808 op_cost(0);
5809 format %{ "[$reg, $off]" %}
5810 interface(MEMORY_INTER) %{
5811 base($reg);
5812 index(0xffffffff);
5813 scale(0x0);
5814 disp($off);
5815 %}
5816 %}
5817
5818 operand indOffI8(iRegP reg, immIOffset8 off)
5819 %{
5820 constraint(ALLOC_IN_RC(ptr_reg));
5821 match(AddP reg off);
5822 op_cost(0);
5823 format %{ "[$reg, $off]" %}
5824 interface(MEMORY_INTER) %{
5825 base($reg);
5826 index(0xffffffff);
5827 scale(0x0);
5828 disp($off);
5829 %}
5830 %}
5831
5832 operand indOffI16(iRegP reg, immIOffset16 off)
5833 %{
5834 constraint(ALLOC_IN_RC(ptr_reg));
5835 match(AddP reg off);
5836 op_cost(0);
5837 format %{ "[$reg, $off]" %}
5838 interface(MEMORY_INTER) %{
5839 base($reg);
5840 index(0xffffffff);
5841 scale(0x0);
5842 disp($off);
5843 %}
5844 %}
5845
5846 operand indOffL(iRegP reg, immLoffset off)
5847 %{
5848 constraint(ALLOC_IN_RC(ptr_reg));
5849 match(AddP reg off);
5850 op_cost(0);
5851 format %{ "[$reg, $off]" %}
5852 interface(MEMORY_INTER) %{
5853 base($reg);
5854 index(0xffffffff);
5855 scale(0x0);
5856 disp($off);
5857 %}
5858 %}
5859
5860 operand indOffL1(iRegP reg, immLoffset1 off)
5861 %{
5862 constraint(ALLOC_IN_RC(ptr_reg));
5863 match(AddP reg off);
5864 op_cost(0);
5865 format %{ "[$reg, $off]" %}
5866 interface(MEMORY_INTER) %{
5867 base($reg);
5868 index(0xffffffff);
5869 scale(0x0);
5870 disp($off);
5871 %}
5872 %}
5873
5874 operand indOffL2(iRegP reg, immLoffset2 off)
5875 %{
5876 constraint(ALLOC_IN_RC(ptr_reg));
5877 match(AddP reg off);
5878 op_cost(0);
5879 format %{ "[$reg, $off]" %}
5880 interface(MEMORY_INTER) %{
5881 base($reg);
5882 index(0xffffffff);
5883 scale(0x0);
5884 disp($off);
5885 %}
5886 %}
5887
5888 operand indOffL4(iRegP reg, immLoffset4 off)
5889 %{
5890 constraint(ALLOC_IN_RC(ptr_reg));
5891 match(AddP reg off);
5892 op_cost(0);
5893 format %{ "[$reg, $off]" %}
5894 interface(MEMORY_INTER) %{
5895 base($reg);
5896 index(0xffffffff);
5897 scale(0x0);
5898 disp($off);
5899 %}
5900 %}
5901
5902 operand indOffL8(iRegP reg, immLoffset8 off)
5903 %{
5904 constraint(ALLOC_IN_RC(ptr_reg));
5905 match(AddP reg off);
5906 op_cost(0);
5907 format %{ "[$reg, $off]" %}
5908 interface(MEMORY_INTER) %{
5909 base($reg);
5910 index(0xffffffff);
5911 scale(0x0);
5912 disp($off);
5913 %}
5914 %}
5915
5916 operand indOffL16(iRegP reg, immLoffset16 off)
5917 %{
5918 constraint(ALLOC_IN_RC(ptr_reg));
5919 match(AddP reg off);
5920 op_cost(0);
5921 format %{ "[$reg, $off]" %}
5922 interface(MEMORY_INTER) %{
5923 base($reg);
5924 index(0xffffffff);
5925 scale(0x0);
5926 disp($off);
5927 %}
5928 %}
5929
5930 operand indirectN(iRegN reg)
5931 %{
5932 predicate(CompressedOops::shift() == 0);
5933 constraint(ALLOC_IN_RC(ptr_reg));
5934 match(DecodeN reg);
5935 op_cost(0);
5936 format %{ "[$reg]\t# narrow" %}
5937 interface(MEMORY_INTER) %{
5938 base($reg);
5939 index(0xffffffff);
5940 scale(0x0);
5941 disp(0x0);
5942 %}
5943 %}
5944
5945 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5946 %{
5947 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5948 constraint(ALLOC_IN_RC(ptr_reg));
5949 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5950 op_cost(0);
5951 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5952 interface(MEMORY_INTER) %{
5953 base($reg);
5954 index($ireg);
5955 scale($scale);
5956 disp(0x0);
5957 %}
5958 %}
5959
5960 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5961 %{
5962 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5963 constraint(ALLOC_IN_RC(ptr_reg));
5964 match(AddP (DecodeN reg) (LShiftL lreg scale));
5965 op_cost(0);
5966 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5967 interface(MEMORY_INTER) %{
5968 base($reg);
5969 index($lreg);
5970 scale($scale);
5971 disp(0x0);
5972 %}
5973 %}
5974
5975 operand indIndexI2LN(iRegN reg, iRegI ireg)
5976 %{
5977 predicate(CompressedOops::shift() == 0);
5978 constraint(ALLOC_IN_RC(ptr_reg));
5979 match(AddP (DecodeN reg) (ConvI2L ireg));
5980 op_cost(0);
5981 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5982 interface(MEMORY_INTER) %{
5983 base($reg);
5984 index($ireg);
5985 scale(0x0);
5986 disp(0x0);
5987 %}
5988 %}
5989
5990 operand indIndexN(iRegN reg, iRegL lreg)
5991 %{
5992 predicate(CompressedOops::shift() == 0);
5993 constraint(ALLOC_IN_RC(ptr_reg));
5994 match(AddP (DecodeN reg) lreg);
5995 op_cost(0);
5996 format %{ "$reg, $lreg\t# narrow" %}
5997 interface(MEMORY_INTER) %{
5998 base($reg);
5999 index($lreg);
6000 scale(0x0);
6001 disp(0x0);
6002 %}
6003 %}
6004
6005 operand indOffIN(iRegN reg, immIOffset off)
6006 %{
6007 predicate(CompressedOops::shift() == 0);
6008 constraint(ALLOC_IN_RC(ptr_reg));
6009 match(AddP (DecodeN reg) off);
6010 op_cost(0);
6011 format %{ "[$reg, $off]\t# narrow" %}
6012 interface(MEMORY_INTER) %{
6013 base($reg);
6014 index(0xffffffff);
6015 scale(0x0);
6016 disp($off);
6017 %}
6018 %}
6019
6020 operand indOffLN(iRegN reg, immLoffset off)
6021 %{
6022 predicate(CompressedOops::shift() == 0);
6023 constraint(ALLOC_IN_RC(ptr_reg));
6024 match(AddP (DecodeN reg) off);
6025 op_cost(0);
6026 format %{ "[$reg, $off]\t# narrow" %}
6027 interface(MEMORY_INTER) %{
6028 base($reg);
6029 index(0xffffffff);
6030 scale(0x0);
6031 disp($off);
6032 %}
6033 %}
6034
6035
6036
6037 // AArch64 opto stubs need to write to the pc slot in the thread anchor
6038 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
6039 %{
6040 constraint(ALLOC_IN_RC(ptr_reg));
6041 match(AddP reg off);
6042 op_cost(0);
6043 format %{ "[$reg, $off]" %}
6044 interface(MEMORY_INTER) %{
6045 base($reg);
6046 index(0xffffffff);
6047 scale(0x0);
6048 disp($off);
6049 %}
6050 %}
6051
6052 //----------Special Memory Operands--------------------------------------------
6053 // Stack Slot Operand - This operand is used for loading and storing temporary
6054 // values on the stack where a match requires a value to
6055 // flow through memory.
6056 operand stackSlotP(sRegP reg)
6057 %{
6058 constraint(ALLOC_IN_RC(stack_slots));
6059 op_cost(100);
6060 // No match rule because this operand is only generated in matching
6061 // match(RegP);
6062 format %{ "[$reg]" %}
6063 interface(MEMORY_INTER) %{
6064 base(0x1e); // RSP
6065 index(0x0); // No Index
6066 scale(0x0); // No Scale
6067 disp($reg); // Stack Offset
6068 %}
6069 %}
6070
6071 operand stackSlotI(sRegI reg)
6072 %{
6073 constraint(ALLOC_IN_RC(stack_slots));
6074 // No match rule because this operand is only generated in matching
6075 // match(RegI);
6076 format %{ "[$reg]" %}
6077 interface(MEMORY_INTER) %{
6078 base(0x1e); // RSP
6079 index(0x0); // No Index
6080 scale(0x0); // No Scale
6081 disp($reg); // Stack Offset
6082 %}
6083 %}
6084
6085 operand stackSlotF(sRegF reg)
6086 %{
6087 constraint(ALLOC_IN_RC(stack_slots));
6088 // No match rule because this operand is only generated in matching
6089 // match(RegF);
6090 format %{ "[$reg]" %}
6091 interface(MEMORY_INTER) %{
6092 base(0x1e); // RSP
6093 index(0x0); // No Index
6094 scale(0x0); // No Scale
6095 disp($reg); // Stack Offset
6096 %}
6097 %}
6098
6099 operand stackSlotD(sRegD reg)
6100 %{
6101 constraint(ALLOC_IN_RC(stack_slots));
6102 // No match rule because this operand is only generated in matching
6103 // match(RegD);
6104 format %{ "[$reg]" %}
6105 interface(MEMORY_INTER) %{
6106 base(0x1e); // RSP
6107 index(0x0); // No Index
6108 scale(0x0); // No Scale
6109 disp($reg); // Stack Offset
6110 %}
6111 %}
6112
6113 operand stackSlotL(sRegL reg)
6114 %{
6115 constraint(ALLOC_IN_RC(stack_slots));
6116 // No match rule because this operand is only generated in matching
6117 // match(RegL);
6118 format %{ "[$reg]" %}
6119 interface(MEMORY_INTER) %{
6120 base(0x1e); // RSP
6121 index(0x0); // No Index
6122 scale(0x0); // No Scale
6123 disp($reg); // Stack Offset
6124 %}
6125 %}
6126
6127 // Operands for expressing Control Flow
6128 // NOTE: Label is a predefined operand which should not be redefined in
6129 // the AD file. It is generically handled within the ADLC.
6130
6131 //----------Conditional Branch Operands----------------------------------------
6132 // Comparison Op - This is the operation of the comparison, and is limited to
6133 // the following set of codes:
6134 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6135 //
6136 // Other attributes of the comparison, such as unsignedness, are specified
6137 // by the comparison instruction that sets a condition code flags register.
6138 // That result is represented by a flags operand whose subtype is appropriate
6139 // to the unsignedness (etc.) of the comparison.
6140 //
6141 // Later, the instruction which matches both the Comparison Op (a Bool) and
6142 // the flags (produced by the Cmp) specifies the coding of the comparison op
6143 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6144
6145 // used for signed integral comparisons and fp comparisons
6146
6147 operand cmpOp()
6148 %{
6149 match(Bool);
6150
6151 format %{ "" %}
6152 interface(COND_INTER) %{
6153 equal(0x0, "eq");
6154 not_equal(0x1, "ne");
6155 less(0xb, "lt");
6156 greater_equal(0xa, "ge");
6157 less_equal(0xd, "le");
6158 greater(0xc, "gt");
6159 overflow(0x6, "vs");
6160 no_overflow(0x7, "vc");
6161 %}
6162 %}
6163
6164 // used for unsigned integral comparisons
6165
6166 operand cmpOpU()
6167 %{
6168 match(Bool);
6169
6170 format %{ "" %}
6171 interface(COND_INTER) %{
6172 equal(0x0, "eq");
6173 not_equal(0x1, "ne");
6174 less(0x3, "lo");
6175 greater_equal(0x2, "hs");
6176 less_equal(0x9, "ls");
6177 greater(0x8, "hi");
6178 overflow(0x6, "vs");
6179 no_overflow(0x7, "vc");
6180 %}
6181 %}
6182
6183 // used for certain integral comparisons which can be
6184 // converted to cbxx or tbxx instructions
6185
6186 operand cmpOpEqNe()
6187 %{
6188 match(Bool);
6189 op_cost(0);
6190 predicate(n->as_Bool()->_test._test == BoolTest::ne
6191 || n->as_Bool()->_test._test == BoolTest::eq);
6192
6193 format %{ "" %}
6194 interface(COND_INTER) %{
6195 equal(0x0, "eq");
6196 not_equal(0x1, "ne");
6197 less(0xb, "lt");
6198 greater_equal(0xa, "ge");
6199 less_equal(0xd, "le");
6200 greater(0xc, "gt");
6201 overflow(0x6, "vs");
6202 no_overflow(0x7, "vc");
6203 %}
6204 %}
6205
6206 // used for certain integral comparisons which can be
6207 // converted to cbxx or tbxx instructions
6208
6209 operand cmpOpLtGe()
6210 %{
6211 match(Bool);
6212 op_cost(0);
6213
6214 predicate(n->as_Bool()->_test._test == BoolTest::lt
6215 || n->as_Bool()->_test._test == BoolTest::ge);
6216
6217 format %{ "" %}
6218 interface(COND_INTER) %{
6219 equal(0x0, "eq");
6220 not_equal(0x1, "ne");
6221 less(0xb, "lt");
6222 greater_equal(0xa, "ge");
6223 less_equal(0xd, "le");
6224 greater(0xc, "gt");
6225 overflow(0x6, "vs");
6226 no_overflow(0x7, "vc");
6227 %}
6228 %}
6229
6230 // used for certain unsigned integral comparisons which can be
6231 // converted to cbxx or tbxx instructions
6232
6233 operand cmpOpUEqNeLtGe()
6234 %{
6235 match(Bool);
6236 op_cost(0);
6237
6238 predicate(n->as_Bool()->_test._test == BoolTest::eq
6239 || n->as_Bool()->_test._test == BoolTest::ne
6240 || n->as_Bool()->_test._test == BoolTest::lt
6241 || n->as_Bool()->_test._test == BoolTest::ge);
6242
6243 format %{ "" %}
6244 interface(COND_INTER) %{
6245 equal(0x0, "eq");
6246 not_equal(0x1, "ne");
6247 less(0xb, "lt");
6248 greater_equal(0xa, "ge");
6249 less_equal(0xd, "le");
6250 greater(0xc, "gt");
6251 overflow(0x6, "vs");
6252 no_overflow(0x7, "vc");
6253 %}
6254 %}
6255
6256 // Special operand allowing long args to int ops to be truncated for free
6257
6258 operand iRegL2I(iRegL reg) %{
6259
6260 op_cost(0);
6261
6262 match(ConvL2I reg);
6263
6264 format %{ "l2i($reg)" %}
6265
6266 interface(REG_INTER)
6267 %}
6268
6269 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
6270 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
6271 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
6272 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
6273
6274 //----------OPERAND CLASSES----------------------------------------------------
6275 // Operand Classes are groups of operands that are used as to simplify
6276 // instruction definitions by not requiring the AD writer to specify
6277 // separate instructions for every form of operand when the
6278 // instruction accepts multiple operand types with the same basic
6279 // encoding and format. The classic case of this is memory operands.
6280
6281 // memory is used to define read/write location for load/store
6282 // instruction defs. we can turn a memory op into an Address
6283
6284 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
6285 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6286
6287 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
6288 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6289
6290 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
6291 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6292
6293 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
6294 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6295
6296 // All of the memory operands. For the pipeline description.
6297 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
6298 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
6299 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6300
6301
6302 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
6303 // operations. it allows the src to be either an iRegI or a (ConvL2I
6304 // iRegL). in the latter case the l2i normally planted for a ConvL2I
6305 // can be elided because the 32-bit instruction will just employ the
6306 // lower 32 bits anyway.
6307 //
6308 // n.b. this does not elide all L2I conversions. if the truncated
6309 // value is consumed by more than one operation then the ConvL2I
6310 // cannot be bundled into the consuming nodes so an l2i gets planted
6311 // (actually a movw $dst $src) and the downstream instructions consume
6312 // the result of the l2i as an iRegI input. That's a shame since the
6313 // movw is actually redundant but its not too costly.
6314
6315 opclass iRegIorL2I(iRegI, iRegL2I);
6316
6317 //----------PIPELINE-----------------------------------------------------------
6318 // Rules which define the behavior of the target architectures pipeline.
6319
6320 // For specific pipelines, eg A53, define the stages of that pipeline
6321 //pipe_desc(ISS, EX1, EX2, WR);
6322 #define ISS S0
6323 #define EX1 S1
6324 #define EX2 S2
6325 #define WR S3
6326
6327 // Integer ALU reg operation
6328 pipeline %{
6329
6330 attributes %{
6331 // ARM instructions are of fixed length
6332 fixed_size_instructions; // Fixed size instructions TODO does
6333 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
6334 // ARM instructions come in 32-bit word units
6335 instruction_unit_size = 4; // An instruction is 4 bytes long
6336 instruction_fetch_unit_size = 64; // The processor fetches one line
6337 instruction_fetch_units = 1; // of 64 bytes
6338
6339 // List of nop instructions
6340 nops( MachNop );
6341 %}
6342
6343 // We don't use an actual pipeline model so don't care about resources
6344 // or description. we do use pipeline classes to introduce fixed
6345 // latencies
6346
6347 //----------RESOURCES----------------------------------------------------------
6348 // Resources are the functional units available to the machine
6349
6350 resources( INS0, INS1, INS01 = INS0 | INS1,
6351 ALU0, ALU1, ALU = ALU0 | ALU1,
6352 MAC,
6353 DIV,
6354 BRANCH,
6355 LDST,
6356 NEON_FP);
6357
6358 //----------PIPELINE DESCRIPTION-----------------------------------------------
6359 // Pipeline Description specifies the stages in the machine's pipeline
6360
6361 // Define the pipeline as a generic 6 stage pipeline
6362 pipe_desc(S0, S1, S2, S3, S4, S5);
6363
6364 //----------PIPELINE CLASSES---------------------------------------------------
6365 // Pipeline Classes describe the stages in which input and output are
6366 // referenced by the hardware pipeline.
6367
6368 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
6369 %{
6370 single_instruction;
6371 src1 : S1(read);
6372 src2 : S2(read);
6373 dst : S5(write);
6374 INS01 : ISS;
6375 NEON_FP : S5;
6376 %}
6377
6378 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6379 %{
6380 single_instruction;
6381 src1 : S1(read);
6382 src2 : S2(read);
6383 dst : S5(write);
6384 INS01 : ISS;
6385 NEON_FP : S5;
6386 %}
6387
6388 pipe_class fp_uop_s(vRegF dst, vRegF src)
6389 %{
6390 single_instruction;
6391 src : S1(read);
6392 dst : S5(write);
6393 INS01 : ISS;
6394 NEON_FP : S5;
6395 %}
6396
6397 pipe_class fp_uop_d(vRegD dst, vRegD src)
6398 %{
6399 single_instruction;
6400 src : S1(read);
6401 dst : S5(write);
6402 INS01 : ISS;
6403 NEON_FP : S5;
6404 %}
6405
6406 pipe_class fp_d2f(vRegF dst, vRegD src)
6407 %{
6408 single_instruction;
6409 src : S1(read);
6410 dst : S5(write);
6411 INS01 : ISS;
6412 NEON_FP : S5;
6413 %}
6414
6415 pipe_class fp_f2d(vRegD dst, vRegF src)
6416 %{
6417 single_instruction;
6418 src : S1(read);
6419 dst : S5(write);
6420 INS01 : ISS;
6421 NEON_FP : S5;
6422 %}
6423
6424 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6425 %{
6426 single_instruction;
6427 src : S1(read);
6428 dst : S5(write);
6429 INS01 : ISS;
6430 NEON_FP : S5;
6431 %}
6432
6433 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6434 %{
6435 single_instruction;
6436 src : S1(read);
6437 dst : S5(write);
6438 INS01 : ISS;
6439 NEON_FP : S5;
6440 %}
6441
6442 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6443 %{
6444 single_instruction;
6445 src : S1(read);
6446 dst : S5(write);
6447 INS01 : ISS;
6448 NEON_FP : S5;
6449 %}
6450
6451 pipe_class fp_l2f(vRegF dst, iRegL src)
6452 %{
6453 single_instruction;
6454 src : S1(read);
6455 dst : S5(write);
6456 INS01 : ISS;
6457 NEON_FP : S5;
6458 %}
6459
6460 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6461 %{
6462 single_instruction;
6463 src : S1(read);
6464 dst : S5(write);
6465 INS01 : ISS;
6466 NEON_FP : S5;
6467 %}
6468
6469 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6470 %{
6471 single_instruction;
6472 src : S1(read);
6473 dst : S5(write);
6474 INS01 : ISS;
6475 NEON_FP : S5;
6476 %}
6477
6478 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6479 %{
6480 single_instruction;
6481 src : S1(read);
6482 dst : S5(write);
6483 INS01 : ISS;
6484 NEON_FP : S5;
6485 %}
6486
6487 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6488 %{
6489 single_instruction;
6490 src : S1(read);
6491 dst : S5(write);
6492 INS01 : ISS;
6493 NEON_FP : S5;
6494 %}
6495
6496 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6497 %{
6498 single_instruction;
6499 src1 : S1(read);
6500 src2 : S2(read);
6501 dst : S5(write);
6502 INS0 : ISS;
6503 NEON_FP : S5;
6504 %}
6505
6506 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6507 %{
6508 single_instruction;
6509 src1 : S1(read);
6510 src2 : S2(read);
6511 dst : S5(write);
6512 INS0 : ISS;
6513 NEON_FP : S5;
6514 %}
6515
6516 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6517 %{
6518 single_instruction;
6519 cr : S1(read);
6520 src1 : S1(read);
6521 src2 : S1(read);
6522 dst : S3(write);
6523 INS01 : ISS;
6524 NEON_FP : S3;
6525 %}
6526
6527 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6528 %{
6529 single_instruction;
6530 cr : S1(read);
6531 src1 : S1(read);
6532 src2 : S1(read);
6533 dst : S3(write);
6534 INS01 : ISS;
6535 NEON_FP : S3;
6536 %}
6537
6538 pipe_class fp_imm_s(vRegF dst)
6539 %{
6540 single_instruction;
6541 dst : S3(write);
6542 INS01 : ISS;
6543 NEON_FP : S3;
6544 %}
6545
6546 pipe_class fp_imm_d(vRegD dst)
6547 %{
6548 single_instruction;
6549 dst : S3(write);
6550 INS01 : ISS;
6551 NEON_FP : S3;
6552 %}
6553
6554 pipe_class fp_load_constant_s(vRegF dst)
6555 %{
6556 single_instruction;
6557 dst : S4(write);
6558 INS01 : ISS;
6559 NEON_FP : S4;
6560 %}
6561
6562 pipe_class fp_load_constant_d(vRegD dst)
6563 %{
6564 single_instruction;
6565 dst : S4(write);
6566 INS01 : ISS;
6567 NEON_FP : S4;
6568 %}
6569
6570 //------- Integer ALU operations --------------------------
6571
6572 // Integer ALU reg-reg operation
6573 // Operands needed in EX1, result generated in EX2
6574 // Eg. ADD x0, x1, x2
6575 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6576 %{
6577 single_instruction;
6578 dst : EX2(write);
6579 src1 : EX1(read);
6580 src2 : EX1(read);
6581 INS01 : ISS; // Dual issue as instruction 0 or 1
6582 ALU : EX2;
6583 %}
6584
6585 // Integer ALU reg-reg operation with constant shift
6586 // Shifted register must be available in LATE_ISS instead of EX1
6587 // Eg. ADD x0, x1, x2, LSL #2
6588 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6589 %{
6590 single_instruction;
6591 dst : EX2(write);
6592 src1 : EX1(read);
6593 src2 : ISS(read);
6594 INS01 : ISS;
6595 ALU : EX2;
6596 %}
6597
6598 // Integer ALU reg operation with constant shift
6599 // Eg. LSL x0, x1, #shift
6600 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6601 %{
6602 single_instruction;
6603 dst : EX2(write);
6604 src1 : ISS(read);
6605 INS01 : ISS;
6606 ALU : EX2;
6607 %}
6608
6609 // Integer ALU reg-reg operation with variable shift
6610 // Both operands must be available in LATE_ISS instead of EX1
6611 // Result is available in EX1 instead of EX2
6612 // Eg. LSLV x0, x1, x2
6613 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6614 %{
6615 single_instruction;
6616 dst : EX1(write);
6617 src1 : ISS(read);
6618 src2 : ISS(read);
6619 INS01 : ISS;
6620 ALU : EX1;
6621 %}
6622
6623 // Integer ALU reg-reg operation with extract
6624 // As for _vshift above, but result generated in EX2
6625 // Eg. EXTR x0, x1, x2, #N
6626 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6627 %{
6628 single_instruction;
6629 dst : EX2(write);
6630 src1 : ISS(read);
6631 src2 : ISS(read);
6632 INS1 : ISS; // Can only dual issue as Instruction 1
6633 ALU : EX1;
6634 %}
6635
6636 // Integer ALU reg operation
6637 // Eg. NEG x0, x1
6638 pipe_class ialu_reg(iRegI dst, iRegI src)
6639 %{
6640 single_instruction;
6641 dst : EX2(write);
6642 src : EX1(read);
6643 INS01 : ISS;
6644 ALU : EX2;
6645 %}
6646
6647 // Integer ALU reg mmediate operation
6648 // Eg. ADD x0, x1, #N
6649 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6650 %{
6651 single_instruction;
6652 dst : EX2(write);
6653 src1 : EX1(read);
6654 INS01 : ISS;
6655 ALU : EX2;
6656 %}
6657
6658 // Integer ALU immediate operation (no source operands)
6659 // Eg. MOV x0, #N
6660 pipe_class ialu_imm(iRegI dst)
6661 %{
6662 single_instruction;
6663 dst : EX1(write);
6664 INS01 : ISS;
6665 ALU : EX1;
6666 %}
6667
6668 //------- Compare operation -------------------------------
6669
6670 // Compare reg-reg
6671 // Eg. CMP x0, x1
6672 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6673 %{
6674 single_instruction;
6675 // fixed_latency(16);
6676 cr : EX2(write);
6677 op1 : EX1(read);
6678 op2 : EX1(read);
6679 INS01 : ISS;
6680 ALU : EX2;
6681 %}
6682
6683 // Compare reg-reg
6684 // Eg. CMP x0, #N
6685 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6686 %{
6687 single_instruction;
6688 // fixed_latency(16);
6689 cr : EX2(write);
6690 op1 : EX1(read);
6691 INS01 : ISS;
6692 ALU : EX2;
6693 %}
6694
6695 //------- Conditional instructions ------------------------
6696
6697 // Conditional no operands
6698 // Eg. CSINC x0, zr, zr, <cond>
6699 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6700 %{
6701 single_instruction;
6702 cr : EX1(read);
6703 dst : EX2(write);
6704 INS01 : ISS;
6705 ALU : EX2;
6706 %}
6707
6708 // Conditional 2 operand
6709 // EG. CSEL X0, X1, X2, <cond>
6710 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6711 %{
6712 single_instruction;
6713 cr : EX1(read);
6714 src1 : EX1(read);
6715 src2 : EX1(read);
6716 dst : EX2(write);
6717 INS01 : ISS;
6718 ALU : EX2;
6719 %}
6720
6721 // Conditional 2 operand
6722 // EG. CSEL X0, X1, X2, <cond>
6723 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6724 %{
6725 single_instruction;
6726 cr : EX1(read);
6727 src : EX1(read);
6728 dst : EX2(write);
6729 INS01 : ISS;
6730 ALU : EX2;
6731 %}
6732
6733 //------- Multiply pipeline operations --------------------
6734
6735 // Multiply reg-reg
6736 // Eg. MUL w0, w1, w2
6737 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6738 %{
6739 single_instruction;
6740 dst : WR(write);
6741 src1 : ISS(read);
6742 src2 : ISS(read);
6743 INS01 : ISS;
6744 MAC : WR;
6745 %}
6746
6747 // Multiply accumulate
6748 // Eg. MADD w0, w1, w2, w3
6749 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6750 %{
6751 single_instruction;
6752 dst : WR(write);
6753 src1 : ISS(read);
6754 src2 : ISS(read);
6755 src3 : ISS(read);
6756 INS01 : ISS;
6757 MAC : WR;
6758 %}
6759
6760 // Eg. MUL w0, w1, w2
6761 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6762 %{
6763 single_instruction;
6764 fixed_latency(3); // Maximum latency for 64 bit mul
6765 dst : WR(write);
6766 src1 : ISS(read);
6767 src2 : ISS(read);
6768 INS01 : ISS;
6769 MAC : WR;
6770 %}
6771
6772 // Multiply accumulate
6773 // Eg. MADD w0, w1, w2, w3
6774 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6775 %{
6776 single_instruction;
6777 fixed_latency(3); // Maximum latency for 64 bit mul
6778 dst : WR(write);
6779 src1 : ISS(read);
6780 src2 : ISS(read);
6781 src3 : ISS(read);
6782 INS01 : ISS;
6783 MAC : WR;
6784 %}
6785
6786 //------- Divide pipeline operations --------------------
6787
6788 // Eg. SDIV w0, w1, w2
6789 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6790 %{
6791 single_instruction;
6792 fixed_latency(8); // Maximum latency for 32 bit divide
6793 dst : WR(write);
6794 src1 : ISS(read);
6795 src2 : ISS(read);
6796 INS0 : ISS; // Can only dual issue as instruction 0
6797 DIV : WR;
6798 %}
6799
6800 // Eg. SDIV x0, x1, x2
6801 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6802 %{
6803 single_instruction;
6804 fixed_latency(16); // Maximum latency for 64 bit divide
6805 dst : WR(write);
6806 src1 : ISS(read);
6807 src2 : ISS(read);
6808 INS0 : ISS; // Can only dual issue as instruction 0
6809 DIV : WR;
6810 %}
6811
6812 //------- Load pipeline operations ------------------------
6813
6814 // Load - prefetch
6815 // Eg. PFRM <mem>
6816 pipe_class iload_prefetch(memory mem)
6817 %{
6818 single_instruction;
6819 mem : ISS(read);
6820 INS01 : ISS;
6821 LDST : WR;
6822 %}
6823
6824 // Load - reg, mem
6825 // Eg. LDR x0, <mem>
6826 pipe_class iload_reg_mem(iRegI dst, memory mem)
6827 %{
6828 single_instruction;
6829 dst : WR(write);
6830 mem : ISS(read);
6831 INS01 : ISS;
6832 LDST : WR;
6833 %}
6834
6835 // Load - reg, reg
6836 // Eg. LDR x0, [sp, x1]
6837 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6838 %{
6839 single_instruction;
6840 dst : WR(write);
6841 src : ISS(read);
6842 INS01 : ISS;
6843 LDST : WR;
6844 %}
6845
6846 //------- Store pipeline operations -----------------------
6847
6848 // Store - zr, mem
6849 // Eg. STR zr, <mem>
6850 pipe_class istore_mem(memory mem)
6851 %{
6852 single_instruction;
6853 mem : ISS(read);
6854 INS01 : ISS;
6855 LDST : WR;
6856 %}
6857
6858 // Store - reg, mem
6859 // Eg. STR x0, <mem>
6860 pipe_class istore_reg_mem(iRegI src, memory mem)
6861 %{
6862 single_instruction;
6863 mem : ISS(read);
6864 src : EX2(read);
6865 INS01 : ISS;
6866 LDST : WR;
6867 %}
6868
6869 // Store - reg, reg
6870 // Eg. STR x0, [sp, x1]
6871 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6872 %{
6873 single_instruction;
6874 dst : ISS(read);
6875 src : EX2(read);
6876 INS01 : ISS;
6877 LDST : WR;
6878 %}
6879
6880 //------- Store pipeline operations -----------------------
6881
6882 // Branch
6883 pipe_class pipe_branch()
6884 %{
6885 single_instruction;
6886 INS01 : ISS;
6887 BRANCH : EX1;
6888 %}
6889
6890 // Conditional branch
6891 pipe_class pipe_branch_cond(rFlagsReg cr)
6892 %{
6893 single_instruction;
6894 cr : EX1(read);
6895 INS01 : ISS;
6896 BRANCH : EX1;
6897 %}
6898
6899 // Compare & Branch
6900 // EG. CBZ/CBNZ
6901 pipe_class pipe_cmp_branch(iRegI op1)
6902 %{
6903 single_instruction;
6904 op1 : EX1(read);
6905 INS01 : ISS;
6906 BRANCH : EX1;
6907 %}
6908
6909 //------- Synchronisation operations ----------------------
6910
6911 // Any operation requiring serialization.
6912 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6913 pipe_class pipe_serial()
6914 %{
6915 single_instruction;
6916 force_serialization;
6917 fixed_latency(16);
6918 INS01 : ISS(2); // Cannot dual issue with any other instruction
6919 LDST : WR;
6920 %}
6921
6922 // Generic big/slow expanded idiom - also serialized
6923 pipe_class pipe_slow()
6924 %{
6925 instruction_count(10);
6926 multiple_bundles;
6927 force_serialization;
6928 fixed_latency(16);
6929 INS01 : ISS(2); // Cannot dual issue with any other instruction
6930 LDST : WR;
6931 %}
6932
6933 // Empty pipeline class
6934 pipe_class pipe_class_empty()
6935 %{
6936 single_instruction;
6937 fixed_latency(0);
6938 %}
6939
6940 // Default pipeline class.
6941 pipe_class pipe_class_default()
6942 %{
6943 single_instruction;
6944 fixed_latency(2);
6945 %}
6946
6947 // Pipeline class for compares.
6948 pipe_class pipe_class_compare()
6949 %{
6950 single_instruction;
6951 fixed_latency(16);
6952 %}
6953
6954 // Pipeline class for memory operations.
6955 pipe_class pipe_class_memory()
6956 %{
6957 single_instruction;
6958 fixed_latency(16);
6959 %}
6960
6961 // Pipeline class for call.
6962 pipe_class pipe_class_call()
6963 %{
6964 single_instruction;
6965 fixed_latency(100);
6966 %}
6967
6968 // Define the class for the Nop node.
6969 define %{
6970 MachNop = pipe_class_empty;
6971 %}
6972
6973 %}
6974 //----------INSTRUCTIONS-------------------------------------------------------
6975 //
6976 // match -- States which machine-independent subtree may be replaced
6977 // by this instruction.
6978 // ins_cost -- The estimated cost of this instruction is used by instruction
6979 // selection to identify a minimum cost tree of machine
6980 // instructions that matches a tree of machine-independent
6981 // instructions.
6982 // format -- A string providing the disassembly for this instruction.
6983 // The value of an instruction's operand may be inserted
6984 // by referring to it with a '$' prefix.
6985 // opcode -- Three instruction opcodes may be provided. These are referred
6986 // to within an encode class as $primary, $secondary, and $tertiary
6987 // rrspectively. The primary opcode is commonly used to
6988 // indicate the type of machine instruction, while secondary
6989 // and tertiary are often used for prefix options or addressing
6990 // modes.
6991 // ins_encode -- A list of encode classes with parameters. The encode class
6992 // name must have been defined in an 'enc_class' specification
6993 // in the encode section of the architecture description.
6994
6995 // ============================================================================
6996 // Memory (Load/Store) Instructions
6997
6998 // Load Instructions
6999
7000 // Load Byte (8 bit signed)
7001 instruct loadB(iRegINoSp dst, memory1 mem)
7002 %{
7003 match(Set dst (LoadB mem));
7004 predicate(!needs_acquiring_load(n));
7005
7006 ins_cost(4 * INSN_COST);
7007 format %{ "ldrsbw $dst, $mem\t# byte" %}
7008
7009 ins_encode(aarch64_enc_ldrsbw(dst, mem));
7010
7011 ins_pipe(iload_reg_mem);
7012 %}
7013
7014 // Load Byte (8 bit signed) into long
7015 instruct loadB2L(iRegLNoSp dst, memory1 mem)
7016 %{
7017 match(Set dst (ConvI2L (LoadB mem)));
7018 predicate(!needs_acquiring_load(n->in(1)));
7019
7020 ins_cost(4 * INSN_COST);
7021 format %{ "ldrsb $dst, $mem\t# byte" %}
7022
7023 ins_encode(aarch64_enc_ldrsb(dst, mem));
7024
7025 ins_pipe(iload_reg_mem);
7026 %}
7027
7028 // Load Byte (8 bit unsigned)
7029 instruct loadUB(iRegINoSp dst, memory1 mem)
7030 %{
7031 match(Set dst (LoadUB mem));
7032 predicate(!needs_acquiring_load(n));
7033
7034 ins_cost(4 * INSN_COST);
7035 format %{ "ldrbw $dst, $mem\t# byte" %}
7036
7037 ins_encode(aarch64_enc_ldrb(dst, mem));
7038
7039 ins_pipe(iload_reg_mem);
7040 %}
7041
7042 // Load Byte (8 bit unsigned) into long
7043 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
7044 %{
7045 match(Set dst (ConvI2L (LoadUB mem)));
7046 predicate(!needs_acquiring_load(n->in(1)));
7047
7048 ins_cost(4 * INSN_COST);
7049 format %{ "ldrb $dst, $mem\t# byte" %}
7050
7051 ins_encode(aarch64_enc_ldrb(dst, mem));
7052
7053 ins_pipe(iload_reg_mem);
7054 %}
7055
7056 // Load Short (16 bit signed)
7057 instruct loadS(iRegINoSp dst, memory2 mem)
7058 %{
7059 match(Set dst (LoadS mem));
7060 predicate(!needs_acquiring_load(n));
7061
7062 ins_cost(4 * INSN_COST);
7063 format %{ "ldrshw $dst, $mem\t# short" %}
7064
7065 ins_encode(aarch64_enc_ldrshw(dst, mem));
7066
7067 ins_pipe(iload_reg_mem);
7068 %}
7069
7070 // Load Short (16 bit signed) into long
7071 instruct loadS2L(iRegLNoSp dst, memory2 mem)
7072 %{
7073 match(Set dst (ConvI2L (LoadS mem)));
7074 predicate(!needs_acquiring_load(n->in(1)));
7075
7076 ins_cost(4 * INSN_COST);
7077 format %{ "ldrsh $dst, $mem\t# short" %}
7078
7079 ins_encode(aarch64_enc_ldrsh(dst, mem));
7080
7081 ins_pipe(iload_reg_mem);
7082 %}
7083
7084 // Load Char (16 bit unsigned)
7085 instruct loadUS(iRegINoSp dst, memory2 mem)
7086 %{
7087 match(Set dst (LoadUS mem));
7088 predicate(!needs_acquiring_load(n));
7089
7090 ins_cost(4 * INSN_COST);
7091 format %{ "ldrh $dst, $mem\t# short" %}
7092
7093 ins_encode(aarch64_enc_ldrh(dst, mem));
7094
7095 ins_pipe(iload_reg_mem);
7096 %}
7097
7098 // Load Short/Char (16 bit unsigned) into long
7099 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
7100 %{
7101 match(Set dst (ConvI2L (LoadUS mem)));
7102 predicate(!needs_acquiring_load(n->in(1)));
7103
7104 ins_cost(4 * INSN_COST);
7105 format %{ "ldrh $dst, $mem\t# short" %}
7106
7107 ins_encode(aarch64_enc_ldrh(dst, mem));
7108
7109 ins_pipe(iload_reg_mem);
7110 %}
7111
7112 // Load Integer (32 bit signed)
7113 instruct loadI(iRegINoSp dst, memory4 mem)
7114 %{
7115 match(Set dst (LoadI mem));
7116 predicate(!needs_acquiring_load(n));
7117
7118 ins_cost(4 * INSN_COST);
7119 format %{ "ldrw $dst, $mem\t# int" %}
7120
7121 ins_encode(aarch64_enc_ldrw(dst, mem));
7122
7123 ins_pipe(iload_reg_mem);
7124 %}
7125
7126 // Load Integer (32 bit signed) into long
7127 instruct loadI2L(iRegLNoSp dst, memory4 mem)
7128 %{
7129 match(Set dst (ConvI2L (LoadI mem)));
7130 predicate(!needs_acquiring_load(n->in(1)));
7131
7132 ins_cost(4 * INSN_COST);
7133 format %{ "ldrsw $dst, $mem\t# int" %}
7134
7135 ins_encode(aarch64_enc_ldrsw(dst, mem));
7136
7137 ins_pipe(iload_reg_mem);
7138 %}
7139
7140 // Load Integer (32 bit unsigned) into long
7141 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
7142 %{
7143 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7144 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
7145
7146 ins_cost(4 * INSN_COST);
7147 format %{ "ldrw $dst, $mem\t# int" %}
7148
7149 ins_encode(aarch64_enc_ldrw(dst, mem));
7150
7151 ins_pipe(iload_reg_mem);
7152 %}
7153
7154 // Load Long (64 bit signed)
7155 instruct loadL(iRegLNoSp dst, memory8 mem)
7156 %{
7157 match(Set dst (LoadL mem));
7158 predicate(!needs_acquiring_load(n));
7159
7160 ins_cost(4 * INSN_COST);
7161 format %{ "ldr $dst, $mem\t# int" %}
7162
7163 ins_encode(aarch64_enc_ldr(dst, mem));
7164
7165 ins_pipe(iload_reg_mem);
7166 %}
7167
7168 // Load Range
7169 instruct loadRange(iRegINoSp dst, memory4 mem)
7170 %{
7171 match(Set dst (LoadRange mem));
7172
7173 ins_cost(4 * INSN_COST);
7174 format %{ "ldrw $dst, $mem\t# range" %}
7175
7176 ins_encode(aarch64_enc_ldrw(dst, mem));
7177
7178 ins_pipe(iload_reg_mem);
7179 %}
7180
7181 // Load Pointer
7182 instruct loadP(iRegPNoSp dst, memory8 mem)
7183 %{
7184 match(Set dst (LoadP mem));
7185 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
7186
7187 ins_cost(4 * INSN_COST);
7188 format %{ "ldr $dst, $mem\t# ptr" %}
7189
7190 ins_encode(aarch64_enc_ldr(dst, mem));
7191
7192 ins_pipe(iload_reg_mem);
7193 %}
7194
7195 // Load Compressed Pointer
7196 instruct loadN(iRegNNoSp dst, memory4 mem)
7197 %{
7198 match(Set dst (LoadN mem));
7199 predicate(!needs_acquiring_load(n));
7200
7201 ins_cost(4 * INSN_COST);
7202 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
7203
7204 ins_encode(aarch64_enc_ldrw(dst, mem));
7205
7206 ins_pipe(iload_reg_mem);
7207 %}
7208
7209 // Load Klass Pointer
7210 instruct loadKlass(iRegPNoSp dst, memory8 mem)
7211 %{
7212 match(Set dst (LoadKlass mem));
7213 predicate(!needs_acquiring_load(n));
7214
7215 ins_cost(4 * INSN_COST);
7216 format %{ "ldr $dst, $mem\t# class" %}
7217
7218 ins_encode(aarch64_enc_ldr(dst, mem));
7219
7220 ins_pipe(iload_reg_mem);
7221 %}
7222
7223 // Load Narrow Klass Pointer
7224 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
7225 %{
7226 match(Set dst (LoadNKlass mem));
7227 predicate(!needs_acquiring_load(n));
7228
7229 ins_cost(4 * INSN_COST);
7230 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7231
7232 ins_encode(aarch64_enc_ldrw(dst, mem));
7233
7234 ins_pipe(iload_reg_mem);
7235 %}
7236
7237 // Load Float
7238 instruct loadF(vRegF dst, memory4 mem)
7239 %{
7240 match(Set dst (LoadF mem));
7241 predicate(!needs_acquiring_load(n));
7242
7243 ins_cost(4 * INSN_COST);
7244 format %{ "ldrs $dst, $mem\t# float" %}
7245
7246 ins_encode( aarch64_enc_ldrs(dst, mem) );
7247
7248 ins_pipe(pipe_class_memory);
7249 %}
7250
7251 // Load Double
7252 instruct loadD(vRegD dst, memory8 mem)
7253 %{
7254 match(Set dst (LoadD mem));
7255 predicate(!needs_acquiring_load(n));
7256
7257 ins_cost(4 * INSN_COST);
7258 format %{ "ldrd $dst, $mem\t# double" %}
7259
7260 ins_encode( aarch64_enc_ldrd(dst, mem) );
7261
7262 ins_pipe(pipe_class_memory);
7263 %}
7264
7265
7266 // Load Int Constant
7267 instruct loadConI(iRegINoSp dst, immI src)
7268 %{
7269 match(Set dst src);
7270
7271 ins_cost(INSN_COST);
7272 format %{ "mov $dst, $src\t# int" %}
7273
7274 ins_encode( aarch64_enc_movw_imm(dst, src) );
7275
7276 ins_pipe(ialu_imm);
7277 %}
7278
7279 // Load Long Constant
7280 instruct loadConL(iRegLNoSp dst, immL src)
7281 %{
7282 match(Set dst src);
7283
7284 ins_cost(INSN_COST);
7285 format %{ "mov $dst, $src\t# long" %}
7286
7287 ins_encode( aarch64_enc_mov_imm(dst, src) );
7288
7289 ins_pipe(ialu_imm);
7290 %}
7291
7292 // Load Pointer Constant
7293
7294 instruct loadConP(iRegPNoSp dst, immP con)
7295 %{
7296 match(Set dst con);
7297
7298 ins_cost(INSN_COST * 4);
7299 format %{
7300 "mov $dst, $con\t# ptr"
7301 %}
7302
7303 ins_encode(aarch64_enc_mov_p(dst, con));
7304
7305 ins_pipe(ialu_imm);
7306 %}
7307
7308 // Load Null Pointer Constant
7309
7310 instruct loadConP0(iRegPNoSp dst, immP0 con)
7311 %{
7312 match(Set dst con);
7313
7314 ins_cost(INSN_COST);
7315 format %{ "mov $dst, $con\t# NULL ptr" %}
7316
7317 ins_encode(aarch64_enc_mov_p0(dst, con));
7318
7319 ins_pipe(ialu_imm);
7320 %}
7321
7322 // Load Pointer Constant One
7323
7324 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7325 %{
7326 match(Set dst con);
7327
7328 ins_cost(INSN_COST);
7329 format %{ "mov $dst, $con\t# NULL ptr" %}
7330
7331 ins_encode(aarch64_enc_mov_p1(dst, con));
7332
7333 ins_pipe(ialu_imm);
7334 %}
7335
7336 // Load Byte Map Base Constant
7337
7338 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7339 %{
7340 match(Set dst con);
7341
7342 ins_cost(INSN_COST);
7343 format %{ "adr $dst, $con\t# Byte Map Base" %}
7344
7345 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7346
7347 ins_pipe(ialu_imm);
7348 %}
7349
7350 // Load Narrow Pointer Constant
7351
7352 instruct loadConN(iRegNNoSp dst, immN con)
7353 %{
7354 match(Set dst con);
7355
7356 ins_cost(INSN_COST * 4);
7357 format %{ "mov $dst, $con\t# compressed ptr" %}
7358
7359 ins_encode(aarch64_enc_mov_n(dst, con));
7360
7361 ins_pipe(ialu_imm);
7362 %}
7363
7364 // Load Narrow Null Pointer Constant
7365
7366 instruct loadConN0(iRegNNoSp dst, immN0 con)
7367 %{
7368 match(Set dst con);
7369
7370 ins_cost(INSN_COST);
7371 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
7372
7373 ins_encode(aarch64_enc_mov_n0(dst, con));
7374
7375 ins_pipe(ialu_imm);
7376 %}
7377
7378 // Load Narrow Klass Constant
7379
7380 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7381 %{
7382 match(Set dst con);
7383
7384 ins_cost(INSN_COST);
7385 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7386
7387 ins_encode(aarch64_enc_mov_nk(dst, con));
7388
7389 ins_pipe(ialu_imm);
7390 %}
7391
7392 // Load Packed Float Constant
7393
7394 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7395 match(Set dst con);
7396 ins_cost(INSN_COST * 4);
7397 format %{ "fmovs $dst, $con"%}
7398 ins_encode %{
7399 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7400 %}
7401
7402 ins_pipe(fp_imm_s);
7403 %}
7404
7405 // Load Float Constant
7406
7407 instruct loadConF(vRegF dst, immF con) %{
7408 match(Set dst con);
7409
7410 ins_cost(INSN_COST * 4);
7411
7412 format %{
7413 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7414 %}
7415
7416 ins_encode %{
7417 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7418 %}
7419
7420 ins_pipe(fp_load_constant_s);
7421 %}
7422
7423 // Load Packed Double Constant
7424
7425 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7426 match(Set dst con);
7427 ins_cost(INSN_COST);
7428 format %{ "fmovd $dst, $con"%}
7429 ins_encode %{
7430 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7431 %}
7432
7433 ins_pipe(fp_imm_d);
7434 %}
7435
7436 // Load Double Constant
7437
7438 instruct loadConD(vRegD dst, immD con) %{
7439 match(Set dst con);
7440
7441 ins_cost(INSN_COST * 5);
7442 format %{
7443 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7444 %}
7445
7446 ins_encode %{
7447 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7448 %}
7449
7450 ins_pipe(fp_load_constant_d);
7451 %}
7452
7453 // Store Instructions
7454
7455 // Store CMS card-mark Immediate
7456 instruct storeimmCM0(immI0 zero, memory1 mem)
7457 %{
7458 match(Set mem (StoreCM mem zero));
7459
7460 ins_cost(INSN_COST);
7461 format %{ "storestore (elided)\n\t"
7462 "strb zr, $mem\t# byte" %}
7463
7464 ins_encode(aarch64_enc_strb0(mem));
7465
7466 ins_pipe(istore_mem);
7467 %}
7468
7469 // Store CMS card-mark Immediate with intervening StoreStore
7470 // needed when using CMS with no conditional card marking
7471 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
7472 %{
7473 match(Set mem (StoreCM mem zero));
7474
7475 ins_cost(INSN_COST * 2);
7476 format %{ "storestore\n\t"
7477 "dmb ishst"
7478 "\n\tstrb zr, $mem\t# byte" %}
7479
7480 ins_encode(aarch64_enc_strb0_ordered(mem));
7481
7482 ins_pipe(istore_mem);
7483 %}
7484
7485 // Store Byte
7486 instruct storeB(iRegIorL2I src, memory1 mem)
7487 %{
7488 match(Set mem (StoreB mem src));
7489 predicate(!needs_releasing_store(n));
7490
7491 ins_cost(INSN_COST);
7492 format %{ "strb $src, $mem\t# byte" %}
7493
7494 ins_encode(aarch64_enc_strb(src, mem));
7495
7496 ins_pipe(istore_reg_mem);
7497 %}
7498
7499
7500 instruct storeimmB0(immI0 zero, memory1 mem)
7501 %{
7502 match(Set mem (StoreB mem zero));
7503 predicate(!needs_releasing_store(n));
7504
7505 ins_cost(INSN_COST);
7506 format %{ "strb rscractch2, $mem\t# byte" %}
7507
7508 ins_encode(aarch64_enc_strb0(mem));
7509
7510 ins_pipe(istore_mem);
7511 %}
7512
7513 // Store Char/Short
7514 instruct storeC(iRegIorL2I src, memory2 mem)
7515 %{
7516 match(Set mem (StoreC mem src));
7517 predicate(!needs_releasing_store(n));
7518
7519 ins_cost(INSN_COST);
7520 format %{ "strh $src, $mem\t# short" %}
7521
7522 ins_encode(aarch64_enc_strh(src, mem));
7523
7524 ins_pipe(istore_reg_mem);
7525 %}
7526
7527 instruct storeimmC0(immI0 zero, memory2 mem)
7528 %{
7529 match(Set mem (StoreC mem zero));
7530 predicate(!needs_releasing_store(n));
7531
7532 ins_cost(INSN_COST);
7533 format %{ "strh zr, $mem\t# short" %}
7534
7535 ins_encode(aarch64_enc_strh0(mem));
7536
7537 ins_pipe(istore_mem);
7538 %}
7539
7540 // Store Integer
7541
7542 instruct storeI(iRegIorL2I src, memory4 mem)
7543 %{
7544 match(Set mem(StoreI mem src));
7545 predicate(!needs_releasing_store(n));
7546
7547 ins_cost(INSN_COST);
7548 format %{ "strw $src, $mem\t# int" %}
7549
7550 ins_encode(aarch64_enc_strw(src, mem));
7551
7552 ins_pipe(istore_reg_mem);
7553 %}
7554
7555 instruct storeimmI0(immI0 zero, memory4 mem)
7556 %{
7557 match(Set mem(StoreI mem zero));
7558 predicate(!needs_releasing_store(n));
7559
7560 ins_cost(INSN_COST);
7561 format %{ "strw zr, $mem\t# int" %}
7562
7563 ins_encode(aarch64_enc_strw0(mem));
7564
7565 ins_pipe(istore_mem);
7566 %}
7567
7568 // Store Long (64 bit signed)
7569 instruct storeL(iRegL src, memory8 mem)
7570 %{
7571 match(Set mem (StoreL mem src));
7572 predicate(!needs_releasing_store(n));
7573
7574 ins_cost(INSN_COST);
7575 format %{ "str $src, $mem\t# int" %}
7576
7577 ins_encode(aarch64_enc_str(src, mem));
7578
7579 ins_pipe(istore_reg_mem);
7580 %}
7581
7582 // Store Long (64 bit signed)
7583 instruct storeimmL0(immL0 zero, memory8 mem)
7584 %{
7585 match(Set mem (StoreL mem zero));
7586 predicate(!needs_releasing_store(n));
7587
7588 ins_cost(INSN_COST);
7589 format %{ "str zr, $mem\t# int" %}
7590
7591 ins_encode(aarch64_enc_str0(mem));
7592
7593 ins_pipe(istore_mem);
7594 %}
7595
7596 // Store Pointer
7597 instruct storeP(iRegP src, memory8 mem)
7598 %{
7599 match(Set mem (StoreP mem src));
7600 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7601
7602 ins_cost(INSN_COST);
7603 format %{ "str $src, $mem\t# ptr" %}
7604
7605 ins_encode(aarch64_enc_str(src, mem));
7606
7607 ins_pipe(istore_reg_mem);
7608 %}
7609
7610 // Store Pointer
7611 instruct storeimmP0(immP0 zero, memory8 mem)
7612 %{
7613 match(Set mem (StoreP mem zero));
7614 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7615
7616 ins_cost(INSN_COST);
7617 format %{ "str zr, $mem\t# ptr" %}
7618
7619 ins_encode(aarch64_enc_str0(mem));
7620
7621 ins_pipe(istore_mem);
7622 %}
7623
7624 // Store Compressed Pointer
7625 instruct storeN(iRegN src, memory4 mem)
7626 %{
7627 match(Set mem (StoreN mem src));
7628 predicate(!needs_releasing_store(n));
7629
7630 ins_cost(INSN_COST);
7631 format %{ "strw $src, $mem\t# compressed ptr" %}
7632
7633 ins_encode(aarch64_enc_strw(src, mem));
7634
7635 ins_pipe(istore_reg_mem);
7636 %}
7637
7638 instruct storeImmN0(immN0 zero, memory4 mem)
7639 %{
7640 match(Set mem (StoreN mem zero));
7641 predicate(!needs_releasing_store(n));
7642
7643 ins_cost(INSN_COST);
7644 format %{ "strw zr, $mem\t# compressed ptr" %}
7645
7646 ins_encode(aarch64_enc_strw0(mem));
7647
7648 ins_pipe(istore_mem);
7649 %}
7650
7651 // Store Float
7652 instruct storeF(vRegF src, memory4 mem)
7653 %{
7654 match(Set mem (StoreF mem src));
7655 predicate(!needs_releasing_store(n));
7656
7657 ins_cost(INSN_COST);
7658 format %{ "strs $src, $mem\t# float" %}
7659
7660 ins_encode( aarch64_enc_strs(src, mem) );
7661
7662 ins_pipe(pipe_class_memory);
7663 %}
7664
7665 // TODO
7666 // implement storeImmF0 and storeFImmPacked
7667
7668 // Store Double
7669 instruct storeD(vRegD src, memory8 mem)
7670 %{
7671 match(Set mem (StoreD mem src));
7672 predicate(!needs_releasing_store(n));
7673
7674 ins_cost(INSN_COST);
7675 format %{ "strd $src, $mem\t# double" %}
7676
7677 ins_encode( aarch64_enc_strd(src, mem) );
7678
7679 ins_pipe(pipe_class_memory);
7680 %}
7681
7682 // Store Compressed Klass Pointer
7683 instruct storeNKlass(iRegN src, memory4 mem)
7684 %{
7685 predicate(!needs_releasing_store(n));
7686 match(Set mem (StoreNKlass mem src));
7687
7688 ins_cost(INSN_COST);
7689 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7690
7691 ins_encode(aarch64_enc_strw(src, mem));
7692
7693 ins_pipe(istore_reg_mem);
7694 %}
7695
7696 // TODO
7697 // implement storeImmD0 and storeDImmPacked
7698
7699 // prefetch instructions
7700 // Must be safe to execute with invalid address (cannot fault).
7701
7702 instruct prefetchalloc( memory8 mem ) %{
7703 match(PrefetchAllocation mem);
7704
7705 ins_cost(INSN_COST);
7706 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7707
7708 ins_encode( aarch64_enc_prefetchw(mem) );
7709
7710 ins_pipe(iload_prefetch);
7711 %}
7712
7713 // ---------------- volatile loads and stores ----------------
7714
7715 // Load Byte (8 bit signed)
7716 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7717 %{
7718 match(Set dst (LoadB mem));
7719
7720 ins_cost(VOLATILE_REF_COST);
7721 format %{ "ldarsb $dst, $mem\t# byte" %}
7722
7723 ins_encode(aarch64_enc_ldarsb(dst, mem));
7724
7725 ins_pipe(pipe_serial);
7726 %}
7727
7728 // Load Byte (8 bit signed) into long
7729 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7730 %{
7731 match(Set dst (ConvI2L (LoadB mem)));
7732
7733 ins_cost(VOLATILE_REF_COST);
7734 format %{ "ldarsb $dst, $mem\t# byte" %}
7735
7736 ins_encode(aarch64_enc_ldarsb(dst, mem));
7737
7738 ins_pipe(pipe_serial);
7739 %}
7740
7741 // Load Byte (8 bit unsigned)
7742 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7743 %{
7744 match(Set dst (LoadUB mem));
7745
7746 ins_cost(VOLATILE_REF_COST);
7747 format %{ "ldarb $dst, $mem\t# byte" %}
7748
7749 ins_encode(aarch64_enc_ldarb(dst, mem));
7750
7751 ins_pipe(pipe_serial);
7752 %}
7753
7754 // Load Byte (8 bit unsigned) into long
7755 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7756 %{
7757 match(Set dst (ConvI2L (LoadUB mem)));
7758
7759 ins_cost(VOLATILE_REF_COST);
7760 format %{ "ldarb $dst, $mem\t# byte" %}
7761
7762 ins_encode(aarch64_enc_ldarb(dst, mem));
7763
7764 ins_pipe(pipe_serial);
7765 %}
7766
7767 // Load Short (16 bit signed)
7768 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7769 %{
7770 match(Set dst (LoadS mem));
7771
7772 ins_cost(VOLATILE_REF_COST);
7773 format %{ "ldarshw $dst, $mem\t# short" %}
7774
7775 ins_encode(aarch64_enc_ldarshw(dst, mem));
7776
7777 ins_pipe(pipe_serial);
7778 %}
7779
7780 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7781 %{
7782 match(Set dst (LoadUS mem));
7783
7784 ins_cost(VOLATILE_REF_COST);
7785 format %{ "ldarhw $dst, $mem\t# short" %}
7786
7787 ins_encode(aarch64_enc_ldarhw(dst, mem));
7788
7789 ins_pipe(pipe_serial);
7790 %}
7791
7792 // Load Short/Char (16 bit unsigned) into long
7793 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7794 %{
7795 match(Set dst (ConvI2L (LoadUS mem)));
7796
7797 ins_cost(VOLATILE_REF_COST);
7798 format %{ "ldarh $dst, $mem\t# short" %}
7799
7800 ins_encode(aarch64_enc_ldarh(dst, mem));
7801
7802 ins_pipe(pipe_serial);
7803 %}
7804
7805 // Load Short/Char (16 bit signed) into long
7806 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7807 %{
7808 match(Set dst (ConvI2L (LoadS mem)));
7809
7810 ins_cost(VOLATILE_REF_COST);
7811 format %{ "ldarh $dst, $mem\t# short" %}
7812
7813 ins_encode(aarch64_enc_ldarsh(dst, mem));
7814
7815 ins_pipe(pipe_serial);
7816 %}
7817
7818 // Load Integer (32 bit signed)
7819 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7820 %{
7821 match(Set dst (LoadI mem));
7822
7823 ins_cost(VOLATILE_REF_COST);
7824 format %{ "ldarw $dst, $mem\t# int" %}
7825
7826 ins_encode(aarch64_enc_ldarw(dst, mem));
7827
7828 ins_pipe(pipe_serial);
7829 %}
7830
7831 // Load Integer (32 bit unsigned) into long
7832 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7833 %{
7834 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7835
7836 ins_cost(VOLATILE_REF_COST);
7837 format %{ "ldarw $dst, $mem\t# int" %}
7838
7839 ins_encode(aarch64_enc_ldarw(dst, mem));
7840
7841 ins_pipe(pipe_serial);
7842 %}
7843
7844 // Load Long (64 bit signed)
7845 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7846 %{
7847 match(Set dst (LoadL mem));
7848
7849 ins_cost(VOLATILE_REF_COST);
7850 format %{ "ldar $dst, $mem\t# int" %}
7851
7852 ins_encode(aarch64_enc_ldar(dst, mem));
7853
7854 ins_pipe(pipe_serial);
7855 %}
7856
7857 // Load Pointer
7858 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7859 %{
7860 match(Set dst (LoadP mem));
7861 predicate(n->as_Load()->barrier_data() == 0);
7862
7863 ins_cost(VOLATILE_REF_COST);
7864 format %{ "ldar $dst, $mem\t# ptr" %}
7865
7866 ins_encode(aarch64_enc_ldar(dst, mem));
7867
7868 ins_pipe(pipe_serial);
7869 %}
7870
7871 // Load Compressed Pointer
7872 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7873 %{
7874 match(Set dst (LoadN mem));
7875
7876 ins_cost(VOLATILE_REF_COST);
7877 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7878
7879 ins_encode(aarch64_enc_ldarw(dst, mem));
7880
7881 ins_pipe(pipe_serial);
7882 %}
7883
7884 // Load Float
7885 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7886 %{
7887 match(Set dst (LoadF mem));
7888
7889 ins_cost(VOLATILE_REF_COST);
7890 format %{ "ldars $dst, $mem\t# float" %}
7891
7892 ins_encode( aarch64_enc_fldars(dst, mem) );
7893
7894 ins_pipe(pipe_serial);
7895 %}
7896
7897 // Load Double
7898 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7899 %{
7900 match(Set dst (LoadD mem));
7901
7902 ins_cost(VOLATILE_REF_COST);
7903 format %{ "ldard $dst, $mem\t# double" %}
7904
7905 ins_encode( aarch64_enc_fldard(dst, mem) );
7906
7907 ins_pipe(pipe_serial);
7908 %}
7909
7910 // Store Byte
7911 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7912 %{
7913 match(Set mem (StoreB mem src));
7914
7915 ins_cost(VOLATILE_REF_COST);
7916 format %{ "stlrb $src, $mem\t# byte" %}
7917
7918 ins_encode(aarch64_enc_stlrb(src, mem));
7919
7920 ins_pipe(pipe_class_memory);
7921 %}
7922
7923 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7924 %{
7925 match(Set mem (StoreB mem zero));
7926
7927 ins_cost(VOLATILE_REF_COST);
7928 format %{ "stlrb zr, $mem\t# byte" %}
7929
7930 ins_encode(aarch64_enc_stlrb0(mem));
7931
7932 ins_pipe(pipe_class_memory);
7933 %}
7934
7935 // Store Char/Short
7936 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7937 %{
7938 match(Set mem (StoreC mem src));
7939
7940 ins_cost(VOLATILE_REF_COST);
7941 format %{ "stlrh $src, $mem\t# short" %}
7942
7943 ins_encode(aarch64_enc_stlrh(src, mem));
7944
7945 ins_pipe(pipe_class_memory);
7946 %}
7947
7948 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7949 %{
7950 match(Set mem (StoreC mem zero));
7951
7952 ins_cost(VOLATILE_REF_COST);
7953 format %{ "stlrh zr, $mem\t# short" %}
7954
7955 ins_encode(aarch64_enc_stlrh0(mem));
7956
7957 ins_pipe(pipe_class_memory);
7958 %}
7959
7960 // Store Integer
7961
7962 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7963 %{
7964 match(Set mem(StoreI mem src));
7965
7966 ins_cost(VOLATILE_REF_COST);
7967 format %{ "stlrw $src, $mem\t# int" %}
7968
7969 ins_encode(aarch64_enc_stlrw(src, mem));
7970
7971 ins_pipe(pipe_class_memory);
7972 %}
7973
7974 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7975 %{
7976 match(Set mem(StoreI mem zero));
7977
7978 ins_cost(VOLATILE_REF_COST);
7979 format %{ "stlrw zr, $mem\t# int" %}
7980
7981 ins_encode(aarch64_enc_stlrw0(mem));
7982
7983 ins_pipe(pipe_class_memory);
7984 %}
7985
7986 // Store Long (64 bit signed)
7987 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7988 %{
7989 match(Set mem (StoreL mem src));
7990
7991 ins_cost(VOLATILE_REF_COST);
7992 format %{ "stlr $src, $mem\t# int" %}
7993
7994 ins_encode(aarch64_enc_stlr(src, mem));
7995
7996 ins_pipe(pipe_class_memory);
7997 %}
7998
7999 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
8000 %{
8001 match(Set mem (StoreL mem zero));
8002
8003 ins_cost(VOLATILE_REF_COST);
8004 format %{ "stlr zr, $mem\t# int" %}
8005
8006 ins_encode(aarch64_enc_stlr0(mem));
8007
8008 ins_pipe(pipe_class_memory);
8009 %}
8010
8011 // Store Pointer
8012 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
8013 %{
8014 match(Set mem (StoreP mem src));
8015 predicate(n->as_Store()->barrier_data() == 0);
8016
8017 ins_cost(VOLATILE_REF_COST);
8018 format %{ "stlr $src, $mem\t# ptr" %}
8019
8020 ins_encode(aarch64_enc_stlr(src, mem));
8021
8022 ins_pipe(pipe_class_memory);
8023 %}
8024
8025 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
8026 %{
8027 match(Set mem (StoreP mem zero));
8028 predicate(n->as_Store()->barrier_data() == 0);
8029
8030 ins_cost(VOLATILE_REF_COST);
8031 format %{ "stlr zr, $mem\t# ptr" %}
8032
8033 ins_encode(aarch64_enc_stlr0(mem));
8034
8035 ins_pipe(pipe_class_memory);
8036 %}
8037
8038 // Store Compressed Pointer
8039 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
8040 %{
8041 match(Set mem (StoreN mem src));
8042
8043 ins_cost(VOLATILE_REF_COST);
8044 format %{ "stlrw $src, $mem\t# compressed ptr" %}
8045
8046 ins_encode(aarch64_enc_stlrw(src, mem));
8047
8048 ins_pipe(pipe_class_memory);
8049 %}
8050
8051 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
8052 %{
8053 match(Set mem (StoreN mem zero));
8054
8055 ins_cost(VOLATILE_REF_COST);
8056 format %{ "stlrw zr, $mem\t# compressed ptr" %}
8057
8058 ins_encode(aarch64_enc_stlrw0(mem));
8059
8060 ins_pipe(pipe_class_memory);
8061 %}
8062
8063 // Store Float
8064 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
8065 %{
8066 match(Set mem (StoreF mem src));
8067
8068 ins_cost(VOLATILE_REF_COST);
8069 format %{ "stlrs $src, $mem\t# float" %}
8070
8071 ins_encode( aarch64_enc_fstlrs(src, mem) );
8072
8073 ins_pipe(pipe_class_memory);
8074 %}
8075
8076 // TODO
8077 // implement storeImmF0 and storeFImmPacked
8078
8079 // Store Double
8080 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
8081 %{
8082 match(Set mem (StoreD mem src));
8083
8084 ins_cost(VOLATILE_REF_COST);
8085 format %{ "stlrd $src, $mem\t# double" %}
8086
8087 ins_encode( aarch64_enc_fstlrd(src, mem) );
8088
8089 ins_pipe(pipe_class_memory);
8090 %}
8091
8092 // ---------------- end of volatile loads and stores ----------------
8093
8094 instruct cacheWB(indirect addr)
8095 %{
8096 predicate(VM_Version::supports_data_cache_line_flush());
8097 match(CacheWB addr);
8098
8099 ins_cost(100);
8100 format %{"cache wb $addr" %}
8101 ins_encode %{
8102 assert($addr->index_position() < 0, "should be");
8103 assert($addr$$disp == 0, "should be");
8104 __ cache_wb(Address($addr$$base$$Register, 0));
8105 %}
8106 ins_pipe(pipe_slow); // XXX
8107 %}
8108
8109 instruct cacheWBPreSync()
8110 %{
8111 predicate(VM_Version::supports_data_cache_line_flush());
8112 match(CacheWBPreSync);
8113
8114 ins_cost(100);
8115 format %{"cache wb presync" %}
8116 ins_encode %{
8117 __ cache_wbsync(true);
8118 %}
8119 ins_pipe(pipe_slow); // XXX
8120 %}
8121
8122 instruct cacheWBPostSync()
8123 %{
8124 predicate(VM_Version::supports_data_cache_line_flush());
8125 match(CacheWBPostSync);
8126
8127 ins_cost(100);
8128 format %{"cache wb postsync" %}
8129 ins_encode %{
8130 __ cache_wbsync(false);
8131 %}
8132 ins_pipe(pipe_slow); // XXX
8133 %}
8134
8135 // ============================================================================
8136 // BSWAP Instructions
8137
8138 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
8139 match(Set dst (ReverseBytesI src));
8140
8141 ins_cost(INSN_COST);
8142 format %{ "revw $dst, $src" %}
8143
8144 ins_encode %{
8145 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
8146 %}
8147
8148 ins_pipe(ialu_reg);
8149 %}
8150
8151 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
8152 match(Set dst (ReverseBytesL src));
8153
8154 ins_cost(INSN_COST);
8155 format %{ "rev $dst, $src" %}
8156
8157 ins_encode %{
8158 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
8159 %}
8160
8161 ins_pipe(ialu_reg);
8162 %}
8163
8164 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
8165 match(Set dst (ReverseBytesUS src));
8166
8167 ins_cost(INSN_COST);
8168 format %{ "rev16w $dst, $src" %}
8169
8170 ins_encode %{
8171 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8172 %}
8173
8174 ins_pipe(ialu_reg);
8175 %}
8176
8177 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
8178 match(Set dst (ReverseBytesS src));
8179
8180 ins_cost(INSN_COST);
8181 format %{ "rev16w $dst, $src\n\t"
8182 "sbfmw $dst, $dst, #0, #15" %}
8183
8184 ins_encode %{
8185 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8186 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
8187 %}
8188
8189 ins_pipe(ialu_reg);
8190 %}
8191
8192 // ============================================================================
8193 // Zero Count Instructions
8194
8195 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8196 match(Set dst (CountLeadingZerosI src));
8197
8198 ins_cost(INSN_COST);
8199 format %{ "clzw $dst, $src" %}
8200 ins_encode %{
8201 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
8202 %}
8203
8204 ins_pipe(ialu_reg);
8205 %}
8206
8207 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
8208 match(Set dst (CountLeadingZerosL src));
8209
8210 ins_cost(INSN_COST);
8211 format %{ "clz $dst, $src" %}
8212 ins_encode %{
8213 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
8214 %}
8215
8216 ins_pipe(ialu_reg);
8217 %}
8218
8219 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8220 match(Set dst (CountTrailingZerosI src));
8221
8222 ins_cost(INSN_COST * 2);
8223 format %{ "rbitw $dst, $src\n\t"
8224 "clzw $dst, $dst" %}
8225 ins_encode %{
8226 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
8227 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
8228 %}
8229
8230 ins_pipe(ialu_reg);
8231 %}
8232
8233 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
8234 match(Set dst (CountTrailingZerosL src));
8235
8236 ins_cost(INSN_COST * 2);
8237 format %{ "rbit $dst, $src\n\t"
8238 "clz $dst, $dst" %}
8239 ins_encode %{
8240 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
8241 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
8242 %}
8243
8244 ins_pipe(ialu_reg);
8245 %}
8246
8247 //---------- Population Count Instructions -------------------------------------
8248 //
8249
8250 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
8251 match(Set dst (PopCountI src));
8252 effect(TEMP tmp);
8253 ins_cost(INSN_COST * 13);
8254
8255 format %{ "movw $src, $src\n\t"
8256 "mov $tmp, $src\t# vector (1D)\n\t"
8257 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8258 "addv $tmp, $tmp\t# vector (8B)\n\t"
8259 "mov $dst, $tmp\t# vector (1D)" %}
8260 ins_encode %{
8261 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
8262 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8263 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8264 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8265 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8266 %}
8267
8268 ins_pipe(pipe_class_default);
8269 %}
8270
8271 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
8272 match(Set dst (PopCountI (LoadI mem)));
8273 effect(TEMP tmp);
8274 ins_cost(INSN_COST * 13);
8275
8276 format %{ "ldrs $tmp, $mem\n\t"
8277 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8278 "addv $tmp, $tmp\t# vector (8B)\n\t"
8279 "mov $dst, $tmp\t# vector (1D)" %}
8280 ins_encode %{
8281 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8282 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
8283 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
8284 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8285 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8286 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8287 %}
8288
8289 ins_pipe(pipe_class_default);
8290 %}
8291
8292 // Note: Long.bitCount(long) returns an int.
8293 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
8294 match(Set dst (PopCountL src));
8295 effect(TEMP tmp);
8296 ins_cost(INSN_COST * 13);
8297
8298 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
8299 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8300 "addv $tmp, $tmp\t# vector (8B)\n\t"
8301 "mov $dst, $tmp\t# vector (1D)" %}
8302 ins_encode %{
8303 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8304 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8305 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8306 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8307 %}
8308
8309 ins_pipe(pipe_class_default);
8310 %}
8311
8312 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
8313 match(Set dst (PopCountL (LoadL mem)));
8314 effect(TEMP tmp);
8315 ins_cost(INSN_COST * 13);
8316
8317 format %{ "ldrd $tmp, $mem\n\t"
8318 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8319 "addv $tmp, $tmp\t# vector (8B)\n\t"
8320 "mov $dst, $tmp\t# vector (1D)" %}
8321 ins_encode %{
8322 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8323 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8324 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
8325 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8326 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8327 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8328 %}
8329
8330 ins_pipe(pipe_class_default);
8331 %}
8332
8333 // ============================================================================
8334 // MemBar Instruction
8335
8336 instruct load_fence() %{
8337 match(LoadFence);
8338 ins_cost(VOLATILE_REF_COST);
8339
8340 format %{ "load_fence" %}
8341
8342 ins_encode %{
8343 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8344 %}
8345 ins_pipe(pipe_serial);
8346 %}
8347
8348 instruct unnecessary_membar_acquire() %{
8349 predicate(unnecessary_acquire(n));
8350 match(MemBarAcquire);
8351 ins_cost(0);
8352
8353 format %{ "membar_acquire (elided)" %}
8354
8355 ins_encode %{
8356 __ block_comment("membar_acquire (elided)");
8357 %}
8358
8359 ins_pipe(pipe_class_empty);
8360 %}
8361
8362 instruct membar_acquire() %{
8363 match(MemBarAcquire);
8364 ins_cost(VOLATILE_REF_COST);
8365
8366 format %{ "membar_acquire\n\t"
8367 "dmb ish" %}
8368
8369 ins_encode %{
8370 __ block_comment("membar_acquire");
8371 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8372 %}
8373
8374 ins_pipe(pipe_serial);
8375 %}
8376
8377
8378 instruct membar_acquire_lock() %{
8379 match(MemBarAcquireLock);
8380 ins_cost(VOLATILE_REF_COST);
8381
8382 format %{ "membar_acquire_lock (elided)" %}
8383
8384 ins_encode %{
8385 __ block_comment("membar_acquire_lock (elided)");
8386 %}
8387
8388 ins_pipe(pipe_serial);
8389 %}
8390
8391 instruct store_fence() %{
8392 match(StoreFence);
8393 ins_cost(VOLATILE_REF_COST);
8394
8395 format %{ "store_fence" %}
8396
8397 ins_encode %{
8398 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8399 %}
8400 ins_pipe(pipe_serial);
8401 %}
8402
8403 instruct unnecessary_membar_release() %{
8404 predicate(unnecessary_release(n));
8405 match(MemBarRelease);
8406 ins_cost(0);
8407
8408 format %{ "membar_release (elided)" %}
8409
8410 ins_encode %{
8411 __ block_comment("membar_release (elided)");
8412 %}
8413 ins_pipe(pipe_serial);
8414 %}
8415
8416 instruct membar_release() %{
8417 match(MemBarRelease);
8418 ins_cost(VOLATILE_REF_COST);
8419
8420 format %{ "membar_release\n\t"
8421 "dmb ish" %}
8422
8423 ins_encode %{
8424 __ block_comment("membar_release");
8425 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8426 %}
8427 ins_pipe(pipe_serial);
8428 %}
8429
8430 instruct membar_storestore() %{
8431 match(MemBarStoreStore);
8432 match(StoreStoreFence);
8433 ins_cost(VOLATILE_REF_COST);
8434
8435 format %{ "MEMBAR-store-store" %}
8436
8437 ins_encode %{
8438 __ membar(Assembler::StoreStore);
8439 %}
8440 ins_pipe(pipe_serial);
8441 %}
8442
8443 instruct membar_release_lock() %{
8444 match(MemBarReleaseLock);
8445 ins_cost(VOLATILE_REF_COST);
8446
8447 format %{ "membar_release_lock (elided)" %}
8448
8449 ins_encode %{
8450 __ block_comment("membar_release_lock (elided)");
8451 %}
8452
8453 ins_pipe(pipe_serial);
8454 %}
8455
8456 instruct unnecessary_membar_volatile() %{
8457 predicate(unnecessary_volatile(n));
8458 match(MemBarVolatile);
8459 ins_cost(0);
8460
8461 format %{ "membar_volatile (elided)" %}
8462
8463 ins_encode %{
8464 __ block_comment("membar_volatile (elided)");
8465 %}
8466
8467 ins_pipe(pipe_serial);
8468 %}
8469
8470 instruct membar_volatile() %{
8471 match(MemBarVolatile);
8472 ins_cost(VOLATILE_REF_COST*100);
8473
8474 format %{ "membar_volatile\n\t"
8475 "dmb ish"%}
8476
8477 ins_encode %{
8478 __ block_comment("membar_volatile");
8479 __ membar(Assembler::StoreLoad);
8480 %}
8481
8482 ins_pipe(pipe_serial);
8483 %}
8484
8485 // ============================================================================
8486 // Cast/Convert Instructions
8487
8488 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8489 match(Set dst (CastX2P src));
8490
8491 ins_cost(INSN_COST);
8492 format %{ "mov $dst, $src\t# long -> ptr" %}
8493
8494 ins_encode %{
8495 if ($dst$$reg != $src$$reg) {
8496 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8497 }
8498 %}
8499
8500 ins_pipe(ialu_reg);
8501 %}
8502
8503 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8504 match(Set dst (CastP2X src));
8505
8506 ins_cost(INSN_COST);
8507 format %{ "mov $dst, $src\t# ptr -> long" %}
8508
8509 ins_encode %{
8510 if ($dst$$reg != $src$$reg) {
8511 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8512 }
8513 %}
8514
8515 ins_pipe(ialu_reg);
8516 %}
8517
8518 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8519 match(Set dst (CastP2X src));
8520
8521 ins_cost(INSN_COST);
8522 format %{ "mov $dst, $src\t# ptr -> long" %}
8523
8524 ins_encode %{
8525 if ($dst$$reg != $src$$reg) {
8526 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8527 }
8528 %}
8529
8530 ins_pipe(ialu_reg);
8531 %}
8532
8533 // Convert oop into int for vectors alignment masking
8534 instruct convP2I(iRegINoSp dst, iRegP src) %{
8535 match(Set dst (ConvL2I (CastP2X src)));
8536
8537 ins_cost(INSN_COST);
8538 format %{ "movw $dst, $src\t# ptr -> int" %}
8539 ins_encode %{
8540 __ movw($dst$$Register, $src$$Register);
8541 %}
8542
8543 ins_pipe(ialu_reg);
8544 %}
8545
8546 // Convert compressed oop into int for vectors alignment masking
8547 // in case of 32bit oops (heap < 4Gb).
8548 instruct convN2I(iRegINoSp dst, iRegN src)
8549 %{
8550 predicate(CompressedOops::shift() == 0);
8551 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8552
8553 ins_cost(INSN_COST);
8554 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8555 ins_encode %{
8556 __ movw($dst$$Register, $src$$Register);
8557 %}
8558
8559 ins_pipe(ialu_reg);
8560 %}
8561
8562
8563 // Convert oop pointer into compressed form
8564 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8565 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8566 match(Set dst (EncodeP src));
8567 effect(KILL cr);
8568 ins_cost(INSN_COST * 3);
8569 format %{ "encode_heap_oop $dst, $src" %}
8570 ins_encode %{
8571 Register s = $src$$Register;
8572 Register d = $dst$$Register;
8573 __ encode_heap_oop(d, s);
8574 %}
8575 ins_pipe(ialu_reg);
8576 %}
8577
8578 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8579 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8580 match(Set dst (EncodeP src));
8581 ins_cost(INSN_COST * 3);
8582 format %{ "encode_heap_oop_not_null $dst, $src" %}
8583 ins_encode %{
8584 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8585 %}
8586 ins_pipe(ialu_reg);
8587 %}
8588
8589 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8590 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8591 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8592 match(Set dst (DecodeN src));
8593 ins_cost(INSN_COST * 3);
8594 format %{ "decode_heap_oop $dst, $src" %}
8595 ins_encode %{
8596 Register s = $src$$Register;
8597 Register d = $dst$$Register;
8598 __ decode_heap_oop(d, s);
8599 %}
8600 ins_pipe(ialu_reg);
8601 %}
8602
8603 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8604 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8605 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8606 match(Set dst (DecodeN src));
8607 ins_cost(INSN_COST * 3);
8608 format %{ "decode_heap_oop_not_null $dst, $src" %}
8609 ins_encode %{
8610 Register s = $src$$Register;
8611 Register d = $dst$$Register;
8612 __ decode_heap_oop_not_null(d, s);
8613 %}
8614 ins_pipe(ialu_reg);
8615 %}
8616
8617 // n.b. AArch64 implementations of encode_klass_not_null and
8618 // decode_klass_not_null do not modify the flags register so, unlike
8619 // Intel, we don't kill CR as a side effect here
8620
8621 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8622 match(Set dst (EncodePKlass src));
8623
8624 ins_cost(INSN_COST * 3);
8625 format %{ "encode_klass_not_null $dst,$src" %}
8626
8627 ins_encode %{
8628 Register src_reg = as_Register($src$$reg);
8629 Register dst_reg = as_Register($dst$$reg);
8630 __ encode_klass_not_null(dst_reg, src_reg);
8631 %}
8632
8633 ins_pipe(ialu_reg);
8634 %}
8635
8636 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8637 match(Set dst (DecodeNKlass src));
8638
8639 ins_cost(INSN_COST * 3);
8640 format %{ "decode_klass_not_null $dst,$src" %}
8641
8642 ins_encode %{
8643 Register src_reg = as_Register($src$$reg);
8644 Register dst_reg = as_Register($dst$$reg);
8645 if (dst_reg != src_reg) {
8646 __ decode_klass_not_null(dst_reg, src_reg);
8647 } else {
8648 __ decode_klass_not_null(dst_reg);
8649 }
8650 %}
8651
8652 ins_pipe(ialu_reg);
8653 %}
8654
8655 instruct checkCastPP(iRegPNoSp dst)
8656 %{
8657 match(Set dst (CheckCastPP dst));
8658
8659 size(0);
8660 format %{ "# checkcastPP of $dst" %}
8661 ins_encode(/* empty encoding */);
8662 ins_pipe(pipe_class_empty);
8663 %}
8664
8665 instruct castPP(iRegPNoSp dst)
8666 %{
8667 match(Set dst (CastPP dst));
8668
8669 size(0);
8670 format %{ "# castPP of $dst" %}
8671 ins_encode(/* empty encoding */);
8672 ins_pipe(pipe_class_empty);
8673 %}
8674
8675 instruct castII(iRegI dst)
8676 %{
8677 match(Set dst (CastII dst));
8678
8679 size(0);
8680 format %{ "# castII of $dst" %}
8681 ins_encode(/* empty encoding */);
8682 ins_cost(0);
8683 ins_pipe(pipe_class_empty);
8684 %}
8685
8686 instruct castLL(iRegL dst)
8687 %{
8688 match(Set dst (CastLL dst));
8689
8690 size(0);
8691 format %{ "# castLL of $dst" %}
8692 ins_encode(/* empty encoding */);
8693 ins_cost(0);
8694 ins_pipe(pipe_class_empty);
8695 %}
8696
8697 instruct castFF(vRegF dst)
8698 %{
8699 match(Set dst (CastFF dst));
8700
8701 size(0);
8702 format %{ "# castFF of $dst" %}
8703 ins_encode(/* empty encoding */);
8704 ins_cost(0);
8705 ins_pipe(pipe_class_empty);
8706 %}
8707
8708 instruct castDD(vRegD dst)
8709 %{
8710 match(Set dst (CastDD dst));
8711
8712 size(0);
8713 format %{ "# castDD of $dst" %}
8714 ins_encode(/* empty encoding */);
8715 ins_cost(0);
8716 ins_pipe(pipe_class_empty);
8717 %}
8718
8719 instruct castVV(vReg dst)
8720 %{
8721 match(Set dst (CastVV dst));
8722
8723 size(0);
8724 format %{ "# castVV of $dst" %}
8725 ins_encode(/* empty encoding */);
8726 ins_cost(0);
8727 ins_pipe(pipe_class_empty);
8728 %}
8729
8730 instruct castVVMask(pRegGov dst)
8731 %{
8732 match(Set dst (CastVV dst));
8733
8734 size(0);
8735 format %{ "# castVV of $dst" %}
8736 ins_encode(/* empty encoding */);
8737 ins_cost(0);
8738 ins_pipe(pipe_class_empty);
8739 %}
8740
8741 // ============================================================================
8742 // Atomic operation instructions
8743 //
8744
8745 // standard CompareAndSwapX when we are using barriers
8746 // these have higher priority than the rules selected by a predicate
8747
8748 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8749 // can't match them
8750
8751 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8752
8753 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8754 ins_cost(2 * VOLATILE_REF_COST);
8755
8756 effect(KILL cr);
8757
8758 format %{
8759 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8760 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8761 %}
8762
8763 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8764 aarch64_enc_cset_eq(res));
8765
8766 ins_pipe(pipe_slow);
8767 %}
8768
8769 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8770
8771 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8772 ins_cost(2 * VOLATILE_REF_COST);
8773
8774 effect(KILL cr);
8775
8776 format %{
8777 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8778 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8779 %}
8780
8781 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8782 aarch64_enc_cset_eq(res));
8783
8784 ins_pipe(pipe_slow);
8785 %}
8786
8787 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8788
8789 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8790 ins_cost(2 * VOLATILE_REF_COST);
8791
8792 effect(KILL cr);
8793
8794 format %{
8795 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8796 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8797 %}
8798
8799 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8800 aarch64_enc_cset_eq(res));
8801
8802 ins_pipe(pipe_slow);
8803 %}
8804
8805 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8806
8807 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8808 ins_cost(2 * VOLATILE_REF_COST);
8809
8810 effect(KILL cr);
8811
8812 format %{
8813 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8814 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8815 %}
8816
8817 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8818 aarch64_enc_cset_eq(res));
8819
8820 ins_pipe(pipe_slow);
8821 %}
8822
8823 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8824
8825 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8826 predicate(n->as_LoadStore()->barrier_data() == 0);
8827 ins_cost(2 * VOLATILE_REF_COST);
8828
8829 effect(KILL cr);
8830
8831 format %{
8832 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8833 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8834 %}
8835
8836 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8837 aarch64_enc_cset_eq(res));
8838
8839 ins_pipe(pipe_slow);
8840 %}
8841
8842 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8843
8844 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8845 ins_cost(2 * VOLATILE_REF_COST);
8846
8847 effect(KILL cr);
8848
8849 format %{
8850 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8851 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8852 %}
8853
8854 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8855 aarch64_enc_cset_eq(res));
8856
8857 ins_pipe(pipe_slow);
8858 %}
8859
8860 // alternative CompareAndSwapX when we are eliding barriers
8861
8862 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8863
8864 predicate(needs_acquiring_load_exclusive(n));
8865 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8866 ins_cost(VOLATILE_REF_COST);
8867
8868 effect(KILL cr);
8869
8870 format %{
8871 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8872 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8873 %}
8874
8875 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8876 aarch64_enc_cset_eq(res));
8877
8878 ins_pipe(pipe_slow);
8879 %}
8880
8881 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8882
8883 predicate(needs_acquiring_load_exclusive(n));
8884 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8885 ins_cost(VOLATILE_REF_COST);
8886
8887 effect(KILL cr);
8888
8889 format %{
8890 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8891 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8892 %}
8893
8894 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8895 aarch64_enc_cset_eq(res));
8896
8897 ins_pipe(pipe_slow);
8898 %}
8899
8900 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8901
8902 predicate(needs_acquiring_load_exclusive(n));
8903 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8904 ins_cost(VOLATILE_REF_COST);
8905
8906 effect(KILL cr);
8907
8908 format %{
8909 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8910 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8911 %}
8912
8913 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8914 aarch64_enc_cset_eq(res));
8915
8916 ins_pipe(pipe_slow);
8917 %}
8918
8919 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8920
8921 predicate(needs_acquiring_load_exclusive(n));
8922 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8923 ins_cost(VOLATILE_REF_COST);
8924
8925 effect(KILL cr);
8926
8927 format %{
8928 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8929 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8930 %}
8931
8932 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8933 aarch64_enc_cset_eq(res));
8934
8935 ins_pipe(pipe_slow);
8936 %}
8937
8938 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8939
8940 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8941 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8942 ins_cost(VOLATILE_REF_COST);
8943
8944 effect(KILL cr);
8945
8946 format %{
8947 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8948 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8949 %}
8950
8951 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8952 aarch64_enc_cset_eq(res));
8953
8954 ins_pipe(pipe_slow);
8955 %}
8956
8957 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8958
8959 predicate(needs_acquiring_load_exclusive(n));
8960 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8961 ins_cost(VOLATILE_REF_COST);
8962
8963 effect(KILL cr);
8964
8965 format %{
8966 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8967 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8968 %}
8969
8970 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8971 aarch64_enc_cset_eq(res));
8972
8973 ins_pipe(pipe_slow);
8974 %}
8975
8976
8977 // ---------------------------------------------------------------------
8978
8979 // BEGIN This section of the file is automatically generated. Do not edit --------------
8980
8981 // Sundry CAS operations. Note that release is always true,
8982 // regardless of the memory ordering of the CAS. This is because we
8983 // need the volatile case to be sequentially consistent but there is
8984 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8985 // can't check the type of memory ordering here, so we always emit a
8986 // STLXR.
8987
8988 // This section is generated from aarch64_ad_cas.m4
8989
8990
8991
8992 // This pattern is generated automatically from cas.m4.
8993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8994 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8995
8996 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8997 ins_cost(2 * VOLATILE_REF_COST);
8998 effect(TEMP_DEF res, KILL cr);
8999 format %{
9000 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9001 %}
9002 ins_encode %{
9003 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9004 Assembler::byte, /*acquire*/ false, /*release*/ true,
9005 /*weak*/ false, $res$$Register);
9006 __ sxtbw($res$$Register, $res$$Register);
9007 %}
9008 ins_pipe(pipe_slow);
9009 %}
9010
9011 // This pattern is generated automatically from cas.m4.
9012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9013 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9014
9015 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9016 ins_cost(2 * VOLATILE_REF_COST);
9017 effect(TEMP_DEF res, KILL cr);
9018 format %{
9019 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9020 %}
9021 ins_encode %{
9022 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9023 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9024 /*weak*/ false, $res$$Register);
9025 __ sxthw($res$$Register, $res$$Register);
9026 %}
9027 ins_pipe(pipe_slow);
9028 %}
9029
9030 // This pattern is generated automatically from cas.m4.
9031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9032 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9033
9034 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9035 ins_cost(2 * VOLATILE_REF_COST);
9036 effect(TEMP_DEF res, KILL cr);
9037 format %{
9038 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9039 %}
9040 ins_encode %{
9041 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9042 Assembler::word, /*acquire*/ false, /*release*/ true,
9043 /*weak*/ false, $res$$Register);
9044 %}
9045 ins_pipe(pipe_slow);
9046 %}
9047
9048 // This pattern is generated automatically from cas.m4.
9049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9050 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9051
9052 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9053 ins_cost(2 * VOLATILE_REF_COST);
9054 effect(TEMP_DEF res, KILL cr);
9055 format %{
9056 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9057 %}
9058 ins_encode %{
9059 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9060 Assembler::xword, /*acquire*/ false, /*release*/ true,
9061 /*weak*/ false, $res$$Register);
9062 %}
9063 ins_pipe(pipe_slow);
9064 %}
9065
9066 // This pattern is generated automatically from cas.m4.
9067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9068 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9069
9070 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9071 ins_cost(2 * VOLATILE_REF_COST);
9072 effect(TEMP_DEF res, KILL cr);
9073 format %{
9074 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9075 %}
9076 ins_encode %{
9077 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9078 Assembler::word, /*acquire*/ false, /*release*/ true,
9079 /*weak*/ false, $res$$Register);
9080 %}
9081 ins_pipe(pipe_slow);
9082 %}
9083
9084 // This pattern is generated automatically from cas.m4.
9085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9086 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9087 predicate(n->as_LoadStore()->barrier_data() == 0);
9088 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9089 ins_cost(2 * VOLATILE_REF_COST);
9090 effect(TEMP_DEF res, KILL cr);
9091 format %{
9092 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9093 %}
9094 ins_encode %{
9095 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9096 Assembler::xword, /*acquire*/ false, /*release*/ true,
9097 /*weak*/ false, $res$$Register);
9098 %}
9099 ins_pipe(pipe_slow);
9100 %}
9101
9102 // This pattern is generated automatically from cas.m4.
9103 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9104 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9105 predicate(needs_acquiring_load_exclusive(n));
9106 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9107 ins_cost(VOLATILE_REF_COST);
9108 effect(TEMP_DEF res, KILL cr);
9109 format %{
9110 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9111 %}
9112 ins_encode %{
9113 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9114 Assembler::byte, /*acquire*/ true, /*release*/ true,
9115 /*weak*/ false, $res$$Register);
9116 __ sxtbw($res$$Register, $res$$Register);
9117 %}
9118 ins_pipe(pipe_slow);
9119 %}
9120
9121 // This pattern is generated automatically from cas.m4.
9122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9123 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9124 predicate(needs_acquiring_load_exclusive(n));
9125 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9126 ins_cost(VOLATILE_REF_COST);
9127 effect(TEMP_DEF res, KILL cr);
9128 format %{
9129 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9130 %}
9131 ins_encode %{
9132 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9133 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9134 /*weak*/ false, $res$$Register);
9135 __ sxthw($res$$Register, $res$$Register);
9136 %}
9137 ins_pipe(pipe_slow);
9138 %}
9139
9140 // This pattern is generated automatically from cas.m4.
9141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9142 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9143 predicate(needs_acquiring_load_exclusive(n));
9144 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9145 ins_cost(VOLATILE_REF_COST);
9146 effect(TEMP_DEF res, KILL cr);
9147 format %{
9148 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9149 %}
9150 ins_encode %{
9151 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9152 Assembler::word, /*acquire*/ true, /*release*/ true,
9153 /*weak*/ false, $res$$Register);
9154 %}
9155 ins_pipe(pipe_slow);
9156 %}
9157
9158 // This pattern is generated automatically from cas.m4.
9159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9160 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9161 predicate(needs_acquiring_load_exclusive(n));
9162 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9163 ins_cost(VOLATILE_REF_COST);
9164 effect(TEMP_DEF res, KILL cr);
9165 format %{
9166 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9167 %}
9168 ins_encode %{
9169 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9170 Assembler::xword, /*acquire*/ true, /*release*/ true,
9171 /*weak*/ false, $res$$Register);
9172 %}
9173 ins_pipe(pipe_slow);
9174 %}
9175
9176 // This pattern is generated automatically from cas.m4.
9177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9178 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9179 predicate(needs_acquiring_load_exclusive(n));
9180 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9181 ins_cost(VOLATILE_REF_COST);
9182 effect(TEMP_DEF res, KILL cr);
9183 format %{
9184 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9185 %}
9186 ins_encode %{
9187 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9188 Assembler::word, /*acquire*/ true, /*release*/ true,
9189 /*weak*/ false, $res$$Register);
9190 %}
9191 ins_pipe(pipe_slow);
9192 %}
9193
9194 // This pattern is generated automatically from cas.m4.
9195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9196 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9197 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9198 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9199 ins_cost(VOLATILE_REF_COST);
9200 effect(TEMP_DEF res, KILL cr);
9201 format %{
9202 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9203 %}
9204 ins_encode %{
9205 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9206 Assembler::xword, /*acquire*/ true, /*release*/ true,
9207 /*weak*/ false, $res$$Register);
9208 %}
9209 ins_pipe(pipe_slow);
9210 %}
9211
9212 // This pattern is generated automatically from cas.m4.
9213 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9214 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9215
9216 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9217 ins_cost(2 * VOLATILE_REF_COST);
9218 effect(KILL cr);
9219 format %{
9220 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9221 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9222 %}
9223 ins_encode %{
9224 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9225 Assembler::byte, /*acquire*/ false, /*release*/ true,
9226 /*weak*/ true, noreg);
9227 __ csetw($res$$Register, Assembler::EQ);
9228 %}
9229 ins_pipe(pipe_slow);
9230 %}
9231
9232 // This pattern is generated automatically from cas.m4.
9233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9234 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9235
9236 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9237 ins_cost(2 * VOLATILE_REF_COST);
9238 effect(KILL cr);
9239 format %{
9240 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9241 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9242 %}
9243 ins_encode %{
9244 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9245 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9246 /*weak*/ true, noreg);
9247 __ csetw($res$$Register, Assembler::EQ);
9248 %}
9249 ins_pipe(pipe_slow);
9250 %}
9251
9252 // This pattern is generated automatically from cas.m4.
9253 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9254 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9255
9256 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9257 ins_cost(2 * VOLATILE_REF_COST);
9258 effect(KILL cr);
9259 format %{
9260 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9261 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9262 %}
9263 ins_encode %{
9264 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9265 Assembler::word, /*acquire*/ false, /*release*/ true,
9266 /*weak*/ true, noreg);
9267 __ csetw($res$$Register, Assembler::EQ);
9268 %}
9269 ins_pipe(pipe_slow);
9270 %}
9271
9272 // This pattern is generated automatically from cas.m4.
9273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9274 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9275
9276 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9277 ins_cost(2 * VOLATILE_REF_COST);
9278 effect(KILL cr);
9279 format %{
9280 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9281 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9282 %}
9283 ins_encode %{
9284 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9285 Assembler::xword, /*acquire*/ false, /*release*/ true,
9286 /*weak*/ true, noreg);
9287 __ csetw($res$$Register, Assembler::EQ);
9288 %}
9289 ins_pipe(pipe_slow);
9290 %}
9291
9292 // This pattern is generated automatically from cas.m4.
9293 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9294 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9295
9296 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9297 ins_cost(2 * VOLATILE_REF_COST);
9298 effect(KILL cr);
9299 format %{
9300 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9301 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9302 %}
9303 ins_encode %{
9304 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9305 Assembler::word, /*acquire*/ false, /*release*/ true,
9306 /*weak*/ true, noreg);
9307 __ csetw($res$$Register, Assembler::EQ);
9308 %}
9309 ins_pipe(pipe_slow);
9310 %}
9311
9312 // This pattern is generated automatically from cas.m4.
9313 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9314 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9315 predicate(n->as_LoadStore()->barrier_data() == 0);
9316 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9317 ins_cost(2 * VOLATILE_REF_COST);
9318 effect(KILL cr);
9319 format %{
9320 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9321 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9322 %}
9323 ins_encode %{
9324 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9325 Assembler::xword, /*acquire*/ false, /*release*/ true,
9326 /*weak*/ true, noreg);
9327 __ csetw($res$$Register, Assembler::EQ);
9328 %}
9329 ins_pipe(pipe_slow);
9330 %}
9331
9332 // This pattern is generated automatically from cas.m4.
9333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9334 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9335 predicate(needs_acquiring_load_exclusive(n));
9336 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9337 ins_cost(VOLATILE_REF_COST);
9338 effect(KILL cr);
9339 format %{
9340 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9341 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9342 %}
9343 ins_encode %{
9344 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9345 Assembler::byte, /*acquire*/ true, /*release*/ true,
9346 /*weak*/ true, noreg);
9347 __ csetw($res$$Register, Assembler::EQ);
9348 %}
9349 ins_pipe(pipe_slow);
9350 %}
9351
9352 // This pattern is generated automatically from cas.m4.
9353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9354 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9355 predicate(needs_acquiring_load_exclusive(n));
9356 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9357 ins_cost(VOLATILE_REF_COST);
9358 effect(KILL cr);
9359 format %{
9360 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9361 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9362 %}
9363 ins_encode %{
9364 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9365 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9366 /*weak*/ true, noreg);
9367 __ csetw($res$$Register, Assembler::EQ);
9368 %}
9369 ins_pipe(pipe_slow);
9370 %}
9371
9372 // This pattern is generated automatically from cas.m4.
9373 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9374 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9375 predicate(needs_acquiring_load_exclusive(n));
9376 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9377 ins_cost(VOLATILE_REF_COST);
9378 effect(KILL cr);
9379 format %{
9380 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9381 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9382 %}
9383 ins_encode %{
9384 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9385 Assembler::word, /*acquire*/ true, /*release*/ true,
9386 /*weak*/ true, noreg);
9387 __ csetw($res$$Register, Assembler::EQ);
9388 %}
9389 ins_pipe(pipe_slow);
9390 %}
9391
9392 // This pattern is generated automatically from cas.m4.
9393 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9394 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9395 predicate(needs_acquiring_load_exclusive(n));
9396 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9397 ins_cost(VOLATILE_REF_COST);
9398 effect(KILL cr);
9399 format %{
9400 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9401 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9402 %}
9403 ins_encode %{
9404 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9405 Assembler::xword, /*acquire*/ true, /*release*/ true,
9406 /*weak*/ true, noreg);
9407 __ csetw($res$$Register, Assembler::EQ);
9408 %}
9409 ins_pipe(pipe_slow);
9410 %}
9411
9412 // This pattern is generated automatically from cas.m4.
9413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9414 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9415 predicate(needs_acquiring_load_exclusive(n));
9416 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9417 ins_cost(VOLATILE_REF_COST);
9418 effect(KILL cr);
9419 format %{
9420 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9421 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9422 %}
9423 ins_encode %{
9424 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9425 Assembler::word, /*acquire*/ true, /*release*/ true,
9426 /*weak*/ true, noreg);
9427 __ csetw($res$$Register, Assembler::EQ);
9428 %}
9429 ins_pipe(pipe_slow);
9430 %}
9431
9432 // This pattern is generated automatically from cas.m4.
9433 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9434 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9435 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9436 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9437 ins_cost(VOLATILE_REF_COST);
9438 effect(KILL cr);
9439 format %{
9440 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9441 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9442 %}
9443 ins_encode %{
9444 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9445 Assembler::xword, /*acquire*/ true, /*release*/ true,
9446 /*weak*/ true, noreg);
9447 __ csetw($res$$Register, Assembler::EQ);
9448 %}
9449 ins_pipe(pipe_slow);
9450 %}
9451
9452 // END This section of the file is automatically generated. Do not edit --------------
9453 // ---------------------------------------------------------------------
9454
9455 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9456 match(Set prev (GetAndSetI mem newv));
9457 ins_cost(2 * VOLATILE_REF_COST);
9458 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9459 ins_encode %{
9460 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9461 %}
9462 ins_pipe(pipe_serial);
9463 %}
9464
9465 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9466 match(Set prev (GetAndSetL mem newv));
9467 ins_cost(2 * VOLATILE_REF_COST);
9468 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9469 ins_encode %{
9470 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9471 %}
9472 ins_pipe(pipe_serial);
9473 %}
9474
9475 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9476 match(Set prev (GetAndSetN mem newv));
9477 ins_cost(2 * VOLATILE_REF_COST);
9478 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9479 ins_encode %{
9480 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9481 %}
9482 ins_pipe(pipe_serial);
9483 %}
9484
9485 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9486 predicate(n->as_LoadStore()->barrier_data() == 0);
9487 match(Set prev (GetAndSetP mem newv));
9488 ins_cost(2 * VOLATILE_REF_COST);
9489 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9490 ins_encode %{
9491 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9492 %}
9493 ins_pipe(pipe_serial);
9494 %}
9495
9496 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9497 predicate(needs_acquiring_load_exclusive(n));
9498 match(Set prev (GetAndSetI mem newv));
9499 ins_cost(VOLATILE_REF_COST);
9500 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9501 ins_encode %{
9502 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9503 %}
9504 ins_pipe(pipe_serial);
9505 %}
9506
9507 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9508 predicate(needs_acquiring_load_exclusive(n));
9509 match(Set prev (GetAndSetL mem newv));
9510 ins_cost(VOLATILE_REF_COST);
9511 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9512 ins_encode %{
9513 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9514 %}
9515 ins_pipe(pipe_serial);
9516 %}
9517
9518 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9519 predicate(needs_acquiring_load_exclusive(n));
9520 match(Set prev (GetAndSetN mem newv));
9521 ins_cost(VOLATILE_REF_COST);
9522 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9523 ins_encode %{
9524 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9525 %}
9526 ins_pipe(pipe_serial);
9527 %}
9528
9529 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9530 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9531 match(Set prev (GetAndSetP mem newv));
9532 ins_cost(VOLATILE_REF_COST);
9533 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9534 ins_encode %{
9535 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9536 %}
9537 ins_pipe(pipe_serial);
9538 %}
9539
9540
9541 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9542 match(Set newval (GetAndAddL mem incr));
9543 ins_cost(2 * VOLATILE_REF_COST + 1);
9544 format %{ "get_and_addL $newval, [$mem], $incr" %}
9545 ins_encode %{
9546 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9547 %}
9548 ins_pipe(pipe_serial);
9549 %}
9550
9551 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9552 predicate(n->as_LoadStore()->result_not_used());
9553 match(Set dummy (GetAndAddL mem incr));
9554 ins_cost(2 * VOLATILE_REF_COST);
9555 format %{ "get_and_addL [$mem], $incr" %}
9556 ins_encode %{
9557 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9558 %}
9559 ins_pipe(pipe_serial);
9560 %}
9561
9562 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9563 match(Set newval (GetAndAddL mem incr));
9564 ins_cost(2 * VOLATILE_REF_COST + 1);
9565 format %{ "get_and_addL $newval, [$mem], $incr" %}
9566 ins_encode %{
9567 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9568 %}
9569 ins_pipe(pipe_serial);
9570 %}
9571
9572 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9573 predicate(n->as_LoadStore()->result_not_used());
9574 match(Set dummy (GetAndAddL mem incr));
9575 ins_cost(2 * VOLATILE_REF_COST);
9576 format %{ "get_and_addL [$mem], $incr" %}
9577 ins_encode %{
9578 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9579 %}
9580 ins_pipe(pipe_serial);
9581 %}
9582
9583 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9584 match(Set newval (GetAndAddI mem incr));
9585 ins_cost(2 * VOLATILE_REF_COST + 1);
9586 format %{ "get_and_addI $newval, [$mem], $incr" %}
9587 ins_encode %{
9588 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9589 %}
9590 ins_pipe(pipe_serial);
9591 %}
9592
9593 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9594 predicate(n->as_LoadStore()->result_not_used());
9595 match(Set dummy (GetAndAddI mem incr));
9596 ins_cost(2 * VOLATILE_REF_COST);
9597 format %{ "get_and_addI [$mem], $incr" %}
9598 ins_encode %{
9599 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9600 %}
9601 ins_pipe(pipe_serial);
9602 %}
9603
9604 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9605 match(Set newval (GetAndAddI mem incr));
9606 ins_cost(2 * VOLATILE_REF_COST + 1);
9607 format %{ "get_and_addI $newval, [$mem], $incr" %}
9608 ins_encode %{
9609 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9610 %}
9611 ins_pipe(pipe_serial);
9612 %}
9613
9614 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9615 predicate(n->as_LoadStore()->result_not_used());
9616 match(Set dummy (GetAndAddI mem incr));
9617 ins_cost(2 * VOLATILE_REF_COST);
9618 format %{ "get_and_addI [$mem], $incr" %}
9619 ins_encode %{
9620 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9621 %}
9622 ins_pipe(pipe_serial);
9623 %}
9624
9625 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9626 predicate(needs_acquiring_load_exclusive(n));
9627 match(Set newval (GetAndAddL mem incr));
9628 ins_cost(VOLATILE_REF_COST + 1);
9629 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9630 ins_encode %{
9631 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9632 %}
9633 ins_pipe(pipe_serial);
9634 %}
9635
9636 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9637 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9638 match(Set dummy (GetAndAddL mem incr));
9639 ins_cost(VOLATILE_REF_COST);
9640 format %{ "get_and_addL_acq [$mem], $incr" %}
9641 ins_encode %{
9642 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9643 %}
9644 ins_pipe(pipe_serial);
9645 %}
9646
9647 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9648 predicate(needs_acquiring_load_exclusive(n));
9649 match(Set newval (GetAndAddL mem incr));
9650 ins_cost(VOLATILE_REF_COST + 1);
9651 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9652 ins_encode %{
9653 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9654 %}
9655 ins_pipe(pipe_serial);
9656 %}
9657
9658 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9659 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9660 match(Set dummy (GetAndAddL mem incr));
9661 ins_cost(VOLATILE_REF_COST);
9662 format %{ "get_and_addL_acq [$mem], $incr" %}
9663 ins_encode %{
9664 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9665 %}
9666 ins_pipe(pipe_serial);
9667 %}
9668
9669 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9670 predicate(needs_acquiring_load_exclusive(n));
9671 match(Set newval (GetAndAddI mem incr));
9672 ins_cost(VOLATILE_REF_COST + 1);
9673 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9674 ins_encode %{
9675 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9676 %}
9677 ins_pipe(pipe_serial);
9678 %}
9679
9680 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9681 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9682 match(Set dummy (GetAndAddI mem incr));
9683 ins_cost(VOLATILE_REF_COST);
9684 format %{ "get_and_addI_acq [$mem], $incr" %}
9685 ins_encode %{
9686 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9687 %}
9688 ins_pipe(pipe_serial);
9689 %}
9690
9691 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9692 predicate(needs_acquiring_load_exclusive(n));
9693 match(Set newval (GetAndAddI mem incr));
9694 ins_cost(VOLATILE_REF_COST + 1);
9695 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9696 ins_encode %{
9697 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9698 %}
9699 ins_pipe(pipe_serial);
9700 %}
9701
9702 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9703 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9704 match(Set dummy (GetAndAddI mem incr));
9705 ins_cost(VOLATILE_REF_COST);
9706 format %{ "get_and_addI_acq [$mem], $incr" %}
9707 ins_encode %{
9708 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9709 %}
9710 ins_pipe(pipe_serial);
9711 %}
9712
9713 // Manifest a CmpU result in an integer register.
9714 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9715 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9716 %{
9717 match(Set dst (CmpU3 src1 src2));
9718 effect(KILL flags);
9719
9720 ins_cost(INSN_COST * 3);
9721 format %{
9722 "cmpw $src1, $src2\n\t"
9723 "csetw $dst, ne\n\t"
9724 "cnegw $dst, lo\t# CmpU3(reg)"
9725 %}
9726 ins_encode %{
9727 __ cmpw($src1$$Register, $src2$$Register);
9728 __ csetw($dst$$Register, Assembler::NE);
9729 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9730 %}
9731
9732 ins_pipe(pipe_class_default);
9733 %}
9734
9735 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9736 %{
9737 match(Set dst (CmpU3 src1 src2));
9738 effect(KILL flags);
9739
9740 ins_cost(INSN_COST * 3);
9741 format %{
9742 "subsw zr, $src1, $src2\n\t"
9743 "csetw $dst, ne\n\t"
9744 "cnegw $dst, lo\t# CmpU3(imm)"
9745 %}
9746 ins_encode %{
9747 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9748 __ csetw($dst$$Register, Assembler::NE);
9749 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9750 %}
9751
9752 ins_pipe(pipe_class_default);
9753 %}
9754
9755 // Manifest a CmpUL result in an integer register.
9756 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9757 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9758 %{
9759 match(Set dst (CmpUL3 src1 src2));
9760 effect(KILL flags);
9761
9762 ins_cost(INSN_COST * 3);
9763 format %{
9764 "cmp $src1, $src2\n\t"
9765 "csetw $dst, ne\n\t"
9766 "cnegw $dst, lo\t# CmpUL3(reg)"
9767 %}
9768 ins_encode %{
9769 __ cmp($src1$$Register, $src2$$Register);
9770 __ csetw($dst$$Register, Assembler::NE);
9771 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9772 %}
9773
9774 ins_pipe(pipe_class_default);
9775 %}
9776
9777 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9778 %{
9779 match(Set dst (CmpUL3 src1 src2));
9780 effect(KILL flags);
9781
9782 ins_cost(INSN_COST * 3);
9783 format %{
9784 "subs zr, $src1, $src2\n\t"
9785 "csetw $dst, ne\n\t"
9786 "cnegw $dst, lo\t# CmpUL3(imm)"
9787 %}
9788 ins_encode %{
9789 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9790 __ csetw($dst$$Register, Assembler::NE);
9791 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9792 %}
9793
9794 ins_pipe(pipe_class_default);
9795 %}
9796
9797 // Manifest a CmpL result in an integer register.
9798 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9799 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9800 %{
9801 match(Set dst (CmpL3 src1 src2));
9802 effect(KILL flags);
9803
9804 ins_cost(INSN_COST * 3);
9805 format %{
9806 "cmp $src1, $src2\n\t"
9807 "csetw $dst, ne\n\t"
9808 "cnegw $dst, lt\t# CmpL3(reg)"
9809 %}
9810 ins_encode %{
9811 __ cmp($src1$$Register, $src2$$Register);
9812 __ csetw($dst$$Register, Assembler::NE);
9813 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9814 %}
9815
9816 ins_pipe(pipe_class_default);
9817 %}
9818
9819 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9820 %{
9821 match(Set dst (CmpL3 src1 src2));
9822 effect(KILL flags);
9823
9824 ins_cost(INSN_COST * 3);
9825 format %{
9826 "subs zr, $src1, $src2\n\t"
9827 "csetw $dst, ne\n\t"
9828 "cnegw $dst, lt\t# CmpL3(imm)"
9829 %}
9830 ins_encode %{
9831 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9832 __ csetw($dst$$Register, Assembler::NE);
9833 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9834 %}
9835
9836 ins_pipe(pipe_class_default);
9837 %}
9838
9839 // ============================================================================
9840 // Conditional Move Instructions
9841
9842 // n.b. we have identical rules for both a signed compare op (cmpOp)
9843 // and an unsigned compare op (cmpOpU). it would be nice if we could
9844 // define an op class which merged both inputs and use it to type the
9845 // argument to a single rule. unfortunatelyt his fails because the
9846 // opclass does not live up to the COND_INTER interface of its
9847 // component operands. When the generic code tries to negate the
9848 // operand it ends up running the generci Machoper::negate method
9849 // which throws a ShouldNotHappen. So, we have to provide two flavours
9850 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9851
9852 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9853 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9854
9855 ins_cost(INSN_COST * 2);
9856 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9857
9858 ins_encode %{
9859 __ cselw(as_Register($dst$$reg),
9860 as_Register($src2$$reg),
9861 as_Register($src1$$reg),
9862 (Assembler::Condition)$cmp$$cmpcode);
9863 %}
9864
9865 ins_pipe(icond_reg_reg);
9866 %}
9867
9868 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9869 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9870
9871 ins_cost(INSN_COST * 2);
9872 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9873
9874 ins_encode %{
9875 __ cselw(as_Register($dst$$reg),
9876 as_Register($src2$$reg),
9877 as_Register($src1$$reg),
9878 (Assembler::Condition)$cmp$$cmpcode);
9879 %}
9880
9881 ins_pipe(icond_reg_reg);
9882 %}
9883
9884 // special cases where one arg is zero
9885
9886 // n.b. this is selected in preference to the rule above because it
9887 // avoids loading constant 0 into a source register
9888
9889 // TODO
9890 // we ought only to be able to cull one of these variants as the ideal
9891 // transforms ought always to order the zero consistently (to left/right?)
9892
9893 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9894 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9895
9896 ins_cost(INSN_COST * 2);
9897 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9898
9899 ins_encode %{
9900 __ cselw(as_Register($dst$$reg),
9901 as_Register($src$$reg),
9902 zr,
9903 (Assembler::Condition)$cmp$$cmpcode);
9904 %}
9905
9906 ins_pipe(icond_reg);
9907 %}
9908
9909 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9910 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9911
9912 ins_cost(INSN_COST * 2);
9913 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9914
9915 ins_encode %{
9916 __ cselw(as_Register($dst$$reg),
9917 as_Register($src$$reg),
9918 zr,
9919 (Assembler::Condition)$cmp$$cmpcode);
9920 %}
9921
9922 ins_pipe(icond_reg);
9923 %}
9924
9925 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9926 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9927
9928 ins_cost(INSN_COST * 2);
9929 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9930
9931 ins_encode %{
9932 __ cselw(as_Register($dst$$reg),
9933 zr,
9934 as_Register($src$$reg),
9935 (Assembler::Condition)$cmp$$cmpcode);
9936 %}
9937
9938 ins_pipe(icond_reg);
9939 %}
9940
9941 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9942 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9943
9944 ins_cost(INSN_COST * 2);
9945 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9946
9947 ins_encode %{
9948 __ cselw(as_Register($dst$$reg),
9949 zr,
9950 as_Register($src$$reg),
9951 (Assembler::Condition)$cmp$$cmpcode);
9952 %}
9953
9954 ins_pipe(icond_reg);
9955 %}
9956
9957 // special case for creating a boolean 0 or 1
9958
9959 // n.b. this is selected in preference to the rule above because it
9960 // avoids loading constants 0 and 1 into a source register
9961
9962 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9963 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9964
9965 ins_cost(INSN_COST * 2);
9966 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9967
9968 ins_encode %{
9969 // equivalently
9970 // cset(as_Register($dst$$reg),
9971 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9972 __ csincw(as_Register($dst$$reg),
9973 zr,
9974 zr,
9975 (Assembler::Condition)$cmp$$cmpcode);
9976 %}
9977
9978 ins_pipe(icond_none);
9979 %}
9980
9981 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9982 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9983
9984 ins_cost(INSN_COST * 2);
9985 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9986
9987 ins_encode %{
9988 // equivalently
9989 // cset(as_Register($dst$$reg),
9990 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9991 __ csincw(as_Register($dst$$reg),
9992 zr,
9993 zr,
9994 (Assembler::Condition)$cmp$$cmpcode);
9995 %}
9996
9997 ins_pipe(icond_none);
9998 %}
9999
10000 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10001 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10002
10003 ins_cost(INSN_COST * 2);
10004 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
10005
10006 ins_encode %{
10007 __ csel(as_Register($dst$$reg),
10008 as_Register($src2$$reg),
10009 as_Register($src1$$reg),
10010 (Assembler::Condition)$cmp$$cmpcode);
10011 %}
10012
10013 ins_pipe(icond_reg_reg);
10014 %}
10015
10016 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10017 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10018
10019 ins_cost(INSN_COST * 2);
10020 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
10021
10022 ins_encode %{
10023 __ csel(as_Register($dst$$reg),
10024 as_Register($src2$$reg),
10025 as_Register($src1$$reg),
10026 (Assembler::Condition)$cmp$$cmpcode);
10027 %}
10028
10029 ins_pipe(icond_reg_reg);
10030 %}
10031
10032 // special cases where one arg is zero
10033
10034 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10035 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10036
10037 ins_cost(INSN_COST * 2);
10038 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
10039
10040 ins_encode %{
10041 __ csel(as_Register($dst$$reg),
10042 zr,
10043 as_Register($src$$reg),
10044 (Assembler::Condition)$cmp$$cmpcode);
10045 %}
10046
10047 ins_pipe(icond_reg);
10048 %}
10049
10050 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10051 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10052
10053 ins_cost(INSN_COST * 2);
10054 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
10055
10056 ins_encode %{
10057 __ csel(as_Register($dst$$reg),
10058 zr,
10059 as_Register($src$$reg),
10060 (Assembler::Condition)$cmp$$cmpcode);
10061 %}
10062
10063 ins_pipe(icond_reg);
10064 %}
10065
10066 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10067 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10068
10069 ins_cost(INSN_COST * 2);
10070 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
10071
10072 ins_encode %{
10073 __ csel(as_Register($dst$$reg),
10074 as_Register($src$$reg),
10075 zr,
10076 (Assembler::Condition)$cmp$$cmpcode);
10077 %}
10078
10079 ins_pipe(icond_reg);
10080 %}
10081
10082 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10083 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10084
10085 ins_cost(INSN_COST * 2);
10086 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
10087
10088 ins_encode %{
10089 __ csel(as_Register($dst$$reg),
10090 as_Register($src$$reg),
10091 zr,
10092 (Assembler::Condition)$cmp$$cmpcode);
10093 %}
10094
10095 ins_pipe(icond_reg);
10096 %}
10097
10098 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10099 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10100
10101 ins_cost(INSN_COST * 2);
10102 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
10103
10104 ins_encode %{
10105 __ csel(as_Register($dst$$reg),
10106 as_Register($src2$$reg),
10107 as_Register($src1$$reg),
10108 (Assembler::Condition)$cmp$$cmpcode);
10109 %}
10110
10111 ins_pipe(icond_reg_reg);
10112 %}
10113
10114 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10115 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10116
10117 ins_cost(INSN_COST * 2);
10118 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
10119
10120 ins_encode %{
10121 __ csel(as_Register($dst$$reg),
10122 as_Register($src2$$reg),
10123 as_Register($src1$$reg),
10124 (Assembler::Condition)$cmp$$cmpcode);
10125 %}
10126
10127 ins_pipe(icond_reg_reg);
10128 %}
10129
10130 // special cases where one arg is zero
10131
10132 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10133 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10134
10135 ins_cost(INSN_COST * 2);
10136 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
10137
10138 ins_encode %{
10139 __ csel(as_Register($dst$$reg),
10140 zr,
10141 as_Register($src$$reg),
10142 (Assembler::Condition)$cmp$$cmpcode);
10143 %}
10144
10145 ins_pipe(icond_reg);
10146 %}
10147
10148 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10149 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10150
10151 ins_cost(INSN_COST * 2);
10152 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
10153
10154 ins_encode %{
10155 __ csel(as_Register($dst$$reg),
10156 zr,
10157 as_Register($src$$reg),
10158 (Assembler::Condition)$cmp$$cmpcode);
10159 %}
10160
10161 ins_pipe(icond_reg);
10162 %}
10163
10164 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10165 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10166
10167 ins_cost(INSN_COST * 2);
10168 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
10169
10170 ins_encode %{
10171 __ csel(as_Register($dst$$reg),
10172 as_Register($src$$reg),
10173 zr,
10174 (Assembler::Condition)$cmp$$cmpcode);
10175 %}
10176
10177 ins_pipe(icond_reg);
10178 %}
10179
10180 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10181 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10182
10183 ins_cost(INSN_COST * 2);
10184 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
10185
10186 ins_encode %{
10187 __ csel(as_Register($dst$$reg),
10188 as_Register($src$$reg),
10189 zr,
10190 (Assembler::Condition)$cmp$$cmpcode);
10191 %}
10192
10193 ins_pipe(icond_reg);
10194 %}
10195
10196 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10197 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10198
10199 ins_cost(INSN_COST * 2);
10200 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10201
10202 ins_encode %{
10203 __ cselw(as_Register($dst$$reg),
10204 as_Register($src2$$reg),
10205 as_Register($src1$$reg),
10206 (Assembler::Condition)$cmp$$cmpcode);
10207 %}
10208
10209 ins_pipe(icond_reg_reg);
10210 %}
10211
10212 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10213 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10214
10215 ins_cost(INSN_COST * 2);
10216 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10217
10218 ins_encode %{
10219 __ cselw(as_Register($dst$$reg),
10220 as_Register($src2$$reg),
10221 as_Register($src1$$reg),
10222 (Assembler::Condition)$cmp$$cmpcode);
10223 %}
10224
10225 ins_pipe(icond_reg_reg);
10226 %}
10227
10228 // special cases where one arg is zero
10229
10230 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10231 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10232
10233 ins_cost(INSN_COST * 2);
10234 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
10235
10236 ins_encode %{
10237 __ cselw(as_Register($dst$$reg),
10238 zr,
10239 as_Register($src$$reg),
10240 (Assembler::Condition)$cmp$$cmpcode);
10241 %}
10242
10243 ins_pipe(icond_reg);
10244 %}
10245
10246 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10247 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10248
10249 ins_cost(INSN_COST * 2);
10250 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
10251
10252 ins_encode %{
10253 __ cselw(as_Register($dst$$reg),
10254 zr,
10255 as_Register($src$$reg),
10256 (Assembler::Condition)$cmp$$cmpcode);
10257 %}
10258
10259 ins_pipe(icond_reg);
10260 %}
10261
10262 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10263 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10264
10265 ins_cost(INSN_COST * 2);
10266 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
10267
10268 ins_encode %{
10269 __ cselw(as_Register($dst$$reg),
10270 as_Register($src$$reg),
10271 zr,
10272 (Assembler::Condition)$cmp$$cmpcode);
10273 %}
10274
10275 ins_pipe(icond_reg);
10276 %}
10277
10278 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10279 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10280
10281 ins_cost(INSN_COST * 2);
10282 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
10283
10284 ins_encode %{
10285 __ cselw(as_Register($dst$$reg),
10286 as_Register($src$$reg),
10287 zr,
10288 (Assembler::Condition)$cmp$$cmpcode);
10289 %}
10290
10291 ins_pipe(icond_reg);
10292 %}
10293
10294 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
10295 %{
10296 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10297
10298 ins_cost(INSN_COST * 3);
10299
10300 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10301 ins_encode %{
10302 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10303 __ fcsels(as_FloatRegister($dst$$reg),
10304 as_FloatRegister($src2$$reg),
10305 as_FloatRegister($src1$$reg),
10306 cond);
10307 %}
10308
10309 ins_pipe(fp_cond_reg_reg_s);
10310 %}
10311
10312 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
10313 %{
10314 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10315
10316 ins_cost(INSN_COST * 3);
10317
10318 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10319 ins_encode %{
10320 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10321 __ fcsels(as_FloatRegister($dst$$reg),
10322 as_FloatRegister($src2$$reg),
10323 as_FloatRegister($src1$$reg),
10324 cond);
10325 %}
10326
10327 ins_pipe(fp_cond_reg_reg_s);
10328 %}
10329
10330 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10331 %{
10332 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10333
10334 ins_cost(INSN_COST * 3);
10335
10336 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10337 ins_encode %{
10338 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10339 __ fcseld(as_FloatRegister($dst$$reg),
10340 as_FloatRegister($src2$$reg),
10341 as_FloatRegister($src1$$reg),
10342 cond);
10343 %}
10344
10345 ins_pipe(fp_cond_reg_reg_d);
10346 %}
10347
10348 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10349 %{
10350 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10351
10352 ins_cost(INSN_COST * 3);
10353
10354 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10355 ins_encode %{
10356 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10357 __ fcseld(as_FloatRegister($dst$$reg),
10358 as_FloatRegister($src2$$reg),
10359 as_FloatRegister($src1$$reg),
10360 cond);
10361 %}
10362
10363 ins_pipe(fp_cond_reg_reg_d);
10364 %}
10365
10366 // ============================================================================
10367 // Arithmetic Instructions
10368 //
10369
10370 // Integer Addition
10371
10372 // TODO
10373 // these currently employ operations which do not set CR and hence are
10374 // not flagged as killing CR but we would like to isolate the cases
10375 // where we want to set flags from those where we don't. need to work
10376 // out how to do that.
10377
10378 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10379 match(Set dst (AddI src1 src2));
10380
10381 ins_cost(INSN_COST);
10382 format %{ "addw $dst, $src1, $src2" %}
10383
10384 ins_encode %{
10385 __ addw(as_Register($dst$$reg),
10386 as_Register($src1$$reg),
10387 as_Register($src2$$reg));
10388 %}
10389
10390 ins_pipe(ialu_reg_reg);
10391 %}
10392
10393 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10394 match(Set dst (AddI src1 src2));
10395
10396 ins_cost(INSN_COST);
10397 format %{ "addw $dst, $src1, $src2" %}
10398
10399 // use opcode to indicate that this is an add not a sub
10400 opcode(0x0);
10401
10402 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10403
10404 ins_pipe(ialu_reg_imm);
10405 %}
10406
10407 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10408 match(Set dst (AddI (ConvL2I src1) src2));
10409
10410 ins_cost(INSN_COST);
10411 format %{ "addw $dst, $src1, $src2" %}
10412
10413 // use opcode to indicate that this is an add not a sub
10414 opcode(0x0);
10415
10416 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10417
10418 ins_pipe(ialu_reg_imm);
10419 %}
10420
10421 // Pointer Addition
10422 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10423 match(Set dst (AddP src1 src2));
10424
10425 ins_cost(INSN_COST);
10426 format %{ "add $dst, $src1, $src2\t# ptr" %}
10427
10428 ins_encode %{
10429 __ add(as_Register($dst$$reg),
10430 as_Register($src1$$reg),
10431 as_Register($src2$$reg));
10432 %}
10433
10434 ins_pipe(ialu_reg_reg);
10435 %}
10436
10437 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10438 match(Set dst (AddP src1 (ConvI2L src2)));
10439
10440 ins_cost(1.9 * INSN_COST);
10441 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10442
10443 ins_encode %{
10444 __ add(as_Register($dst$$reg),
10445 as_Register($src1$$reg),
10446 as_Register($src2$$reg), ext::sxtw);
10447 %}
10448
10449 ins_pipe(ialu_reg_reg);
10450 %}
10451
10452 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10453 match(Set dst (AddP src1 (LShiftL src2 scale)));
10454
10455 ins_cost(1.9 * INSN_COST);
10456 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10457
10458 ins_encode %{
10459 __ lea(as_Register($dst$$reg),
10460 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10461 Address::lsl($scale$$constant)));
10462 %}
10463
10464 ins_pipe(ialu_reg_reg_shift);
10465 %}
10466
10467 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10468 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10469
10470 ins_cost(1.9 * INSN_COST);
10471 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10472
10473 ins_encode %{
10474 __ lea(as_Register($dst$$reg),
10475 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10476 Address::sxtw($scale$$constant)));
10477 %}
10478
10479 ins_pipe(ialu_reg_reg_shift);
10480 %}
10481
10482 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10483 match(Set dst (LShiftL (ConvI2L src) scale));
10484
10485 ins_cost(INSN_COST);
10486 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10487
10488 ins_encode %{
10489 __ sbfiz(as_Register($dst$$reg),
10490 as_Register($src$$reg),
10491 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10492 %}
10493
10494 ins_pipe(ialu_reg_shift);
10495 %}
10496
10497 // Pointer Immediate Addition
10498 // n.b. this needs to be more expensive than using an indirect memory
10499 // operand
10500 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10501 match(Set dst (AddP src1 src2));
10502
10503 ins_cost(INSN_COST);
10504 format %{ "add $dst, $src1, $src2\t# ptr" %}
10505
10506 // use opcode to indicate that this is an add not a sub
10507 opcode(0x0);
10508
10509 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10510
10511 ins_pipe(ialu_reg_imm);
10512 %}
10513
10514 // Long Addition
10515 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10516
10517 match(Set dst (AddL src1 src2));
10518
10519 ins_cost(INSN_COST);
10520 format %{ "add $dst, $src1, $src2" %}
10521
10522 ins_encode %{
10523 __ add(as_Register($dst$$reg),
10524 as_Register($src1$$reg),
10525 as_Register($src2$$reg));
10526 %}
10527
10528 ins_pipe(ialu_reg_reg);
10529 %}
10530
10531 // No constant pool entries requiredLong Immediate Addition.
10532 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10533 match(Set dst (AddL src1 src2));
10534
10535 ins_cost(INSN_COST);
10536 format %{ "add $dst, $src1, $src2" %}
10537
10538 // use opcode to indicate that this is an add not a sub
10539 opcode(0x0);
10540
10541 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10542
10543 ins_pipe(ialu_reg_imm);
10544 %}
10545
10546 // Integer Subtraction
10547 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10548 match(Set dst (SubI src1 src2));
10549
10550 ins_cost(INSN_COST);
10551 format %{ "subw $dst, $src1, $src2" %}
10552
10553 ins_encode %{
10554 __ subw(as_Register($dst$$reg),
10555 as_Register($src1$$reg),
10556 as_Register($src2$$reg));
10557 %}
10558
10559 ins_pipe(ialu_reg_reg);
10560 %}
10561
10562 // Immediate Subtraction
10563 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10564 match(Set dst (SubI src1 src2));
10565
10566 ins_cost(INSN_COST);
10567 format %{ "subw $dst, $src1, $src2" %}
10568
10569 // use opcode to indicate that this is a sub not an add
10570 opcode(0x1);
10571
10572 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10573
10574 ins_pipe(ialu_reg_imm);
10575 %}
10576
10577 // Long Subtraction
10578 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10579
10580 match(Set dst (SubL src1 src2));
10581
10582 ins_cost(INSN_COST);
10583 format %{ "sub $dst, $src1, $src2" %}
10584
10585 ins_encode %{
10586 __ sub(as_Register($dst$$reg),
10587 as_Register($src1$$reg),
10588 as_Register($src2$$reg));
10589 %}
10590
10591 ins_pipe(ialu_reg_reg);
10592 %}
10593
10594 // No constant pool entries requiredLong Immediate Subtraction.
10595 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10596 match(Set dst (SubL src1 src2));
10597
10598 ins_cost(INSN_COST);
10599 format %{ "sub$dst, $src1, $src2" %}
10600
10601 // use opcode to indicate that this is a sub not an add
10602 opcode(0x1);
10603
10604 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10605
10606 ins_pipe(ialu_reg_imm);
10607 %}
10608
10609 // Integer Negation (special case for sub)
10610
10611 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10612 match(Set dst (SubI zero src));
10613
10614 ins_cost(INSN_COST);
10615 format %{ "negw $dst, $src\t# int" %}
10616
10617 ins_encode %{
10618 __ negw(as_Register($dst$$reg),
10619 as_Register($src$$reg));
10620 %}
10621
10622 ins_pipe(ialu_reg);
10623 %}
10624
10625 // Long Negation
10626
10627 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10628 match(Set dst (SubL zero src));
10629
10630 ins_cost(INSN_COST);
10631 format %{ "neg $dst, $src\t# long" %}
10632
10633 ins_encode %{
10634 __ neg(as_Register($dst$$reg),
10635 as_Register($src$$reg));
10636 %}
10637
10638 ins_pipe(ialu_reg);
10639 %}
10640
10641 // Integer Multiply
10642
10643 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10644 match(Set dst (MulI src1 src2));
10645
10646 ins_cost(INSN_COST * 3);
10647 format %{ "mulw $dst, $src1, $src2" %}
10648
10649 ins_encode %{
10650 __ mulw(as_Register($dst$$reg),
10651 as_Register($src1$$reg),
10652 as_Register($src2$$reg));
10653 %}
10654
10655 ins_pipe(imul_reg_reg);
10656 %}
10657
10658 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10659 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10660
10661 ins_cost(INSN_COST * 3);
10662 format %{ "smull $dst, $src1, $src2" %}
10663
10664 ins_encode %{
10665 __ smull(as_Register($dst$$reg),
10666 as_Register($src1$$reg),
10667 as_Register($src2$$reg));
10668 %}
10669
10670 ins_pipe(imul_reg_reg);
10671 %}
10672
10673 // Long Multiply
10674
10675 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10676 match(Set dst (MulL src1 src2));
10677
10678 ins_cost(INSN_COST * 5);
10679 format %{ "mul $dst, $src1, $src2" %}
10680
10681 ins_encode %{
10682 __ mul(as_Register($dst$$reg),
10683 as_Register($src1$$reg),
10684 as_Register($src2$$reg));
10685 %}
10686
10687 ins_pipe(lmul_reg_reg);
10688 %}
10689
10690 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10691 %{
10692 match(Set dst (MulHiL src1 src2));
10693
10694 ins_cost(INSN_COST * 7);
10695 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10696
10697 ins_encode %{
10698 __ smulh(as_Register($dst$$reg),
10699 as_Register($src1$$reg),
10700 as_Register($src2$$reg));
10701 %}
10702
10703 ins_pipe(lmul_reg_reg);
10704 %}
10705
10706 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10707 %{
10708 match(Set dst (UMulHiL src1 src2));
10709
10710 ins_cost(INSN_COST * 7);
10711 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10712
10713 ins_encode %{
10714 __ umulh(as_Register($dst$$reg),
10715 as_Register($src1$$reg),
10716 as_Register($src2$$reg));
10717 %}
10718
10719 ins_pipe(lmul_reg_reg);
10720 %}
10721
10722 // Combined Integer Multiply & Add/Sub
10723
10724 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10725 match(Set dst (AddI src3 (MulI src1 src2)));
10726
10727 ins_cost(INSN_COST * 3);
10728 format %{ "madd $dst, $src1, $src2, $src3" %}
10729
10730 ins_encode %{
10731 __ maddw(as_Register($dst$$reg),
10732 as_Register($src1$$reg),
10733 as_Register($src2$$reg),
10734 as_Register($src3$$reg));
10735 %}
10736
10737 ins_pipe(imac_reg_reg);
10738 %}
10739
10740 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10741 match(Set dst (SubI src3 (MulI src1 src2)));
10742
10743 ins_cost(INSN_COST * 3);
10744 format %{ "msub $dst, $src1, $src2, $src3" %}
10745
10746 ins_encode %{
10747 __ msubw(as_Register($dst$$reg),
10748 as_Register($src1$$reg),
10749 as_Register($src2$$reg),
10750 as_Register($src3$$reg));
10751 %}
10752
10753 ins_pipe(imac_reg_reg);
10754 %}
10755
10756 // Combined Integer Multiply & Neg
10757
10758 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10759 match(Set dst (MulI (SubI zero src1) src2));
10760
10761 ins_cost(INSN_COST * 3);
10762 format %{ "mneg $dst, $src1, $src2" %}
10763
10764 ins_encode %{
10765 __ mnegw(as_Register($dst$$reg),
10766 as_Register($src1$$reg),
10767 as_Register($src2$$reg));
10768 %}
10769
10770 ins_pipe(imac_reg_reg);
10771 %}
10772
10773 // Combined Long Multiply & Add/Sub
10774
10775 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10776 match(Set dst (AddL src3 (MulL src1 src2)));
10777
10778 ins_cost(INSN_COST * 5);
10779 format %{ "madd $dst, $src1, $src2, $src3" %}
10780
10781 ins_encode %{
10782 __ madd(as_Register($dst$$reg),
10783 as_Register($src1$$reg),
10784 as_Register($src2$$reg),
10785 as_Register($src3$$reg));
10786 %}
10787
10788 ins_pipe(lmac_reg_reg);
10789 %}
10790
10791 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10792 match(Set dst (SubL src3 (MulL src1 src2)));
10793
10794 ins_cost(INSN_COST * 5);
10795 format %{ "msub $dst, $src1, $src2, $src3" %}
10796
10797 ins_encode %{
10798 __ msub(as_Register($dst$$reg),
10799 as_Register($src1$$reg),
10800 as_Register($src2$$reg),
10801 as_Register($src3$$reg));
10802 %}
10803
10804 ins_pipe(lmac_reg_reg);
10805 %}
10806
10807 // Combined Long Multiply & Neg
10808
10809 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10810 match(Set dst (MulL (SubL zero src1) src2));
10811
10812 ins_cost(INSN_COST * 5);
10813 format %{ "mneg $dst, $src1, $src2" %}
10814
10815 ins_encode %{
10816 __ mneg(as_Register($dst$$reg),
10817 as_Register($src1$$reg),
10818 as_Register($src2$$reg));
10819 %}
10820
10821 ins_pipe(lmac_reg_reg);
10822 %}
10823
10824 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10825
10826 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10827 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10828
10829 ins_cost(INSN_COST * 3);
10830 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10831
10832 ins_encode %{
10833 __ smaddl(as_Register($dst$$reg),
10834 as_Register($src1$$reg),
10835 as_Register($src2$$reg),
10836 as_Register($src3$$reg));
10837 %}
10838
10839 ins_pipe(imac_reg_reg);
10840 %}
10841
10842 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10843 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10844
10845 ins_cost(INSN_COST * 3);
10846 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10847
10848 ins_encode %{
10849 __ smsubl(as_Register($dst$$reg),
10850 as_Register($src1$$reg),
10851 as_Register($src2$$reg),
10852 as_Register($src3$$reg));
10853 %}
10854
10855 ins_pipe(imac_reg_reg);
10856 %}
10857
10858 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10859 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10860
10861 ins_cost(INSN_COST * 3);
10862 format %{ "smnegl $dst, $src1, $src2" %}
10863
10864 ins_encode %{
10865 __ smnegl(as_Register($dst$$reg),
10866 as_Register($src1$$reg),
10867 as_Register($src2$$reg));
10868 %}
10869
10870 ins_pipe(imac_reg_reg);
10871 %}
10872
10873 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10874
10875 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10876 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10877
10878 ins_cost(INSN_COST * 5);
10879 format %{ "mulw rscratch1, $src1, $src2\n\t"
10880 "maddw $dst, $src3, $src4, rscratch1" %}
10881
10882 ins_encode %{
10883 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10884 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10885
10886 ins_pipe(imac_reg_reg);
10887 %}
10888
10889 // Integer Divide
10890
10891 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10892 match(Set dst (DivI src1 src2));
10893
10894 ins_cost(INSN_COST * 19);
10895 format %{ "sdivw $dst, $src1, $src2" %}
10896
10897 ins_encode(aarch64_enc_divw(dst, src1, src2));
10898 ins_pipe(idiv_reg_reg);
10899 %}
10900
10901 // Long Divide
10902
10903 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10904 match(Set dst (DivL src1 src2));
10905
10906 ins_cost(INSN_COST * 35);
10907 format %{ "sdiv $dst, $src1, $src2" %}
10908
10909 ins_encode(aarch64_enc_div(dst, src1, src2));
10910 ins_pipe(ldiv_reg_reg);
10911 %}
10912
10913 // Integer Remainder
10914
10915 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10916 match(Set dst (ModI src1 src2));
10917
10918 ins_cost(INSN_COST * 22);
10919 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10920 "msubw $dst, rscratch1, $src2, $src1" %}
10921
10922 ins_encode(aarch64_enc_modw(dst, src1, src2));
10923 ins_pipe(idiv_reg_reg);
10924 %}
10925
10926 // Long Remainder
10927
10928 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10929 match(Set dst (ModL src1 src2));
10930
10931 ins_cost(INSN_COST * 38);
10932 format %{ "sdiv rscratch1, $src1, $src2\n"
10933 "msub $dst, rscratch1, $src2, $src1" %}
10934
10935 ins_encode(aarch64_enc_mod(dst, src1, src2));
10936 ins_pipe(ldiv_reg_reg);
10937 %}
10938
10939 // Unsigned Integer Divide
10940
10941 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10942 match(Set dst (UDivI src1 src2));
10943
10944 ins_cost(INSN_COST * 19);
10945 format %{ "udivw $dst, $src1, $src2" %}
10946
10947 ins_encode %{
10948 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10949 %}
10950
10951 ins_pipe(idiv_reg_reg);
10952 %}
10953
10954 // Unsigned Long Divide
10955
10956 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10957 match(Set dst (UDivL src1 src2));
10958
10959 ins_cost(INSN_COST * 35);
10960 format %{ "udiv $dst, $src1, $src2" %}
10961
10962 ins_encode %{
10963 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10964 %}
10965
10966 ins_pipe(ldiv_reg_reg);
10967 %}
10968
10969 // Unsigned Integer Remainder
10970
10971 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10972 match(Set dst (UModI src1 src2));
10973
10974 ins_cost(INSN_COST * 22);
10975 format %{ "udivw rscratch1, $src1, $src2\n\t"
10976 "msubw $dst, rscratch1, $src2, $src1" %}
10977
10978 ins_encode %{
10979 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10980 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10981 %}
10982
10983 ins_pipe(idiv_reg_reg);
10984 %}
10985
10986 // Unsigned Long Remainder
10987
10988 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10989 match(Set dst (UModL src1 src2));
10990
10991 ins_cost(INSN_COST * 38);
10992 format %{ "udiv rscratch1, $src1, $src2\n"
10993 "msub $dst, rscratch1, $src2, $src1" %}
10994
10995 ins_encode %{
10996 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10997 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10998 %}
10999
11000 ins_pipe(ldiv_reg_reg);
11001 %}
11002
11003 // Integer Shifts
11004
11005 // Shift Left Register
11006 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11007 match(Set dst (LShiftI src1 src2));
11008
11009 ins_cost(INSN_COST * 2);
11010 format %{ "lslvw $dst, $src1, $src2" %}
11011
11012 ins_encode %{
11013 __ lslvw(as_Register($dst$$reg),
11014 as_Register($src1$$reg),
11015 as_Register($src2$$reg));
11016 %}
11017
11018 ins_pipe(ialu_reg_reg_vshift);
11019 %}
11020
11021 // Shift Left Immediate
11022 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11023 match(Set dst (LShiftI src1 src2));
11024
11025 ins_cost(INSN_COST);
11026 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11027
11028 ins_encode %{
11029 __ lslw(as_Register($dst$$reg),
11030 as_Register($src1$$reg),
11031 $src2$$constant & 0x1f);
11032 %}
11033
11034 ins_pipe(ialu_reg_shift);
11035 %}
11036
11037 // Shift Right Logical Register
11038 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11039 match(Set dst (URShiftI src1 src2));
11040
11041 ins_cost(INSN_COST * 2);
11042 format %{ "lsrvw $dst, $src1, $src2" %}
11043
11044 ins_encode %{
11045 __ lsrvw(as_Register($dst$$reg),
11046 as_Register($src1$$reg),
11047 as_Register($src2$$reg));
11048 %}
11049
11050 ins_pipe(ialu_reg_reg_vshift);
11051 %}
11052
11053 // Shift Right Logical Immediate
11054 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11055 match(Set dst (URShiftI src1 src2));
11056
11057 ins_cost(INSN_COST);
11058 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11059
11060 ins_encode %{
11061 __ lsrw(as_Register($dst$$reg),
11062 as_Register($src1$$reg),
11063 $src2$$constant & 0x1f);
11064 %}
11065
11066 ins_pipe(ialu_reg_shift);
11067 %}
11068
11069 // Shift Right Arithmetic Register
11070 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11071 match(Set dst (RShiftI src1 src2));
11072
11073 ins_cost(INSN_COST * 2);
11074 format %{ "asrvw $dst, $src1, $src2" %}
11075
11076 ins_encode %{
11077 __ asrvw(as_Register($dst$$reg),
11078 as_Register($src1$$reg),
11079 as_Register($src2$$reg));
11080 %}
11081
11082 ins_pipe(ialu_reg_reg_vshift);
11083 %}
11084
11085 // Shift Right Arithmetic Immediate
11086 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11087 match(Set dst (RShiftI src1 src2));
11088
11089 ins_cost(INSN_COST);
11090 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11091
11092 ins_encode %{
11093 __ asrw(as_Register($dst$$reg),
11094 as_Register($src1$$reg),
11095 $src2$$constant & 0x1f);
11096 %}
11097
11098 ins_pipe(ialu_reg_shift);
11099 %}
11100
11101 // Combined Int Mask and Right Shift (using UBFM)
11102 // TODO
11103
11104 // Long Shifts
11105
11106 // Shift Left Register
11107 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11108 match(Set dst (LShiftL src1 src2));
11109
11110 ins_cost(INSN_COST * 2);
11111 format %{ "lslv $dst, $src1, $src2" %}
11112
11113 ins_encode %{
11114 __ lslv(as_Register($dst$$reg),
11115 as_Register($src1$$reg),
11116 as_Register($src2$$reg));
11117 %}
11118
11119 ins_pipe(ialu_reg_reg_vshift);
11120 %}
11121
11122 // Shift Left Immediate
11123 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11124 match(Set dst (LShiftL src1 src2));
11125
11126 ins_cost(INSN_COST);
11127 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11128
11129 ins_encode %{
11130 __ lsl(as_Register($dst$$reg),
11131 as_Register($src1$$reg),
11132 $src2$$constant & 0x3f);
11133 %}
11134
11135 ins_pipe(ialu_reg_shift);
11136 %}
11137
11138 // Shift Right Logical Register
11139 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11140 match(Set dst (URShiftL src1 src2));
11141
11142 ins_cost(INSN_COST * 2);
11143 format %{ "lsrv $dst, $src1, $src2" %}
11144
11145 ins_encode %{
11146 __ lsrv(as_Register($dst$$reg),
11147 as_Register($src1$$reg),
11148 as_Register($src2$$reg));
11149 %}
11150
11151 ins_pipe(ialu_reg_reg_vshift);
11152 %}
11153
11154 // Shift Right Logical Immediate
11155 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11156 match(Set dst (URShiftL src1 src2));
11157
11158 ins_cost(INSN_COST);
11159 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11160
11161 ins_encode %{
11162 __ lsr(as_Register($dst$$reg),
11163 as_Register($src1$$reg),
11164 $src2$$constant & 0x3f);
11165 %}
11166
11167 ins_pipe(ialu_reg_shift);
11168 %}
11169
11170 // A special-case pattern for card table stores.
11171 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11172 match(Set dst (URShiftL (CastP2X src1) src2));
11173
11174 ins_cost(INSN_COST);
11175 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11176
11177 ins_encode %{
11178 __ lsr(as_Register($dst$$reg),
11179 as_Register($src1$$reg),
11180 $src2$$constant & 0x3f);
11181 %}
11182
11183 ins_pipe(ialu_reg_shift);
11184 %}
11185
11186 // Shift Right Arithmetic Register
11187 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11188 match(Set dst (RShiftL src1 src2));
11189
11190 ins_cost(INSN_COST * 2);
11191 format %{ "asrv $dst, $src1, $src2" %}
11192
11193 ins_encode %{
11194 __ asrv(as_Register($dst$$reg),
11195 as_Register($src1$$reg),
11196 as_Register($src2$$reg));
11197 %}
11198
11199 ins_pipe(ialu_reg_reg_vshift);
11200 %}
11201
11202 // Shift Right Arithmetic Immediate
11203 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11204 match(Set dst (RShiftL src1 src2));
11205
11206 ins_cost(INSN_COST);
11207 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11208
11209 ins_encode %{
11210 __ asr(as_Register($dst$$reg),
11211 as_Register($src1$$reg),
11212 $src2$$constant & 0x3f);
11213 %}
11214
11215 ins_pipe(ialu_reg_shift);
11216 %}
11217
11218 // BEGIN This section of the file is automatically generated. Do not edit --------------
11219 // This section is generated from aarch64_ad.m4
11220
11221
11222 // This pattern is automatically generated from aarch64_ad.m4
11223 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11224 instruct regL_not_reg(iRegLNoSp dst,
11225 iRegL src1, immL_M1 m1,
11226 rFlagsReg cr) %{
11227 match(Set dst (XorL src1 m1));
11228 ins_cost(INSN_COST);
11229 format %{ "eon $dst, $src1, zr" %}
11230
11231 ins_encode %{
11232 __ eon(as_Register($dst$$reg),
11233 as_Register($src1$$reg),
11234 zr,
11235 Assembler::LSL, 0);
11236 %}
11237
11238 ins_pipe(ialu_reg);
11239 %}
11240
11241 // This pattern is automatically generated from aarch64_ad.m4
11242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11243 instruct regI_not_reg(iRegINoSp dst,
11244 iRegIorL2I src1, immI_M1 m1,
11245 rFlagsReg cr) %{
11246 match(Set dst (XorI src1 m1));
11247 ins_cost(INSN_COST);
11248 format %{ "eonw $dst, $src1, zr" %}
11249
11250 ins_encode %{
11251 __ eonw(as_Register($dst$$reg),
11252 as_Register($src1$$reg),
11253 zr,
11254 Assembler::LSL, 0);
11255 %}
11256
11257 ins_pipe(ialu_reg);
11258 %}
11259
11260 // This pattern is automatically generated from aarch64_ad.m4
11261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11262 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11263 immI0 zero, iRegIorL2I src1, immI src2) %{
11264 match(Set dst (SubI zero (URShiftI src1 src2)));
11265
11266 ins_cost(1.9 * INSN_COST);
11267 format %{ "negw $dst, $src1, LSR $src2" %}
11268
11269 ins_encode %{
11270 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11271 Assembler::LSR, $src2$$constant & 0x1f);
11272 %}
11273
11274 ins_pipe(ialu_reg_shift);
11275 %}
11276
11277 // This pattern is automatically generated from aarch64_ad.m4
11278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11279 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11280 immI0 zero, iRegIorL2I src1, immI src2) %{
11281 match(Set dst (SubI zero (RShiftI src1 src2)));
11282
11283 ins_cost(1.9 * INSN_COST);
11284 format %{ "negw $dst, $src1, ASR $src2" %}
11285
11286 ins_encode %{
11287 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11288 Assembler::ASR, $src2$$constant & 0x1f);
11289 %}
11290
11291 ins_pipe(ialu_reg_shift);
11292 %}
11293
11294 // This pattern is automatically generated from aarch64_ad.m4
11295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11296 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11297 immI0 zero, iRegIorL2I src1, immI src2) %{
11298 match(Set dst (SubI zero (LShiftI src1 src2)));
11299
11300 ins_cost(1.9 * INSN_COST);
11301 format %{ "negw $dst, $src1, LSL $src2" %}
11302
11303 ins_encode %{
11304 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11305 Assembler::LSL, $src2$$constant & 0x1f);
11306 %}
11307
11308 ins_pipe(ialu_reg_shift);
11309 %}
11310
11311 // This pattern is automatically generated from aarch64_ad.m4
11312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11313 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11314 immL0 zero, iRegL src1, immI src2) %{
11315 match(Set dst (SubL zero (URShiftL src1 src2)));
11316
11317 ins_cost(1.9 * INSN_COST);
11318 format %{ "neg $dst, $src1, LSR $src2" %}
11319
11320 ins_encode %{
11321 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11322 Assembler::LSR, $src2$$constant & 0x3f);
11323 %}
11324
11325 ins_pipe(ialu_reg_shift);
11326 %}
11327
11328 // This pattern is automatically generated from aarch64_ad.m4
11329 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11330 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11331 immL0 zero, iRegL src1, immI src2) %{
11332 match(Set dst (SubL zero (RShiftL src1 src2)));
11333
11334 ins_cost(1.9 * INSN_COST);
11335 format %{ "neg $dst, $src1, ASR $src2" %}
11336
11337 ins_encode %{
11338 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11339 Assembler::ASR, $src2$$constant & 0x3f);
11340 %}
11341
11342 ins_pipe(ialu_reg_shift);
11343 %}
11344
11345 // This pattern is automatically generated from aarch64_ad.m4
11346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11347 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11348 immL0 zero, iRegL src1, immI src2) %{
11349 match(Set dst (SubL zero (LShiftL src1 src2)));
11350
11351 ins_cost(1.9 * INSN_COST);
11352 format %{ "neg $dst, $src1, LSL $src2" %}
11353
11354 ins_encode %{
11355 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11356 Assembler::LSL, $src2$$constant & 0x3f);
11357 %}
11358
11359 ins_pipe(ialu_reg_shift);
11360 %}
11361
11362 // This pattern is automatically generated from aarch64_ad.m4
11363 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11364 instruct AndI_reg_not_reg(iRegINoSp dst,
11365 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11366 match(Set dst (AndI src1 (XorI src2 m1)));
11367 ins_cost(INSN_COST);
11368 format %{ "bicw $dst, $src1, $src2" %}
11369
11370 ins_encode %{
11371 __ bicw(as_Register($dst$$reg),
11372 as_Register($src1$$reg),
11373 as_Register($src2$$reg),
11374 Assembler::LSL, 0);
11375 %}
11376
11377 ins_pipe(ialu_reg_reg);
11378 %}
11379
11380 // This pattern is automatically generated from aarch64_ad.m4
11381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11382 instruct AndL_reg_not_reg(iRegLNoSp dst,
11383 iRegL src1, iRegL src2, immL_M1 m1) %{
11384 match(Set dst (AndL src1 (XorL src2 m1)));
11385 ins_cost(INSN_COST);
11386 format %{ "bic $dst, $src1, $src2" %}
11387
11388 ins_encode %{
11389 __ bic(as_Register($dst$$reg),
11390 as_Register($src1$$reg),
11391 as_Register($src2$$reg),
11392 Assembler::LSL, 0);
11393 %}
11394
11395 ins_pipe(ialu_reg_reg);
11396 %}
11397
11398 // This pattern is automatically generated from aarch64_ad.m4
11399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11400 instruct OrI_reg_not_reg(iRegINoSp dst,
11401 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11402 match(Set dst (OrI src1 (XorI src2 m1)));
11403 ins_cost(INSN_COST);
11404 format %{ "ornw $dst, $src1, $src2" %}
11405
11406 ins_encode %{
11407 __ ornw(as_Register($dst$$reg),
11408 as_Register($src1$$reg),
11409 as_Register($src2$$reg),
11410 Assembler::LSL, 0);
11411 %}
11412
11413 ins_pipe(ialu_reg_reg);
11414 %}
11415
11416 // This pattern is automatically generated from aarch64_ad.m4
11417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11418 instruct OrL_reg_not_reg(iRegLNoSp dst,
11419 iRegL src1, iRegL src2, immL_M1 m1) %{
11420 match(Set dst (OrL src1 (XorL src2 m1)));
11421 ins_cost(INSN_COST);
11422 format %{ "orn $dst, $src1, $src2" %}
11423
11424 ins_encode %{
11425 __ orn(as_Register($dst$$reg),
11426 as_Register($src1$$reg),
11427 as_Register($src2$$reg),
11428 Assembler::LSL, 0);
11429 %}
11430
11431 ins_pipe(ialu_reg_reg);
11432 %}
11433
11434 // This pattern is automatically generated from aarch64_ad.m4
11435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11436 instruct XorI_reg_not_reg(iRegINoSp dst,
11437 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11438 match(Set dst (XorI m1 (XorI src2 src1)));
11439 ins_cost(INSN_COST);
11440 format %{ "eonw $dst, $src1, $src2" %}
11441
11442 ins_encode %{
11443 __ eonw(as_Register($dst$$reg),
11444 as_Register($src1$$reg),
11445 as_Register($src2$$reg),
11446 Assembler::LSL, 0);
11447 %}
11448
11449 ins_pipe(ialu_reg_reg);
11450 %}
11451
11452 // This pattern is automatically generated from aarch64_ad.m4
11453 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11454 instruct XorL_reg_not_reg(iRegLNoSp dst,
11455 iRegL src1, iRegL src2, immL_M1 m1) %{
11456 match(Set dst (XorL m1 (XorL src2 src1)));
11457 ins_cost(INSN_COST);
11458 format %{ "eon $dst, $src1, $src2" %}
11459
11460 ins_encode %{
11461 __ eon(as_Register($dst$$reg),
11462 as_Register($src1$$reg),
11463 as_Register($src2$$reg),
11464 Assembler::LSL, 0);
11465 %}
11466
11467 ins_pipe(ialu_reg_reg);
11468 %}
11469
11470 // This pattern is automatically generated from aarch64_ad.m4
11471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11472 // val & (-1 ^ (val >>> shift)) ==> bicw
11473 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11474 iRegIorL2I src1, iRegIorL2I src2,
11475 immI src3, immI_M1 src4) %{
11476 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11477 ins_cost(1.9 * INSN_COST);
11478 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11479
11480 ins_encode %{
11481 __ bicw(as_Register($dst$$reg),
11482 as_Register($src1$$reg),
11483 as_Register($src2$$reg),
11484 Assembler::LSR,
11485 $src3$$constant & 0x1f);
11486 %}
11487
11488 ins_pipe(ialu_reg_reg_shift);
11489 %}
11490
11491 // This pattern is automatically generated from aarch64_ad.m4
11492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11493 // val & (-1 ^ (val >>> shift)) ==> bic
11494 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11495 iRegL src1, iRegL src2,
11496 immI src3, immL_M1 src4) %{
11497 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11498 ins_cost(1.9 * INSN_COST);
11499 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11500
11501 ins_encode %{
11502 __ bic(as_Register($dst$$reg),
11503 as_Register($src1$$reg),
11504 as_Register($src2$$reg),
11505 Assembler::LSR,
11506 $src3$$constant & 0x3f);
11507 %}
11508
11509 ins_pipe(ialu_reg_reg_shift);
11510 %}
11511
11512 // This pattern is automatically generated from aarch64_ad.m4
11513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11514 // val & (-1 ^ (val >> shift)) ==> bicw
11515 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11516 iRegIorL2I src1, iRegIorL2I src2,
11517 immI src3, immI_M1 src4) %{
11518 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11519 ins_cost(1.9 * INSN_COST);
11520 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11521
11522 ins_encode %{
11523 __ bicw(as_Register($dst$$reg),
11524 as_Register($src1$$reg),
11525 as_Register($src2$$reg),
11526 Assembler::ASR,
11527 $src3$$constant & 0x1f);
11528 %}
11529
11530 ins_pipe(ialu_reg_reg_shift);
11531 %}
11532
11533 // This pattern is automatically generated from aarch64_ad.m4
11534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11535 // val & (-1 ^ (val >> shift)) ==> bic
11536 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11537 iRegL src1, iRegL src2,
11538 immI src3, immL_M1 src4) %{
11539 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11540 ins_cost(1.9 * INSN_COST);
11541 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11542
11543 ins_encode %{
11544 __ bic(as_Register($dst$$reg),
11545 as_Register($src1$$reg),
11546 as_Register($src2$$reg),
11547 Assembler::ASR,
11548 $src3$$constant & 0x3f);
11549 %}
11550
11551 ins_pipe(ialu_reg_reg_shift);
11552 %}
11553
11554 // This pattern is automatically generated from aarch64_ad.m4
11555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11556 // val & (-1 ^ (val ror shift)) ==> bicw
11557 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11558 iRegIorL2I src1, iRegIorL2I src2,
11559 immI src3, immI_M1 src4) %{
11560 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11561 ins_cost(1.9 * INSN_COST);
11562 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11563
11564 ins_encode %{
11565 __ bicw(as_Register($dst$$reg),
11566 as_Register($src1$$reg),
11567 as_Register($src2$$reg),
11568 Assembler::ROR,
11569 $src3$$constant & 0x1f);
11570 %}
11571
11572 ins_pipe(ialu_reg_reg_shift);
11573 %}
11574
11575 // This pattern is automatically generated from aarch64_ad.m4
11576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11577 // val & (-1 ^ (val ror shift)) ==> bic
11578 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11579 iRegL src1, iRegL src2,
11580 immI src3, immL_M1 src4) %{
11581 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11582 ins_cost(1.9 * INSN_COST);
11583 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11584
11585 ins_encode %{
11586 __ bic(as_Register($dst$$reg),
11587 as_Register($src1$$reg),
11588 as_Register($src2$$reg),
11589 Assembler::ROR,
11590 $src3$$constant & 0x3f);
11591 %}
11592
11593 ins_pipe(ialu_reg_reg_shift);
11594 %}
11595
11596 // This pattern is automatically generated from aarch64_ad.m4
11597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11598 // val & (-1 ^ (val << shift)) ==> bicw
11599 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11600 iRegIorL2I src1, iRegIorL2I src2,
11601 immI src3, immI_M1 src4) %{
11602 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11603 ins_cost(1.9 * INSN_COST);
11604 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11605
11606 ins_encode %{
11607 __ bicw(as_Register($dst$$reg),
11608 as_Register($src1$$reg),
11609 as_Register($src2$$reg),
11610 Assembler::LSL,
11611 $src3$$constant & 0x1f);
11612 %}
11613
11614 ins_pipe(ialu_reg_reg_shift);
11615 %}
11616
11617 // This pattern is automatically generated from aarch64_ad.m4
11618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11619 // val & (-1 ^ (val << shift)) ==> bic
11620 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11621 iRegL src1, iRegL src2,
11622 immI src3, immL_M1 src4) %{
11623 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11624 ins_cost(1.9 * INSN_COST);
11625 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11626
11627 ins_encode %{
11628 __ bic(as_Register($dst$$reg),
11629 as_Register($src1$$reg),
11630 as_Register($src2$$reg),
11631 Assembler::LSL,
11632 $src3$$constant & 0x3f);
11633 %}
11634
11635 ins_pipe(ialu_reg_reg_shift);
11636 %}
11637
11638 // This pattern is automatically generated from aarch64_ad.m4
11639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11640 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11641 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11642 iRegIorL2I src1, iRegIorL2I src2,
11643 immI src3, immI_M1 src4) %{
11644 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11645 ins_cost(1.9 * INSN_COST);
11646 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11647
11648 ins_encode %{
11649 __ eonw(as_Register($dst$$reg),
11650 as_Register($src1$$reg),
11651 as_Register($src2$$reg),
11652 Assembler::LSR,
11653 $src3$$constant & 0x1f);
11654 %}
11655
11656 ins_pipe(ialu_reg_reg_shift);
11657 %}
11658
11659 // This pattern is automatically generated from aarch64_ad.m4
11660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11661 // val ^ (-1 ^ (val >>> shift)) ==> eon
11662 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11663 iRegL src1, iRegL src2,
11664 immI src3, immL_M1 src4) %{
11665 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11666 ins_cost(1.9 * INSN_COST);
11667 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11668
11669 ins_encode %{
11670 __ eon(as_Register($dst$$reg),
11671 as_Register($src1$$reg),
11672 as_Register($src2$$reg),
11673 Assembler::LSR,
11674 $src3$$constant & 0x3f);
11675 %}
11676
11677 ins_pipe(ialu_reg_reg_shift);
11678 %}
11679
11680 // This pattern is automatically generated from aarch64_ad.m4
11681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11682 // val ^ (-1 ^ (val >> shift)) ==> eonw
11683 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11684 iRegIorL2I src1, iRegIorL2I src2,
11685 immI src3, immI_M1 src4) %{
11686 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11687 ins_cost(1.9 * INSN_COST);
11688 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11689
11690 ins_encode %{
11691 __ eonw(as_Register($dst$$reg),
11692 as_Register($src1$$reg),
11693 as_Register($src2$$reg),
11694 Assembler::ASR,
11695 $src3$$constant & 0x1f);
11696 %}
11697
11698 ins_pipe(ialu_reg_reg_shift);
11699 %}
11700
11701 // This pattern is automatically generated from aarch64_ad.m4
11702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11703 // val ^ (-1 ^ (val >> shift)) ==> eon
11704 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11705 iRegL src1, iRegL src2,
11706 immI src3, immL_M1 src4) %{
11707 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11708 ins_cost(1.9 * INSN_COST);
11709 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11710
11711 ins_encode %{
11712 __ eon(as_Register($dst$$reg),
11713 as_Register($src1$$reg),
11714 as_Register($src2$$reg),
11715 Assembler::ASR,
11716 $src3$$constant & 0x3f);
11717 %}
11718
11719 ins_pipe(ialu_reg_reg_shift);
11720 %}
11721
11722 // This pattern is automatically generated from aarch64_ad.m4
11723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11724 // val ^ (-1 ^ (val ror shift)) ==> eonw
11725 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11726 iRegIorL2I src1, iRegIorL2I src2,
11727 immI src3, immI_M1 src4) %{
11728 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11729 ins_cost(1.9 * INSN_COST);
11730 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11731
11732 ins_encode %{
11733 __ eonw(as_Register($dst$$reg),
11734 as_Register($src1$$reg),
11735 as_Register($src2$$reg),
11736 Assembler::ROR,
11737 $src3$$constant & 0x1f);
11738 %}
11739
11740 ins_pipe(ialu_reg_reg_shift);
11741 %}
11742
11743 // This pattern is automatically generated from aarch64_ad.m4
11744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11745 // val ^ (-1 ^ (val ror shift)) ==> eon
11746 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11747 iRegL src1, iRegL src2,
11748 immI src3, immL_M1 src4) %{
11749 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11750 ins_cost(1.9 * INSN_COST);
11751 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11752
11753 ins_encode %{
11754 __ eon(as_Register($dst$$reg),
11755 as_Register($src1$$reg),
11756 as_Register($src2$$reg),
11757 Assembler::ROR,
11758 $src3$$constant & 0x3f);
11759 %}
11760
11761 ins_pipe(ialu_reg_reg_shift);
11762 %}
11763
11764 // This pattern is automatically generated from aarch64_ad.m4
11765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11766 // val ^ (-1 ^ (val << shift)) ==> eonw
11767 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11768 iRegIorL2I src1, iRegIorL2I src2,
11769 immI src3, immI_M1 src4) %{
11770 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11771 ins_cost(1.9 * INSN_COST);
11772 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11773
11774 ins_encode %{
11775 __ eonw(as_Register($dst$$reg),
11776 as_Register($src1$$reg),
11777 as_Register($src2$$reg),
11778 Assembler::LSL,
11779 $src3$$constant & 0x1f);
11780 %}
11781
11782 ins_pipe(ialu_reg_reg_shift);
11783 %}
11784
11785 // This pattern is automatically generated from aarch64_ad.m4
11786 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11787 // val ^ (-1 ^ (val << shift)) ==> eon
11788 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11789 iRegL src1, iRegL src2,
11790 immI src3, immL_M1 src4) %{
11791 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11792 ins_cost(1.9 * INSN_COST);
11793 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11794
11795 ins_encode %{
11796 __ eon(as_Register($dst$$reg),
11797 as_Register($src1$$reg),
11798 as_Register($src2$$reg),
11799 Assembler::LSL,
11800 $src3$$constant & 0x3f);
11801 %}
11802
11803 ins_pipe(ialu_reg_reg_shift);
11804 %}
11805
11806 // This pattern is automatically generated from aarch64_ad.m4
11807 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11808 // val | (-1 ^ (val >>> shift)) ==> ornw
11809 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11810 iRegIorL2I src1, iRegIorL2I src2,
11811 immI src3, immI_M1 src4) %{
11812 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11813 ins_cost(1.9 * INSN_COST);
11814 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11815
11816 ins_encode %{
11817 __ ornw(as_Register($dst$$reg),
11818 as_Register($src1$$reg),
11819 as_Register($src2$$reg),
11820 Assembler::LSR,
11821 $src3$$constant & 0x1f);
11822 %}
11823
11824 ins_pipe(ialu_reg_reg_shift);
11825 %}
11826
11827 // This pattern is automatically generated from aarch64_ad.m4
11828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11829 // val | (-1 ^ (val >>> shift)) ==> orn
11830 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11831 iRegL src1, iRegL src2,
11832 immI src3, immL_M1 src4) %{
11833 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11834 ins_cost(1.9 * INSN_COST);
11835 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11836
11837 ins_encode %{
11838 __ orn(as_Register($dst$$reg),
11839 as_Register($src1$$reg),
11840 as_Register($src2$$reg),
11841 Assembler::LSR,
11842 $src3$$constant & 0x3f);
11843 %}
11844
11845 ins_pipe(ialu_reg_reg_shift);
11846 %}
11847
11848 // This pattern is automatically generated from aarch64_ad.m4
11849 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11850 // val | (-1 ^ (val >> shift)) ==> ornw
11851 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11852 iRegIorL2I src1, iRegIorL2I src2,
11853 immI src3, immI_M1 src4) %{
11854 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11855 ins_cost(1.9 * INSN_COST);
11856 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11857
11858 ins_encode %{
11859 __ ornw(as_Register($dst$$reg),
11860 as_Register($src1$$reg),
11861 as_Register($src2$$reg),
11862 Assembler::ASR,
11863 $src3$$constant & 0x1f);
11864 %}
11865
11866 ins_pipe(ialu_reg_reg_shift);
11867 %}
11868
11869 // This pattern is automatically generated from aarch64_ad.m4
11870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11871 // val | (-1 ^ (val >> shift)) ==> orn
11872 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11873 iRegL src1, iRegL src2,
11874 immI src3, immL_M1 src4) %{
11875 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11876 ins_cost(1.9 * INSN_COST);
11877 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11878
11879 ins_encode %{
11880 __ orn(as_Register($dst$$reg),
11881 as_Register($src1$$reg),
11882 as_Register($src2$$reg),
11883 Assembler::ASR,
11884 $src3$$constant & 0x3f);
11885 %}
11886
11887 ins_pipe(ialu_reg_reg_shift);
11888 %}
11889
11890 // This pattern is automatically generated from aarch64_ad.m4
11891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11892 // val | (-1 ^ (val ror shift)) ==> ornw
11893 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11894 iRegIorL2I src1, iRegIorL2I src2,
11895 immI src3, immI_M1 src4) %{
11896 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11897 ins_cost(1.9 * INSN_COST);
11898 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11899
11900 ins_encode %{
11901 __ ornw(as_Register($dst$$reg),
11902 as_Register($src1$$reg),
11903 as_Register($src2$$reg),
11904 Assembler::ROR,
11905 $src3$$constant & 0x1f);
11906 %}
11907
11908 ins_pipe(ialu_reg_reg_shift);
11909 %}
11910
11911 // This pattern is automatically generated from aarch64_ad.m4
11912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11913 // val | (-1 ^ (val ror shift)) ==> orn
11914 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11915 iRegL src1, iRegL src2,
11916 immI src3, immL_M1 src4) %{
11917 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11918 ins_cost(1.9 * INSN_COST);
11919 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11920
11921 ins_encode %{
11922 __ orn(as_Register($dst$$reg),
11923 as_Register($src1$$reg),
11924 as_Register($src2$$reg),
11925 Assembler::ROR,
11926 $src3$$constant & 0x3f);
11927 %}
11928
11929 ins_pipe(ialu_reg_reg_shift);
11930 %}
11931
11932 // This pattern is automatically generated from aarch64_ad.m4
11933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11934 // val | (-1 ^ (val << shift)) ==> ornw
11935 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11936 iRegIorL2I src1, iRegIorL2I src2,
11937 immI src3, immI_M1 src4) %{
11938 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11939 ins_cost(1.9 * INSN_COST);
11940 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11941
11942 ins_encode %{
11943 __ ornw(as_Register($dst$$reg),
11944 as_Register($src1$$reg),
11945 as_Register($src2$$reg),
11946 Assembler::LSL,
11947 $src3$$constant & 0x1f);
11948 %}
11949
11950 ins_pipe(ialu_reg_reg_shift);
11951 %}
11952
11953 // This pattern is automatically generated from aarch64_ad.m4
11954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11955 // val | (-1 ^ (val << shift)) ==> orn
11956 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11957 iRegL src1, iRegL src2,
11958 immI src3, immL_M1 src4) %{
11959 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11960 ins_cost(1.9 * INSN_COST);
11961 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11962
11963 ins_encode %{
11964 __ orn(as_Register($dst$$reg),
11965 as_Register($src1$$reg),
11966 as_Register($src2$$reg),
11967 Assembler::LSL,
11968 $src3$$constant & 0x3f);
11969 %}
11970
11971 ins_pipe(ialu_reg_reg_shift);
11972 %}
11973
11974 // This pattern is automatically generated from aarch64_ad.m4
11975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11976 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11977 iRegIorL2I src1, iRegIorL2I src2,
11978 immI src3) %{
11979 match(Set dst (AndI src1 (URShiftI src2 src3)));
11980
11981 ins_cost(1.9 * INSN_COST);
11982 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11983
11984 ins_encode %{
11985 __ andw(as_Register($dst$$reg),
11986 as_Register($src1$$reg),
11987 as_Register($src2$$reg),
11988 Assembler::LSR,
11989 $src3$$constant & 0x1f);
11990 %}
11991
11992 ins_pipe(ialu_reg_reg_shift);
11993 %}
11994
11995 // This pattern is automatically generated from aarch64_ad.m4
11996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11997 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11998 iRegL src1, iRegL src2,
11999 immI src3) %{
12000 match(Set dst (AndL src1 (URShiftL src2 src3)));
12001
12002 ins_cost(1.9 * INSN_COST);
12003 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
12004
12005 ins_encode %{
12006 __ andr(as_Register($dst$$reg),
12007 as_Register($src1$$reg),
12008 as_Register($src2$$reg),
12009 Assembler::LSR,
12010 $src3$$constant & 0x3f);
12011 %}
12012
12013 ins_pipe(ialu_reg_reg_shift);
12014 %}
12015
12016 // This pattern is automatically generated from aarch64_ad.m4
12017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12018 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12019 iRegIorL2I src1, iRegIorL2I src2,
12020 immI src3) %{
12021 match(Set dst (AndI src1 (RShiftI src2 src3)));
12022
12023 ins_cost(1.9 * INSN_COST);
12024 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
12025
12026 ins_encode %{
12027 __ andw(as_Register($dst$$reg),
12028 as_Register($src1$$reg),
12029 as_Register($src2$$reg),
12030 Assembler::ASR,
12031 $src3$$constant & 0x1f);
12032 %}
12033
12034 ins_pipe(ialu_reg_reg_shift);
12035 %}
12036
12037 // This pattern is automatically generated from aarch64_ad.m4
12038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12039 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12040 iRegL src1, iRegL src2,
12041 immI src3) %{
12042 match(Set dst (AndL src1 (RShiftL src2 src3)));
12043
12044 ins_cost(1.9 * INSN_COST);
12045 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
12046
12047 ins_encode %{
12048 __ andr(as_Register($dst$$reg),
12049 as_Register($src1$$reg),
12050 as_Register($src2$$reg),
12051 Assembler::ASR,
12052 $src3$$constant & 0x3f);
12053 %}
12054
12055 ins_pipe(ialu_reg_reg_shift);
12056 %}
12057
12058 // This pattern is automatically generated from aarch64_ad.m4
12059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12060 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12061 iRegIorL2I src1, iRegIorL2I src2,
12062 immI src3) %{
12063 match(Set dst (AndI src1 (LShiftI src2 src3)));
12064
12065 ins_cost(1.9 * INSN_COST);
12066 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
12067
12068 ins_encode %{
12069 __ andw(as_Register($dst$$reg),
12070 as_Register($src1$$reg),
12071 as_Register($src2$$reg),
12072 Assembler::LSL,
12073 $src3$$constant & 0x1f);
12074 %}
12075
12076 ins_pipe(ialu_reg_reg_shift);
12077 %}
12078
12079 // This pattern is automatically generated from aarch64_ad.m4
12080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12081 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12082 iRegL src1, iRegL src2,
12083 immI src3) %{
12084 match(Set dst (AndL src1 (LShiftL src2 src3)));
12085
12086 ins_cost(1.9 * INSN_COST);
12087 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
12088
12089 ins_encode %{
12090 __ andr(as_Register($dst$$reg),
12091 as_Register($src1$$reg),
12092 as_Register($src2$$reg),
12093 Assembler::LSL,
12094 $src3$$constant & 0x3f);
12095 %}
12096
12097 ins_pipe(ialu_reg_reg_shift);
12098 %}
12099
12100 // This pattern is automatically generated from aarch64_ad.m4
12101 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12102 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12103 iRegIorL2I src1, iRegIorL2I src2,
12104 immI src3) %{
12105 match(Set dst (AndI src1 (RotateRight src2 src3)));
12106
12107 ins_cost(1.9 * INSN_COST);
12108 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
12109
12110 ins_encode %{
12111 __ andw(as_Register($dst$$reg),
12112 as_Register($src1$$reg),
12113 as_Register($src2$$reg),
12114 Assembler::ROR,
12115 $src3$$constant & 0x1f);
12116 %}
12117
12118 ins_pipe(ialu_reg_reg_shift);
12119 %}
12120
12121 // This pattern is automatically generated from aarch64_ad.m4
12122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12123 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12124 iRegL src1, iRegL src2,
12125 immI src3) %{
12126 match(Set dst (AndL src1 (RotateRight src2 src3)));
12127
12128 ins_cost(1.9 * INSN_COST);
12129 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
12130
12131 ins_encode %{
12132 __ andr(as_Register($dst$$reg),
12133 as_Register($src1$$reg),
12134 as_Register($src2$$reg),
12135 Assembler::ROR,
12136 $src3$$constant & 0x3f);
12137 %}
12138
12139 ins_pipe(ialu_reg_reg_shift);
12140 %}
12141
12142 // This pattern is automatically generated from aarch64_ad.m4
12143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12144 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12145 iRegIorL2I src1, iRegIorL2I src2,
12146 immI src3) %{
12147 match(Set dst (XorI src1 (URShiftI src2 src3)));
12148
12149 ins_cost(1.9 * INSN_COST);
12150 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
12151
12152 ins_encode %{
12153 __ eorw(as_Register($dst$$reg),
12154 as_Register($src1$$reg),
12155 as_Register($src2$$reg),
12156 Assembler::LSR,
12157 $src3$$constant & 0x1f);
12158 %}
12159
12160 ins_pipe(ialu_reg_reg_shift);
12161 %}
12162
12163 // This pattern is automatically generated from aarch64_ad.m4
12164 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12165 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12166 iRegL src1, iRegL src2,
12167 immI src3) %{
12168 match(Set dst (XorL src1 (URShiftL src2 src3)));
12169
12170 ins_cost(1.9 * INSN_COST);
12171 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
12172
12173 ins_encode %{
12174 __ eor(as_Register($dst$$reg),
12175 as_Register($src1$$reg),
12176 as_Register($src2$$reg),
12177 Assembler::LSR,
12178 $src3$$constant & 0x3f);
12179 %}
12180
12181 ins_pipe(ialu_reg_reg_shift);
12182 %}
12183
12184 // This pattern is automatically generated from aarch64_ad.m4
12185 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12186 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12187 iRegIorL2I src1, iRegIorL2I src2,
12188 immI src3) %{
12189 match(Set dst (XorI src1 (RShiftI src2 src3)));
12190
12191 ins_cost(1.9 * INSN_COST);
12192 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
12193
12194 ins_encode %{
12195 __ eorw(as_Register($dst$$reg),
12196 as_Register($src1$$reg),
12197 as_Register($src2$$reg),
12198 Assembler::ASR,
12199 $src3$$constant & 0x1f);
12200 %}
12201
12202 ins_pipe(ialu_reg_reg_shift);
12203 %}
12204
12205 // This pattern is automatically generated from aarch64_ad.m4
12206 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12207 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12208 iRegL src1, iRegL src2,
12209 immI src3) %{
12210 match(Set dst (XorL src1 (RShiftL src2 src3)));
12211
12212 ins_cost(1.9 * INSN_COST);
12213 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
12214
12215 ins_encode %{
12216 __ eor(as_Register($dst$$reg),
12217 as_Register($src1$$reg),
12218 as_Register($src2$$reg),
12219 Assembler::ASR,
12220 $src3$$constant & 0x3f);
12221 %}
12222
12223 ins_pipe(ialu_reg_reg_shift);
12224 %}
12225
12226 // This pattern is automatically generated from aarch64_ad.m4
12227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12228 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12229 iRegIorL2I src1, iRegIorL2I src2,
12230 immI src3) %{
12231 match(Set dst (XorI src1 (LShiftI src2 src3)));
12232
12233 ins_cost(1.9 * INSN_COST);
12234 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
12235
12236 ins_encode %{
12237 __ eorw(as_Register($dst$$reg),
12238 as_Register($src1$$reg),
12239 as_Register($src2$$reg),
12240 Assembler::LSL,
12241 $src3$$constant & 0x1f);
12242 %}
12243
12244 ins_pipe(ialu_reg_reg_shift);
12245 %}
12246
12247 // This pattern is automatically generated from aarch64_ad.m4
12248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12249 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12250 iRegL src1, iRegL src2,
12251 immI src3) %{
12252 match(Set dst (XorL src1 (LShiftL src2 src3)));
12253
12254 ins_cost(1.9 * INSN_COST);
12255 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
12256
12257 ins_encode %{
12258 __ eor(as_Register($dst$$reg),
12259 as_Register($src1$$reg),
12260 as_Register($src2$$reg),
12261 Assembler::LSL,
12262 $src3$$constant & 0x3f);
12263 %}
12264
12265 ins_pipe(ialu_reg_reg_shift);
12266 %}
12267
12268 // This pattern is automatically generated from aarch64_ad.m4
12269 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12270 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12271 iRegIorL2I src1, iRegIorL2I src2,
12272 immI src3) %{
12273 match(Set dst (XorI src1 (RotateRight src2 src3)));
12274
12275 ins_cost(1.9 * INSN_COST);
12276 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
12277
12278 ins_encode %{
12279 __ eorw(as_Register($dst$$reg),
12280 as_Register($src1$$reg),
12281 as_Register($src2$$reg),
12282 Assembler::ROR,
12283 $src3$$constant & 0x1f);
12284 %}
12285
12286 ins_pipe(ialu_reg_reg_shift);
12287 %}
12288
12289 // This pattern is automatically generated from aarch64_ad.m4
12290 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12291 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12292 iRegL src1, iRegL src2,
12293 immI src3) %{
12294 match(Set dst (XorL src1 (RotateRight src2 src3)));
12295
12296 ins_cost(1.9 * INSN_COST);
12297 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
12298
12299 ins_encode %{
12300 __ eor(as_Register($dst$$reg),
12301 as_Register($src1$$reg),
12302 as_Register($src2$$reg),
12303 Assembler::ROR,
12304 $src3$$constant & 0x3f);
12305 %}
12306
12307 ins_pipe(ialu_reg_reg_shift);
12308 %}
12309
12310 // This pattern is automatically generated from aarch64_ad.m4
12311 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12312 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12313 iRegIorL2I src1, iRegIorL2I src2,
12314 immI src3) %{
12315 match(Set dst (OrI src1 (URShiftI src2 src3)));
12316
12317 ins_cost(1.9 * INSN_COST);
12318 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
12319
12320 ins_encode %{
12321 __ orrw(as_Register($dst$$reg),
12322 as_Register($src1$$reg),
12323 as_Register($src2$$reg),
12324 Assembler::LSR,
12325 $src3$$constant & 0x1f);
12326 %}
12327
12328 ins_pipe(ialu_reg_reg_shift);
12329 %}
12330
12331 // This pattern is automatically generated from aarch64_ad.m4
12332 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12333 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12334 iRegL src1, iRegL src2,
12335 immI src3) %{
12336 match(Set dst (OrL src1 (URShiftL src2 src3)));
12337
12338 ins_cost(1.9 * INSN_COST);
12339 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12340
12341 ins_encode %{
12342 __ orr(as_Register($dst$$reg),
12343 as_Register($src1$$reg),
12344 as_Register($src2$$reg),
12345 Assembler::LSR,
12346 $src3$$constant & 0x3f);
12347 %}
12348
12349 ins_pipe(ialu_reg_reg_shift);
12350 %}
12351
12352 // This pattern is automatically generated from aarch64_ad.m4
12353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12354 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12355 iRegIorL2I src1, iRegIorL2I src2,
12356 immI src3) %{
12357 match(Set dst (OrI src1 (RShiftI src2 src3)));
12358
12359 ins_cost(1.9 * INSN_COST);
12360 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12361
12362 ins_encode %{
12363 __ orrw(as_Register($dst$$reg),
12364 as_Register($src1$$reg),
12365 as_Register($src2$$reg),
12366 Assembler::ASR,
12367 $src3$$constant & 0x1f);
12368 %}
12369
12370 ins_pipe(ialu_reg_reg_shift);
12371 %}
12372
12373 // This pattern is automatically generated from aarch64_ad.m4
12374 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12375 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12376 iRegL src1, iRegL src2,
12377 immI src3) %{
12378 match(Set dst (OrL src1 (RShiftL src2 src3)));
12379
12380 ins_cost(1.9 * INSN_COST);
12381 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12382
12383 ins_encode %{
12384 __ orr(as_Register($dst$$reg),
12385 as_Register($src1$$reg),
12386 as_Register($src2$$reg),
12387 Assembler::ASR,
12388 $src3$$constant & 0x3f);
12389 %}
12390
12391 ins_pipe(ialu_reg_reg_shift);
12392 %}
12393
12394 // This pattern is automatically generated from aarch64_ad.m4
12395 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12396 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12397 iRegIorL2I src1, iRegIorL2I src2,
12398 immI src3) %{
12399 match(Set dst (OrI src1 (LShiftI src2 src3)));
12400
12401 ins_cost(1.9 * INSN_COST);
12402 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12403
12404 ins_encode %{
12405 __ orrw(as_Register($dst$$reg),
12406 as_Register($src1$$reg),
12407 as_Register($src2$$reg),
12408 Assembler::LSL,
12409 $src3$$constant & 0x1f);
12410 %}
12411
12412 ins_pipe(ialu_reg_reg_shift);
12413 %}
12414
12415 // This pattern is automatically generated from aarch64_ad.m4
12416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12417 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12418 iRegL src1, iRegL src2,
12419 immI src3) %{
12420 match(Set dst (OrL src1 (LShiftL src2 src3)));
12421
12422 ins_cost(1.9 * INSN_COST);
12423 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12424
12425 ins_encode %{
12426 __ orr(as_Register($dst$$reg),
12427 as_Register($src1$$reg),
12428 as_Register($src2$$reg),
12429 Assembler::LSL,
12430 $src3$$constant & 0x3f);
12431 %}
12432
12433 ins_pipe(ialu_reg_reg_shift);
12434 %}
12435
12436 // This pattern is automatically generated from aarch64_ad.m4
12437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12438 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12439 iRegIorL2I src1, iRegIorL2I src2,
12440 immI src3) %{
12441 match(Set dst (OrI src1 (RotateRight src2 src3)));
12442
12443 ins_cost(1.9 * INSN_COST);
12444 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12445
12446 ins_encode %{
12447 __ orrw(as_Register($dst$$reg),
12448 as_Register($src1$$reg),
12449 as_Register($src2$$reg),
12450 Assembler::ROR,
12451 $src3$$constant & 0x1f);
12452 %}
12453
12454 ins_pipe(ialu_reg_reg_shift);
12455 %}
12456
12457 // This pattern is automatically generated from aarch64_ad.m4
12458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12459 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12460 iRegL src1, iRegL src2,
12461 immI src3) %{
12462 match(Set dst (OrL src1 (RotateRight src2 src3)));
12463
12464 ins_cost(1.9 * INSN_COST);
12465 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12466
12467 ins_encode %{
12468 __ orr(as_Register($dst$$reg),
12469 as_Register($src1$$reg),
12470 as_Register($src2$$reg),
12471 Assembler::ROR,
12472 $src3$$constant & 0x3f);
12473 %}
12474
12475 ins_pipe(ialu_reg_reg_shift);
12476 %}
12477
12478 // This pattern is automatically generated from aarch64_ad.m4
12479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12480 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12481 iRegIorL2I src1, iRegIorL2I src2,
12482 immI src3) %{
12483 match(Set dst (AddI src1 (URShiftI src2 src3)));
12484
12485 ins_cost(1.9 * INSN_COST);
12486 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12487
12488 ins_encode %{
12489 __ addw(as_Register($dst$$reg),
12490 as_Register($src1$$reg),
12491 as_Register($src2$$reg),
12492 Assembler::LSR,
12493 $src3$$constant & 0x1f);
12494 %}
12495
12496 ins_pipe(ialu_reg_reg_shift);
12497 %}
12498
12499 // This pattern is automatically generated from aarch64_ad.m4
12500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12501 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12502 iRegL src1, iRegL src2,
12503 immI src3) %{
12504 match(Set dst (AddL src1 (URShiftL src2 src3)));
12505
12506 ins_cost(1.9 * INSN_COST);
12507 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12508
12509 ins_encode %{
12510 __ add(as_Register($dst$$reg),
12511 as_Register($src1$$reg),
12512 as_Register($src2$$reg),
12513 Assembler::LSR,
12514 $src3$$constant & 0x3f);
12515 %}
12516
12517 ins_pipe(ialu_reg_reg_shift);
12518 %}
12519
12520 // This pattern is automatically generated from aarch64_ad.m4
12521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12522 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12523 iRegIorL2I src1, iRegIorL2I src2,
12524 immI src3) %{
12525 match(Set dst (AddI src1 (RShiftI src2 src3)));
12526
12527 ins_cost(1.9 * INSN_COST);
12528 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12529
12530 ins_encode %{
12531 __ addw(as_Register($dst$$reg),
12532 as_Register($src1$$reg),
12533 as_Register($src2$$reg),
12534 Assembler::ASR,
12535 $src3$$constant & 0x1f);
12536 %}
12537
12538 ins_pipe(ialu_reg_reg_shift);
12539 %}
12540
12541 // This pattern is automatically generated from aarch64_ad.m4
12542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12543 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12544 iRegL src1, iRegL src2,
12545 immI src3) %{
12546 match(Set dst (AddL src1 (RShiftL src2 src3)));
12547
12548 ins_cost(1.9 * INSN_COST);
12549 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12550
12551 ins_encode %{
12552 __ add(as_Register($dst$$reg),
12553 as_Register($src1$$reg),
12554 as_Register($src2$$reg),
12555 Assembler::ASR,
12556 $src3$$constant & 0x3f);
12557 %}
12558
12559 ins_pipe(ialu_reg_reg_shift);
12560 %}
12561
12562 // This pattern is automatically generated from aarch64_ad.m4
12563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12564 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12565 iRegIorL2I src1, iRegIorL2I src2,
12566 immI src3) %{
12567 match(Set dst (AddI src1 (LShiftI src2 src3)));
12568
12569 ins_cost(1.9 * INSN_COST);
12570 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12571
12572 ins_encode %{
12573 __ addw(as_Register($dst$$reg),
12574 as_Register($src1$$reg),
12575 as_Register($src2$$reg),
12576 Assembler::LSL,
12577 $src3$$constant & 0x1f);
12578 %}
12579
12580 ins_pipe(ialu_reg_reg_shift);
12581 %}
12582
12583 // This pattern is automatically generated from aarch64_ad.m4
12584 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12585 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12586 iRegL src1, iRegL src2,
12587 immI src3) %{
12588 match(Set dst (AddL src1 (LShiftL src2 src3)));
12589
12590 ins_cost(1.9 * INSN_COST);
12591 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12592
12593 ins_encode %{
12594 __ add(as_Register($dst$$reg),
12595 as_Register($src1$$reg),
12596 as_Register($src2$$reg),
12597 Assembler::LSL,
12598 $src3$$constant & 0x3f);
12599 %}
12600
12601 ins_pipe(ialu_reg_reg_shift);
12602 %}
12603
12604 // This pattern is automatically generated from aarch64_ad.m4
12605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12606 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12607 iRegIorL2I src1, iRegIorL2I src2,
12608 immI src3) %{
12609 match(Set dst (SubI src1 (URShiftI src2 src3)));
12610
12611 ins_cost(1.9 * INSN_COST);
12612 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12613
12614 ins_encode %{
12615 __ subw(as_Register($dst$$reg),
12616 as_Register($src1$$reg),
12617 as_Register($src2$$reg),
12618 Assembler::LSR,
12619 $src3$$constant & 0x1f);
12620 %}
12621
12622 ins_pipe(ialu_reg_reg_shift);
12623 %}
12624
12625 // This pattern is automatically generated from aarch64_ad.m4
12626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12627 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12628 iRegL src1, iRegL src2,
12629 immI src3) %{
12630 match(Set dst (SubL src1 (URShiftL src2 src3)));
12631
12632 ins_cost(1.9 * INSN_COST);
12633 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12634
12635 ins_encode %{
12636 __ sub(as_Register($dst$$reg),
12637 as_Register($src1$$reg),
12638 as_Register($src2$$reg),
12639 Assembler::LSR,
12640 $src3$$constant & 0x3f);
12641 %}
12642
12643 ins_pipe(ialu_reg_reg_shift);
12644 %}
12645
12646 // This pattern is automatically generated from aarch64_ad.m4
12647 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12648 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12649 iRegIorL2I src1, iRegIorL2I src2,
12650 immI src3) %{
12651 match(Set dst (SubI src1 (RShiftI src2 src3)));
12652
12653 ins_cost(1.9 * INSN_COST);
12654 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12655
12656 ins_encode %{
12657 __ subw(as_Register($dst$$reg),
12658 as_Register($src1$$reg),
12659 as_Register($src2$$reg),
12660 Assembler::ASR,
12661 $src3$$constant & 0x1f);
12662 %}
12663
12664 ins_pipe(ialu_reg_reg_shift);
12665 %}
12666
12667 // This pattern is automatically generated from aarch64_ad.m4
12668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12669 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12670 iRegL src1, iRegL src2,
12671 immI src3) %{
12672 match(Set dst (SubL src1 (RShiftL src2 src3)));
12673
12674 ins_cost(1.9 * INSN_COST);
12675 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12676
12677 ins_encode %{
12678 __ sub(as_Register($dst$$reg),
12679 as_Register($src1$$reg),
12680 as_Register($src2$$reg),
12681 Assembler::ASR,
12682 $src3$$constant & 0x3f);
12683 %}
12684
12685 ins_pipe(ialu_reg_reg_shift);
12686 %}
12687
12688 // This pattern is automatically generated from aarch64_ad.m4
12689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12690 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12691 iRegIorL2I src1, iRegIorL2I src2,
12692 immI src3) %{
12693 match(Set dst (SubI src1 (LShiftI src2 src3)));
12694
12695 ins_cost(1.9 * INSN_COST);
12696 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12697
12698 ins_encode %{
12699 __ subw(as_Register($dst$$reg),
12700 as_Register($src1$$reg),
12701 as_Register($src2$$reg),
12702 Assembler::LSL,
12703 $src3$$constant & 0x1f);
12704 %}
12705
12706 ins_pipe(ialu_reg_reg_shift);
12707 %}
12708
12709 // This pattern is automatically generated from aarch64_ad.m4
12710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12711 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12712 iRegL src1, iRegL src2,
12713 immI src3) %{
12714 match(Set dst (SubL src1 (LShiftL src2 src3)));
12715
12716 ins_cost(1.9 * INSN_COST);
12717 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12718
12719 ins_encode %{
12720 __ sub(as_Register($dst$$reg),
12721 as_Register($src1$$reg),
12722 as_Register($src2$$reg),
12723 Assembler::LSL,
12724 $src3$$constant & 0x3f);
12725 %}
12726
12727 ins_pipe(ialu_reg_reg_shift);
12728 %}
12729
12730 // This pattern is automatically generated from aarch64_ad.m4
12731 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12732
12733 // Shift Left followed by Shift Right.
12734 // This idiom is used by the compiler for the i2b bytecode etc.
12735 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12736 %{
12737 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12738 ins_cost(INSN_COST * 2);
12739 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12740 ins_encode %{
12741 int lshift = $lshift_count$$constant & 63;
12742 int rshift = $rshift_count$$constant & 63;
12743 int s = 63 - lshift;
12744 int r = (rshift - lshift) & 63;
12745 __ sbfm(as_Register($dst$$reg),
12746 as_Register($src$$reg),
12747 r, s);
12748 %}
12749
12750 ins_pipe(ialu_reg_shift);
12751 %}
12752
12753 // This pattern is automatically generated from aarch64_ad.m4
12754 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12755
12756 // Shift Left followed by Shift Right.
12757 // This idiom is used by the compiler for the i2b bytecode etc.
12758 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12759 %{
12760 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12761 ins_cost(INSN_COST * 2);
12762 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12763 ins_encode %{
12764 int lshift = $lshift_count$$constant & 31;
12765 int rshift = $rshift_count$$constant & 31;
12766 int s = 31 - lshift;
12767 int r = (rshift - lshift) & 31;
12768 __ sbfmw(as_Register($dst$$reg),
12769 as_Register($src$$reg),
12770 r, s);
12771 %}
12772
12773 ins_pipe(ialu_reg_shift);
12774 %}
12775
12776 // This pattern is automatically generated from aarch64_ad.m4
12777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12778
12779 // Shift Left followed by Shift Right.
12780 // This idiom is used by the compiler for the i2b bytecode etc.
12781 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12782 %{
12783 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12784 ins_cost(INSN_COST * 2);
12785 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12786 ins_encode %{
12787 int lshift = $lshift_count$$constant & 63;
12788 int rshift = $rshift_count$$constant & 63;
12789 int s = 63 - lshift;
12790 int r = (rshift - lshift) & 63;
12791 __ ubfm(as_Register($dst$$reg),
12792 as_Register($src$$reg),
12793 r, s);
12794 %}
12795
12796 ins_pipe(ialu_reg_shift);
12797 %}
12798
12799 // This pattern is automatically generated from aarch64_ad.m4
12800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12801
12802 // Shift Left followed by Shift Right.
12803 // This idiom is used by the compiler for the i2b bytecode etc.
12804 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12805 %{
12806 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12807 ins_cost(INSN_COST * 2);
12808 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12809 ins_encode %{
12810 int lshift = $lshift_count$$constant & 31;
12811 int rshift = $rshift_count$$constant & 31;
12812 int s = 31 - lshift;
12813 int r = (rshift - lshift) & 31;
12814 __ ubfmw(as_Register($dst$$reg),
12815 as_Register($src$$reg),
12816 r, s);
12817 %}
12818
12819 ins_pipe(ialu_reg_shift);
12820 %}
12821
12822 // Bitfield extract with shift & mask
12823
12824 // This pattern is automatically generated from aarch64_ad.m4
12825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12826 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12827 %{
12828 match(Set dst (AndI (URShiftI src rshift) mask));
12829 // Make sure we are not going to exceed what ubfxw can do.
12830 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12831
12832 ins_cost(INSN_COST);
12833 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12834 ins_encode %{
12835 int rshift = $rshift$$constant & 31;
12836 intptr_t mask = $mask$$constant;
12837 int width = exact_log2(mask+1);
12838 __ ubfxw(as_Register($dst$$reg),
12839 as_Register($src$$reg), rshift, width);
12840 %}
12841 ins_pipe(ialu_reg_shift);
12842 %}
12843
12844 // This pattern is automatically generated from aarch64_ad.m4
12845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12846 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12847 %{
12848 match(Set dst (AndL (URShiftL src rshift) mask));
12849 // Make sure we are not going to exceed what ubfx can do.
12850 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12851
12852 ins_cost(INSN_COST);
12853 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12854 ins_encode %{
12855 int rshift = $rshift$$constant & 63;
12856 intptr_t mask = $mask$$constant;
12857 int width = exact_log2_long(mask+1);
12858 __ ubfx(as_Register($dst$$reg),
12859 as_Register($src$$reg), rshift, width);
12860 %}
12861 ins_pipe(ialu_reg_shift);
12862 %}
12863
12864
12865 // This pattern is automatically generated from aarch64_ad.m4
12866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12867
12868 // We can use ubfx when extending an And with a mask when we know mask
12869 // is positive. We know that because immI_bitmask guarantees it.
12870 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12871 %{
12872 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12873 // Make sure we are not going to exceed what ubfxw can do.
12874 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12875
12876 ins_cost(INSN_COST * 2);
12877 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12878 ins_encode %{
12879 int rshift = $rshift$$constant & 31;
12880 intptr_t mask = $mask$$constant;
12881 int width = exact_log2(mask+1);
12882 __ ubfx(as_Register($dst$$reg),
12883 as_Register($src$$reg), rshift, width);
12884 %}
12885 ins_pipe(ialu_reg_shift);
12886 %}
12887
12888
12889 // This pattern is automatically generated from aarch64_ad.m4
12890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12891
12892 // We can use ubfiz when masking by a positive number and then left shifting the result.
12893 // We know that the mask is positive because immI_bitmask guarantees it.
12894 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12895 %{
12896 match(Set dst (LShiftI (AndI src mask) lshift));
12897 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12898
12899 ins_cost(INSN_COST);
12900 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12901 ins_encode %{
12902 int lshift = $lshift$$constant & 31;
12903 intptr_t mask = $mask$$constant;
12904 int width = exact_log2(mask+1);
12905 __ ubfizw(as_Register($dst$$reg),
12906 as_Register($src$$reg), lshift, width);
12907 %}
12908 ins_pipe(ialu_reg_shift);
12909 %}
12910
12911 // This pattern is automatically generated from aarch64_ad.m4
12912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12913
12914 // We can use ubfiz when masking by a positive number and then left shifting the result.
12915 // We know that the mask is positive because immL_bitmask guarantees it.
12916 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12917 %{
12918 match(Set dst (LShiftL (AndL src mask) lshift));
12919 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12920
12921 ins_cost(INSN_COST);
12922 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12923 ins_encode %{
12924 int lshift = $lshift$$constant & 63;
12925 intptr_t mask = $mask$$constant;
12926 int width = exact_log2_long(mask+1);
12927 __ ubfiz(as_Register($dst$$reg),
12928 as_Register($src$$reg), lshift, width);
12929 %}
12930 ins_pipe(ialu_reg_shift);
12931 %}
12932
12933 // This pattern is automatically generated from aarch64_ad.m4
12934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12935
12936 // We can use ubfiz when masking by a positive number and then left shifting the result.
12937 // We know that the mask is positive because immI_bitmask guarantees it.
12938 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12939 %{
12940 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12941 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12942
12943 ins_cost(INSN_COST);
12944 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12945 ins_encode %{
12946 int lshift = $lshift$$constant & 31;
12947 intptr_t mask = $mask$$constant;
12948 int width = exact_log2(mask+1);
12949 __ ubfizw(as_Register($dst$$reg),
12950 as_Register($src$$reg), lshift, width);
12951 %}
12952 ins_pipe(ialu_reg_shift);
12953 %}
12954
12955 // This pattern is automatically generated from aarch64_ad.m4
12956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12957
12958 // We can use ubfiz when masking by a positive number and then left shifting the result.
12959 // We know that the mask is positive because immL_bitmask guarantees it.
12960 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12961 %{
12962 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12963 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12964
12965 ins_cost(INSN_COST);
12966 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12967 ins_encode %{
12968 int lshift = $lshift$$constant & 63;
12969 intptr_t mask = $mask$$constant;
12970 int width = exact_log2_long(mask+1);
12971 __ ubfiz(as_Register($dst$$reg),
12972 as_Register($src$$reg), lshift, width);
12973 %}
12974 ins_pipe(ialu_reg_shift);
12975 %}
12976
12977
12978 // This pattern is automatically generated from aarch64_ad.m4
12979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12980
12981 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12982 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12983 %{
12984 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12985 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12986
12987 ins_cost(INSN_COST);
12988 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12989 ins_encode %{
12990 int lshift = $lshift$$constant & 63;
12991 intptr_t mask = $mask$$constant;
12992 int width = exact_log2(mask+1);
12993 __ ubfiz(as_Register($dst$$reg),
12994 as_Register($src$$reg), lshift, width);
12995 %}
12996 ins_pipe(ialu_reg_shift);
12997 %}
12998
12999 // This pattern is automatically generated from aarch64_ad.m4
13000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13001
13002 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13003 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13004 %{
13005 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13006 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13007
13008 ins_cost(INSN_COST);
13009 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13010 ins_encode %{
13011 int lshift = $lshift$$constant & 31;
13012 intptr_t mask = $mask$$constant;
13013 int width = exact_log2(mask+1);
13014 __ ubfiz(as_Register($dst$$reg),
13015 as_Register($src$$reg), lshift, width);
13016 %}
13017 ins_pipe(ialu_reg_shift);
13018 %}
13019
13020 // This pattern is automatically generated from aarch64_ad.m4
13021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13022
13023 // Can skip int2long conversions after AND with small bitmask
13024 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13025 %{
13026 match(Set dst (ConvI2L (AndI src msk)));
13027 ins_cost(INSN_COST);
13028 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13029 ins_encode %{
13030 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13031 %}
13032 ins_pipe(ialu_reg_shift);
13033 %}
13034
13035
13036 // Rotations
13037 // This pattern is automatically generated from aarch64_ad.m4
13038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13039 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13040 %{
13041 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13042 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13043
13044 ins_cost(INSN_COST);
13045 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13046
13047 ins_encode %{
13048 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13049 $rshift$$constant & 63);
13050 %}
13051 ins_pipe(ialu_reg_reg_extr);
13052 %}
13053
13054
13055 // This pattern is automatically generated from aarch64_ad.m4
13056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13057 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13058 %{
13059 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13060 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13061
13062 ins_cost(INSN_COST);
13063 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13064
13065 ins_encode %{
13066 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13067 $rshift$$constant & 31);
13068 %}
13069 ins_pipe(ialu_reg_reg_extr);
13070 %}
13071
13072
13073 // This pattern is automatically generated from aarch64_ad.m4
13074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13075 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13076 %{
13077 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13078 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13079
13080 ins_cost(INSN_COST);
13081 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13082
13083 ins_encode %{
13084 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13085 $rshift$$constant & 63);
13086 %}
13087 ins_pipe(ialu_reg_reg_extr);
13088 %}
13089
13090
13091 // This pattern is automatically generated from aarch64_ad.m4
13092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13093 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13094 %{
13095 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13096 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13097
13098 ins_cost(INSN_COST);
13099 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13100
13101 ins_encode %{
13102 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13103 $rshift$$constant & 31);
13104 %}
13105 ins_pipe(ialu_reg_reg_extr);
13106 %}
13107
13108
13109 // This pattern is automatically generated from aarch64_ad.m4
13110 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13111 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13112 %{
13113 match(Set dst (RotateRight src shift));
13114
13115 ins_cost(INSN_COST);
13116 format %{ "ror $dst, $src, $shift" %}
13117
13118 ins_encode %{
13119 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13120 $shift$$constant & 0x1f);
13121 %}
13122 ins_pipe(ialu_reg_reg_vshift);
13123 %}
13124
13125 // This pattern is automatically generated from aarch64_ad.m4
13126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13127 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13128 %{
13129 match(Set dst (RotateRight src shift));
13130
13131 ins_cost(INSN_COST);
13132 format %{ "ror $dst, $src, $shift" %}
13133
13134 ins_encode %{
13135 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13136 $shift$$constant & 0x3f);
13137 %}
13138 ins_pipe(ialu_reg_reg_vshift);
13139 %}
13140
13141 // This pattern is automatically generated from aarch64_ad.m4
13142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13143 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13144 %{
13145 match(Set dst (RotateRight src shift));
13146
13147 ins_cost(INSN_COST);
13148 format %{ "ror $dst, $src, $shift" %}
13149
13150 ins_encode %{
13151 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13152 %}
13153 ins_pipe(ialu_reg_reg_vshift);
13154 %}
13155
13156 // This pattern is automatically generated from aarch64_ad.m4
13157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13158 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13159 %{
13160 match(Set dst (RotateRight src shift));
13161
13162 ins_cost(INSN_COST);
13163 format %{ "ror $dst, $src, $shift" %}
13164
13165 ins_encode %{
13166 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13167 %}
13168 ins_pipe(ialu_reg_reg_vshift);
13169 %}
13170
13171 // This pattern is automatically generated from aarch64_ad.m4
13172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13173 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13174 %{
13175 match(Set dst (RotateLeft src shift));
13176
13177 ins_cost(INSN_COST);
13178 format %{ "rol $dst, $src, $shift" %}
13179
13180 ins_encode %{
13181 __ subw(rscratch1, zr, as_Register($shift$$reg));
13182 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13183 %}
13184 ins_pipe(ialu_reg_reg_vshift);
13185 %}
13186
13187 // This pattern is automatically generated from aarch64_ad.m4
13188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13189 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13190 %{
13191 match(Set dst (RotateLeft src shift));
13192
13193 ins_cost(INSN_COST);
13194 format %{ "rol $dst, $src, $shift" %}
13195
13196 ins_encode %{
13197 __ subw(rscratch1, zr, as_Register($shift$$reg));
13198 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13199 %}
13200 ins_pipe(ialu_reg_reg_vshift);
13201 %}
13202
13203
13204 // Add/subtract (extended)
13205
13206 // This pattern is automatically generated from aarch64_ad.m4
13207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13208 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13209 %{
13210 match(Set dst (AddL src1 (ConvI2L src2)));
13211 ins_cost(INSN_COST);
13212 format %{ "add $dst, $src1, $src2, sxtw" %}
13213
13214 ins_encode %{
13215 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13216 as_Register($src2$$reg), ext::sxtw);
13217 %}
13218 ins_pipe(ialu_reg_reg);
13219 %}
13220
13221 // This pattern is automatically generated from aarch64_ad.m4
13222 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13223 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13224 %{
13225 match(Set dst (SubL src1 (ConvI2L src2)));
13226 ins_cost(INSN_COST);
13227 format %{ "sub $dst, $src1, $src2, sxtw" %}
13228
13229 ins_encode %{
13230 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13231 as_Register($src2$$reg), ext::sxtw);
13232 %}
13233 ins_pipe(ialu_reg_reg);
13234 %}
13235
13236 // This pattern is automatically generated from aarch64_ad.m4
13237 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13238 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13239 %{
13240 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13241 ins_cost(INSN_COST);
13242 format %{ "add $dst, $src1, $src2, sxth" %}
13243
13244 ins_encode %{
13245 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13246 as_Register($src2$$reg), ext::sxth);
13247 %}
13248 ins_pipe(ialu_reg_reg);
13249 %}
13250
13251 // This pattern is automatically generated from aarch64_ad.m4
13252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13253 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13254 %{
13255 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13256 ins_cost(INSN_COST);
13257 format %{ "add $dst, $src1, $src2, sxtb" %}
13258
13259 ins_encode %{
13260 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13261 as_Register($src2$$reg), ext::sxtb);
13262 %}
13263 ins_pipe(ialu_reg_reg);
13264 %}
13265
13266 // This pattern is automatically generated from aarch64_ad.m4
13267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13268 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13269 %{
13270 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13271 ins_cost(INSN_COST);
13272 format %{ "add $dst, $src1, $src2, uxtb" %}
13273
13274 ins_encode %{
13275 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13276 as_Register($src2$$reg), ext::uxtb);
13277 %}
13278 ins_pipe(ialu_reg_reg);
13279 %}
13280
13281 // This pattern is automatically generated from aarch64_ad.m4
13282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13283 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13284 %{
13285 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13286 ins_cost(INSN_COST);
13287 format %{ "add $dst, $src1, $src2, sxth" %}
13288
13289 ins_encode %{
13290 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13291 as_Register($src2$$reg), ext::sxth);
13292 %}
13293 ins_pipe(ialu_reg_reg);
13294 %}
13295
13296 // This pattern is automatically generated from aarch64_ad.m4
13297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13298 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13299 %{
13300 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13301 ins_cost(INSN_COST);
13302 format %{ "add $dst, $src1, $src2, sxtw" %}
13303
13304 ins_encode %{
13305 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13306 as_Register($src2$$reg), ext::sxtw);
13307 %}
13308 ins_pipe(ialu_reg_reg);
13309 %}
13310
13311 // This pattern is automatically generated from aarch64_ad.m4
13312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13313 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13314 %{
13315 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13316 ins_cost(INSN_COST);
13317 format %{ "add $dst, $src1, $src2, sxtb" %}
13318
13319 ins_encode %{
13320 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13321 as_Register($src2$$reg), ext::sxtb);
13322 %}
13323 ins_pipe(ialu_reg_reg);
13324 %}
13325
13326 // This pattern is automatically generated from aarch64_ad.m4
13327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13328 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13329 %{
13330 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13331 ins_cost(INSN_COST);
13332 format %{ "add $dst, $src1, $src2, uxtb" %}
13333
13334 ins_encode %{
13335 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13336 as_Register($src2$$reg), ext::uxtb);
13337 %}
13338 ins_pipe(ialu_reg_reg);
13339 %}
13340
13341 // This pattern is automatically generated from aarch64_ad.m4
13342 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13343 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13344 %{
13345 match(Set dst (AddI src1 (AndI src2 mask)));
13346 ins_cost(INSN_COST);
13347 format %{ "addw $dst, $src1, $src2, uxtb" %}
13348
13349 ins_encode %{
13350 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13351 as_Register($src2$$reg), ext::uxtb);
13352 %}
13353 ins_pipe(ialu_reg_reg);
13354 %}
13355
13356 // This pattern is automatically generated from aarch64_ad.m4
13357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13358 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13359 %{
13360 match(Set dst (AddI src1 (AndI src2 mask)));
13361 ins_cost(INSN_COST);
13362 format %{ "addw $dst, $src1, $src2, uxth" %}
13363
13364 ins_encode %{
13365 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13366 as_Register($src2$$reg), ext::uxth);
13367 %}
13368 ins_pipe(ialu_reg_reg);
13369 %}
13370
13371 // This pattern is automatically generated from aarch64_ad.m4
13372 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13373 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13374 %{
13375 match(Set dst (AddL src1 (AndL src2 mask)));
13376 ins_cost(INSN_COST);
13377 format %{ "add $dst, $src1, $src2, uxtb" %}
13378
13379 ins_encode %{
13380 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13381 as_Register($src2$$reg), ext::uxtb);
13382 %}
13383 ins_pipe(ialu_reg_reg);
13384 %}
13385
13386 // This pattern is automatically generated from aarch64_ad.m4
13387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13388 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13389 %{
13390 match(Set dst (AddL src1 (AndL src2 mask)));
13391 ins_cost(INSN_COST);
13392 format %{ "add $dst, $src1, $src2, uxth" %}
13393
13394 ins_encode %{
13395 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13396 as_Register($src2$$reg), ext::uxth);
13397 %}
13398 ins_pipe(ialu_reg_reg);
13399 %}
13400
13401 // This pattern is automatically generated from aarch64_ad.m4
13402 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13403 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13404 %{
13405 match(Set dst (AddL src1 (AndL src2 mask)));
13406 ins_cost(INSN_COST);
13407 format %{ "add $dst, $src1, $src2, uxtw" %}
13408
13409 ins_encode %{
13410 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13411 as_Register($src2$$reg), ext::uxtw);
13412 %}
13413 ins_pipe(ialu_reg_reg);
13414 %}
13415
13416 // This pattern is automatically generated from aarch64_ad.m4
13417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13418 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13419 %{
13420 match(Set dst (SubI src1 (AndI src2 mask)));
13421 ins_cost(INSN_COST);
13422 format %{ "subw $dst, $src1, $src2, uxtb" %}
13423
13424 ins_encode %{
13425 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13426 as_Register($src2$$reg), ext::uxtb);
13427 %}
13428 ins_pipe(ialu_reg_reg);
13429 %}
13430
13431 // This pattern is automatically generated from aarch64_ad.m4
13432 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13433 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13434 %{
13435 match(Set dst (SubI src1 (AndI src2 mask)));
13436 ins_cost(INSN_COST);
13437 format %{ "subw $dst, $src1, $src2, uxth" %}
13438
13439 ins_encode %{
13440 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13441 as_Register($src2$$reg), ext::uxth);
13442 %}
13443 ins_pipe(ialu_reg_reg);
13444 %}
13445
13446 // This pattern is automatically generated from aarch64_ad.m4
13447 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13448 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13449 %{
13450 match(Set dst (SubL src1 (AndL src2 mask)));
13451 ins_cost(INSN_COST);
13452 format %{ "sub $dst, $src1, $src2, uxtb" %}
13453
13454 ins_encode %{
13455 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13456 as_Register($src2$$reg), ext::uxtb);
13457 %}
13458 ins_pipe(ialu_reg_reg);
13459 %}
13460
13461 // This pattern is automatically generated from aarch64_ad.m4
13462 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13463 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13464 %{
13465 match(Set dst (SubL src1 (AndL src2 mask)));
13466 ins_cost(INSN_COST);
13467 format %{ "sub $dst, $src1, $src2, uxth" %}
13468
13469 ins_encode %{
13470 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13471 as_Register($src2$$reg), ext::uxth);
13472 %}
13473 ins_pipe(ialu_reg_reg);
13474 %}
13475
13476 // This pattern is automatically generated from aarch64_ad.m4
13477 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13478 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13479 %{
13480 match(Set dst (SubL src1 (AndL src2 mask)));
13481 ins_cost(INSN_COST);
13482 format %{ "sub $dst, $src1, $src2, uxtw" %}
13483
13484 ins_encode %{
13485 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13486 as_Register($src2$$reg), ext::uxtw);
13487 %}
13488 ins_pipe(ialu_reg_reg);
13489 %}
13490
13491
13492 // This pattern is automatically generated from aarch64_ad.m4
13493 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13494 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13495 %{
13496 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13497 ins_cost(1.9 * INSN_COST);
13498 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13499
13500 ins_encode %{
13501 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13502 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13503 %}
13504 ins_pipe(ialu_reg_reg_shift);
13505 %}
13506
13507 // This pattern is automatically generated from aarch64_ad.m4
13508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13509 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13510 %{
13511 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13512 ins_cost(1.9 * INSN_COST);
13513 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13514
13515 ins_encode %{
13516 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13517 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13518 %}
13519 ins_pipe(ialu_reg_reg_shift);
13520 %}
13521
13522 // This pattern is automatically generated from aarch64_ad.m4
13523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13524 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13525 %{
13526 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13527 ins_cost(1.9 * INSN_COST);
13528 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13529
13530 ins_encode %{
13531 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13532 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13533 %}
13534 ins_pipe(ialu_reg_reg_shift);
13535 %}
13536
13537 // This pattern is automatically generated from aarch64_ad.m4
13538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13539 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13540 %{
13541 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13542 ins_cost(1.9 * INSN_COST);
13543 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13544
13545 ins_encode %{
13546 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13547 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13548 %}
13549 ins_pipe(ialu_reg_reg_shift);
13550 %}
13551
13552 // This pattern is automatically generated from aarch64_ad.m4
13553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13554 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13555 %{
13556 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13557 ins_cost(1.9 * INSN_COST);
13558 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13559
13560 ins_encode %{
13561 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13562 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13563 %}
13564 ins_pipe(ialu_reg_reg_shift);
13565 %}
13566
13567 // This pattern is automatically generated from aarch64_ad.m4
13568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13569 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13570 %{
13571 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13572 ins_cost(1.9 * INSN_COST);
13573 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13574
13575 ins_encode %{
13576 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13577 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13578 %}
13579 ins_pipe(ialu_reg_reg_shift);
13580 %}
13581
13582 // This pattern is automatically generated from aarch64_ad.m4
13583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13584 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13585 %{
13586 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13587 ins_cost(1.9 * INSN_COST);
13588 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13589
13590 ins_encode %{
13591 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13592 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13593 %}
13594 ins_pipe(ialu_reg_reg_shift);
13595 %}
13596
13597 // This pattern is automatically generated from aarch64_ad.m4
13598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13599 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13600 %{
13601 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13602 ins_cost(1.9 * INSN_COST);
13603 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13604
13605 ins_encode %{
13606 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13607 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13608 %}
13609 ins_pipe(ialu_reg_reg_shift);
13610 %}
13611
13612 // This pattern is automatically generated from aarch64_ad.m4
13613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13614 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13615 %{
13616 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13617 ins_cost(1.9 * INSN_COST);
13618 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13619
13620 ins_encode %{
13621 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13622 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13623 %}
13624 ins_pipe(ialu_reg_reg_shift);
13625 %}
13626
13627 // This pattern is automatically generated from aarch64_ad.m4
13628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13629 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13630 %{
13631 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13632 ins_cost(1.9 * INSN_COST);
13633 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13634
13635 ins_encode %{
13636 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13637 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13638 %}
13639 ins_pipe(ialu_reg_reg_shift);
13640 %}
13641
13642 // This pattern is automatically generated from aarch64_ad.m4
13643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13644 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13645 %{
13646 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13647 ins_cost(1.9 * INSN_COST);
13648 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13649
13650 ins_encode %{
13651 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13652 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13653 %}
13654 ins_pipe(ialu_reg_reg_shift);
13655 %}
13656
13657 // This pattern is automatically generated from aarch64_ad.m4
13658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13659 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13660 %{
13661 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13662 ins_cost(1.9 * INSN_COST);
13663 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13664
13665 ins_encode %{
13666 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13667 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13668 %}
13669 ins_pipe(ialu_reg_reg_shift);
13670 %}
13671
13672 // This pattern is automatically generated from aarch64_ad.m4
13673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13674 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13675 %{
13676 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13677 ins_cost(1.9 * INSN_COST);
13678 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13679
13680 ins_encode %{
13681 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13682 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13683 %}
13684 ins_pipe(ialu_reg_reg_shift);
13685 %}
13686
13687 // This pattern is automatically generated from aarch64_ad.m4
13688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13689 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13690 %{
13691 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13692 ins_cost(1.9 * INSN_COST);
13693 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13694
13695 ins_encode %{
13696 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13697 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13698 %}
13699 ins_pipe(ialu_reg_reg_shift);
13700 %}
13701
13702 // This pattern is automatically generated from aarch64_ad.m4
13703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13704 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13705 %{
13706 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13707 ins_cost(1.9 * INSN_COST);
13708 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13709
13710 ins_encode %{
13711 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13712 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13713 %}
13714 ins_pipe(ialu_reg_reg_shift);
13715 %}
13716
13717 // This pattern is automatically generated from aarch64_ad.m4
13718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13719 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13720 %{
13721 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13722 ins_cost(1.9 * INSN_COST);
13723 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13724
13725 ins_encode %{
13726 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13727 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13728 %}
13729 ins_pipe(ialu_reg_reg_shift);
13730 %}
13731
13732 // This pattern is automatically generated from aarch64_ad.m4
13733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13734 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13735 %{
13736 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13737 ins_cost(1.9 * INSN_COST);
13738 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13739
13740 ins_encode %{
13741 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13742 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13743 %}
13744 ins_pipe(ialu_reg_reg_shift);
13745 %}
13746
13747 // This pattern is automatically generated from aarch64_ad.m4
13748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13749 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13750 %{
13751 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13752 ins_cost(1.9 * INSN_COST);
13753 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13754
13755 ins_encode %{
13756 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13757 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13758 %}
13759 ins_pipe(ialu_reg_reg_shift);
13760 %}
13761
13762 // This pattern is automatically generated from aarch64_ad.m4
13763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13764 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13765 %{
13766 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13767 ins_cost(1.9 * INSN_COST);
13768 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13769
13770 ins_encode %{
13771 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13772 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13773 %}
13774 ins_pipe(ialu_reg_reg_shift);
13775 %}
13776
13777 // This pattern is automatically generated from aarch64_ad.m4
13778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13779 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13780 %{
13781 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13782 ins_cost(1.9 * INSN_COST);
13783 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13784
13785 ins_encode %{
13786 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13787 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13788 %}
13789 ins_pipe(ialu_reg_reg_shift);
13790 %}
13791
13792 // This pattern is automatically generated from aarch64_ad.m4
13793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13794 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13795 %{
13796 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13797 ins_cost(1.9 * INSN_COST);
13798 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13799
13800 ins_encode %{
13801 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13802 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13803 %}
13804 ins_pipe(ialu_reg_reg_shift);
13805 %}
13806
13807 // This pattern is automatically generated from aarch64_ad.m4
13808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13809 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13810 %{
13811 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13812 ins_cost(1.9 * INSN_COST);
13813 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13814
13815 ins_encode %{
13816 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13817 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13818 %}
13819 ins_pipe(ialu_reg_reg_shift);
13820 %}
13821
13822
13823
13824 // END This section of the file is automatically generated. Do not edit --------------
13825
13826
13827 // ============================================================================
13828 // Floating Point Arithmetic Instructions
13829
13830 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13831 match(Set dst (AddF src1 src2));
13832
13833 ins_cost(INSN_COST * 5);
13834 format %{ "fadds $dst, $src1, $src2" %}
13835
13836 ins_encode %{
13837 __ fadds(as_FloatRegister($dst$$reg),
13838 as_FloatRegister($src1$$reg),
13839 as_FloatRegister($src2$$reg));
13840 %}
13841
13842 ins_pipe(fp_dop_reg_reg_s);
13843 %}
13844
13845 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13846 match(Set dst (AddD src1 src2));
13847
13848 ins_cost(INSN_COST * 5);
13849 format %{ "faddd $dst, $src1, $src2" %}
13850
13851 ins_encode %{
13852 __ faddd(as_FloatRegister($dst$$reg),
13853 as_FloatRegister($src1$$reg),
13854 as_FloatRegister($src2$$reg));
13855 %}
13856
13857 ins_pipe(fp_dop_reg_reg_d);
13858 %}
13859
13860 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13861 match(Set dst (SubF src1 src2));
13862
13863 ins_cost(INSN_COST * 5);
13864 format %{ "fsubs $dst, $src1, $src2" %}
13865
13866 ins_encode %{
13867 __ fsubs(as_FloatRegister($dst$$reg),
13868 as_FloatRegister($src1$$reg),
13869 as_FloatRegister($src2$$reg));
13870 %}
13871
13872 ins_pipe(fp_dop_reg_reg_s);
13873 %}
13874
13875 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13876 match(Set dst (SubD src1 src2));
13877
13878 ins_cost(INSN_COST * 5);
13879 format %{ "fsubd $dst, $src1, $src2" %}
13880
13881 ins_encode %{
13882 __ fsubd(as_FloatRegister($dst$$reg),
13883 as_FloatRegister($src1$$reg),
13884 as_FloatRegister($src2$$reg));
13885 %}
13886
13887 ins_pipe(fp_dop_reg_reg_d);
13888 %}
13889
13890 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13891 match(Set dst (MulF src1 src2));
13892
13893 ins_cost(INSN_COST * 6);
13894 format %{ "fmuls $dst, $src1, $src2" %}
13895
13896 ins_encode %{
13897 __ fmuls(as_FloatRegister($dst$$reg),
13898 as_FloatRegister($src1$$reg),
13899 as_FloatRegister($src2$$reg));
13900 %}
13901
13902 ins_pipe(fp_dop_reg_reg_s);
13903 %}
13904
13905 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13906 match(Set dst (MulD src1 src2));
13907
13908 ins_cost(INSN_COST * 6);
13909 format %{ "fmuld $dst, $src1, $src2" %}
13910
13911 ins_encode %{
13912 __ fmuld(as_FloatRegister($dst$$reg),
13913 as_FloatRegister($src1$$reg),
13914 as_FloatRegister($src2$$reg));
13915 %}
13916
13917 ins_pipe(fp_dop_reg_reg_d);
13918 %}
13919
13920 // src1 * src2 + src3
13921 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13922 predicate(UseFMA);
13923 match(Set dst (FmaF src3 (Binary src1 src2)));
13924
13925 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13926
13927 ins_encode %{
13928 __ fmadds(as_FloatRegister($dst$$reg),
13929 as_FloatRegister($src1$$reg),
13930 as_FloatRegister($src2$$reg),
13931 as_FloatRegister($src3$$reg));
13932 %}
13933
13934 ins_pipe(pipe_class_default);
13935 %}
13936
13937 // src1 * src2 + src3
13938 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13939 predicate(UseFMA);
13940 match(Set dst (FmaD src3 (Binary src1 src2)));
13941
13942 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13943
13944 ins_encode %{
13945 __ fmaddd(as_FloatRegister($dst$$reg),
13946 as_FloatRegister($src1$$reg),
13947 as_FloatRegister($src2$$reg),
13948 as_FloatRegister($src3$$reg));
13949 %}
13950
13951 ins_pipe(pipe_class_default);
13952 %}
13953
13954 // -src1 * src2 + src3
13955 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13956 predicate(UseFMA);
13957 match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
13958 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13959
13960 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13961
13962 ins_encode %{
13963 __ fmsubs(as_FloatRegister($dst$$reg),
13964 as_FloatRegister($src1$$reg),
13965 as_FloatRegister($src2$$reg),
13966 as_FloatRegister($src3$$reg));
13967 %}
13968
13969 ins_pipe(pipe_class_default);
13970 %}
13971
13972 // -src1 * src2 + src3
13973 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13974 predicate(UseFMA);
13975 match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
13976 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13977
13978 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13979
13980 ins_encode %{
13981 __ fmsubd(as_FloatRegister($dst$$reg),
13982 as_FloatRegister($src1$$reg),
13983 as_FloatRegister($src2$$reg),
13984 as_FloatRegister($src3$$reg));
13985 %}
13986
13987 ins_pipe(pipe_class_default);
13988 %}
13989
13990 // -src1 * src2 - src3
13991 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13992 predicate(UseFMA);
13993 match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
13994 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13995
13996 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13997
13998 ins_encode %{
13999 __ fnmadds(as_FloatRegister($dst$$reg),
14000 as_FloatRegister($src1$$reg),
14001 as_FloatRegister($src2$$reg),
14002 as_FloatRegister($src3$$reg));
14003 %}
14004
14005 ins_pipe(pipe_class_default);
14006 %}
14007
14008 // -src1 * src2 - src3
14009 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14010 predicate(UseFMA);
14011 match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14012 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14013
14014 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14015
14016 ins_encode %{
14017 __ fnmaddd(as_FloatRegister($dst$$reg),
14018 as_FloatRegister($src1$$reg),
14019 as_FloatRegister($src2$$reg),
14020 as_FloatRegister($src3$$reg));
14021 %}
14022
14023 ins_pipe(pipe_class_default);
14024 %}
14025
14026 // src1 * src2 - src3
14027 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14028 predicate(UseFMA);
14029 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14030
14031 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14032
14033 ins_encode %{
14034 __ fnmsubs(as_FloatRegister($dst$$reg),
14035 as_FloatRegister($src1$$reg),
14036 as_FloatRegister($src2$$reg),
14037 as_FloatRegister($src3$$reg));
14038 %}
14039
14040 ins_pipe(pipe_class_default);
14041 %}
14042
14043 // src1 * src2 - src3
14044 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14045 predicate(UseFMA);
14046 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14047
14048 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14049
14050 ins_encode %{
14051 // n.b. insn name should be fnmsubd
14052 __ fnmsub(as_FloatRegister($dst$$reg),
14053 as_FloatRegister($src1$$reg),
14054 as_FloatRegister($src2$$reg),
14055 as_FloatRegister($src3$$reg));
14056 %}
14057
14058 ins_pipe(pipe_class_default);
14059 %}
14060
14061
14062 // Math.max(FF)F
14063 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14064 match(Set dst (MaxF src1 src2));
14065
14066 format %{ "fmaxs $dst, $src1, $src2" %}
14067 ins_encode %{
14068 __ fmaxs(as_FloatRegister($dst$$reg),
14069 as_FloatRegister($src1$$reg),
14070 as_FloatRegister($src2$$reg));
14071 %}
14072
14073 ins_pipe(fp_dop_reg_reg_s);
14074 %}
14075
14076 // Math.min(FF)F
14077 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14078 match(Set dst (MinF src1 src2));
14079
14080 format %{ "fmins $dst, $src1, $src2" %}
14081 ins_encode %{
14082 __ fmins(as_FloatRegister($dst$$reg),
14083 as_FloatRegister($src1$$reg),
14084 as_FloatRegister($src2$$reg));
14085 %}
14086
14087 ins_pipe(fp_dop_reg_reg_s);
14088 %}
14089
14090 // Math.max(DD)D
14091 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14092 match(Set dst (MaxD src1 src2));
14093
14094 format %{ "fmaxd $dst, $src1, $src2" %}
14095 ins_encode %{
14096 __ fmaxd(as_FloatRegister($dst$$reg),
14097 as_FloatRegister($src1$$reg),
14098 as_FloatRegister($src2$$reg));
14099 %}
14100
14101 ins_pipe(fp_dop_reg_reg_d);
14102 %}
14103
14104 // Math.min(DD)D
14105 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14106 match(Set dst (MinD src1 src2));
14107
14108 format %{ "fmind $dst, $src1, $src2" %}
14109 ins_encode %{
14110 __ fmind(as_FloatRegister($dst$$reg),
14111 as_FloatRegister($src1$$reg),
14112 as_FloatRegister($src2$$reg));
14113 %}
14114
14115 ins_pipe(fp_dop_reg_reg_d);
14116 %}
14117
14118
14119 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14120 match(Set dst (DivF src1 src2));
14121
14122 ins_cost(INSN_COST * 18);
14123 format %{ "fdivs $dst, $src1, $src2" %}
14124
14125 ins_encode %{
14126 __ fdivs(as_FloatRegister($dst$$reg),
14127 as_FloatRegister($src1$$reg),
14128 as_FloatRegister($src2$$reg));
14129 %}
14130
14131 ins_pipe(fp_div_s);
14132 %}
14133
14134 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14135 match(Set dst (DivD src1 src2));
14136
14137 ins_cost(INSN_COST * 32);
14138 format %{ "fdivd $dst, $src1, $src2" %}
14139
14140 ins_encode %{
14141 __ fdivd(as_FloatRegister($dst$$reg),
14142 as_FloatRegister($src1$$reg),
14143 as_FloatRegister($src2$$reg));
14144 %}
14145
14146 ins_pipe(fp_div_d);
14147 %}
14148
14149 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14150 match(Set dst (NegF src));
14151
14152 ins_cost(INSN_COST * 3);
14153 format %{ "fneg $dst, $src" %}
14154
14155 ins_encode %{
14156 __ fnegs(as_FloatRegister($dst$$reg),
14157 as_FloatRegister($src$$reg));
14158 %}
14159
14160 ins_pipe(fp_uop_s);
14161 %}
14162
14163 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14164 match(Set dst (NegD src));
14165
14166 ins_cost(INSN_COST * 3);
14167 format %{ "fnegd $dst, $src" %}
14168
14169 ins_encode %{
14170 __ fnegd(as_FloatRegister($dst$$reg),
14171 as_FloatRegister($src$$reg));
14172 %}
14173
14174 ins_pipe(fp_uop_d);
14175 %}
14176
14177 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14178 %{
14179 match(Set dst (AbsI src));
14180
14181 effect(KILL cr);
14182 ins_cost(INSN_COST * 2);
14183 format %{ "cmpw $src, zr\n\t"
14184 "cnegw $dst, $src, Assembler::LT\t# int abs"
14185 %}
14186
14187 ins_encode %{
14188 __ cmpw(as_Register($src$$reg), zr);
14189 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14190 %}
14191 ins_pipe(pipe_class_default);
14192 %}
14193
14194 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14195 %{
14196 match(Set dst (AbsL src));
14197
14198 effect(KILL cr);
14199 ins_cost(INSN_COST * 2);
14200 format %{ "cmp $src, zr\n\t"
14201 "cneg $dst, $src, Assembler::LT\t# long abs"
14202 %}
14203
14204 ins_encode %{
14205 __ cmp(as_Register($src$$reg), zr);
14206 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14207 %}
14208 ins_pipe(pipe_class_default);
14209 %}
14210
14211 instruct absF_reg(vRegF dst, vRegF src) %{
14212 match(Set dst (AbsF src));
14213
14214 ins_cost(INSN_COST * 3);
14215 format %{ "fabss $dst, $src" %}
14216 ins_encode %{
14217 __ fabss(as_FloatRegister($dst$$reg),
14218 as_FloatRegister($src$$reg));
14219 %}
14220
14221 ins_pipe(fp_uop_s);
14222 %}
14223
14224 instruct absD_reg(vRegD dst, vRegD src) %{
14225 match(Set dst (AbsD src));
14226
14227 ins_cost(INSN_COST * 3);
14228 format %{ "fabsd $dst, $src" %}
14229 ins_encode %{
14230 __ fabsd(as_FloatRegister($dst$$reg),
14231 as_FloatRegister($src$$reg));
14232 %}
14233
14234 ins_pipe(fp_uop_d);
14235 %}
14236
14237 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14238 match(Set dst (AbsF (SubF src1 src2)));
14239
14240 ins_cost(INSN_COST * 3);
14241 format %{ "fabds $dst, $src1, $src2" %}
14242 ins_encode %{
14243 __ fabds(as_FloatRegister($dst$$reg),
14244 as_FloatRegister($src1$$reg),
14245 as_FloatRegister($src2$$reg));
14246 %}
14247
14248 ins_pipe(fp_uop_s);
14249 %}
14250
14251 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14252 match(Set dst (AbsD (SubD src1 src2)));
14253
14254 ins_cost(INSN_COST * 3);
14255 format %{ "fabdd $dst, $src1, $src2" %}
14256 ins_encode %{
14257 __ fabdd(as_FloatRegister($dst$$reg),
14258 as_FloatRegister($src1$$reg),
14259 as_FloatRegister($src2$$reg));
14260 %}
14261
14262 ins_pipe(fp_uop_d);
14263 %}
14264
14265 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14266 match(Set dst (SqrtD src));
14267
14268 ins_cost(INSN_COST * 50);
14269 format %{ "fsqrtd $dst, $src" %}
14270 ins_encode %{
14271 __ fsqrtd(as_FloatRegister($dst$$reg),
14272 as_FloatRegister($src$$reg));
14273 %}
14274
14275 ins_pipe(fp_div_s);
14276 %}
14277
14278 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14279 match(Set dst (SqrtF src));
14280
14281 ins_cost(INSN_COST * 50);
14282 format %{ "fsqrts $dst, $src" %}
14283 ins_encode %{
14284 __ fsqrts(as_FloatRegister($dst$$reg),
14285 as_FloatRegister($src$$reg));
14286 %}
14287
14288 ins_pipe(fp_div_d);
14289 %}
14290
14291 // Math.rint, floor, ceil
14292 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14293 match(Set dst (RoundDoubleMode src rmode));
14294 format %{ "frint $dst, $src, $rmode" %}
14295 ins_encode %{
14296 switch ($rmode$$constant) {
14297 case RoundDoubleModeNode::rmode_rint:
14298 __ frintnd(as_FloatRegister($dst$$reg),
14299 as_FloatRegister($src$$reg));
14300 break;
14301 case RoundDoubleModeNode::rmode_floor:
14302 __ frintmd(as_FloatRegister($dst$$reg),
14303 as_FloatRegister($src$$reg));
14304 break;
14305 case RoundDoubleModeNode::rmode_ceil:
14306 __ frintpd(as_FloatRegister($dst$$reg),
14307 as_FloatRegister($src$$reg));
14308 break;
14309 }
14310 %}
14311 ins_pipe(fp_uop_d);
14312 %}
14313
14314 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14315 match(Set dst (CopySignD src1 (Binary src2 zero)));
14316 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14317 format %{ "CopySignD $dst $src1 $src2" %}
14318 ins_encode %{
14319 FloatRegister dst = as_FloatRegister($dst$$reg),
14320 src1 = as_FloatRegister($src1$$reg),
14321 src2 = as_FloatRegister($src2$$reg),
14322 zero = as_FloatRegister($zero$$reg);
14323 __ fnegd(dst, zero);
14324 __ bsl(dst, __ T8B, src2, src1);
14325 %}
14326 ins_pipe(fp_uop_d);
14327 %}
14328
14329 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14330 match(Set dst (CopySignF src1 src2));
14331 effect(TEMP_DEF dst, USE src1, USE src2);
14332 format %{ "CopySignF $dst $src1 $src2" %}
14333 ins_encode %{
14334 FloatRegister dst = as_FloatRegister($dst$$reg),
14335 src1 = as_FloatRegister($src1$$reg),
14336 src2 = as_FloatRegister($src2$$reg);
14337 __ movi(dst, __ T2S, 0x80, 24);
14338 __ bsl(dst, __ T8B, src2, src1);
14339 %}
14340 ins_pipe(fp_uop_d);
14341 %}
14342
14343 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14344 match(Set dst (SignumD src (Binary zero one)));
14345 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14346 format %{ "signumD $dst, $src" %}
14347 ins_encode %{
14348 FloatRegister src = as_FloatRegister($src$$reg),
14349 dst = as_FloatRegister($dst$$reg),
14350 zero = as_FloatRegister($zero$$reg),
14351 one = as_FloatRegister($one$$reg);
14352 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14353 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14354 // Bit selection instruction gets bit from "one" for each enabled bit in
14355 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14356 // NaN the whole "src" will be copied because "dst" is zero. For all other
14357 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14358 // from "src", and all other bits are copied from 1.0.
14359 __ bsl(dst, __ T8B, one, src);
14360 %}
14361 ins_pipe(fp_uop_d);
14362 %}
14363
14364 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14365 match(Set dst (SignumF src (Binary zero one)));
14366 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14367 format %{ "signumF $dst, $src" %}
14368 ins_encode %{
14369 FloatRegister src = as_FloatRegister($src$$reg),
14370 dst = as_FloatRegister($dst$$reg),
14371 zero = as_FloatRegister($zero$$reg),
14372 one = as_FloatRegister($one$$reg);
14373 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14374 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14375 // Bit selection instruction gets bit from "one" for each enabled bit in
14376 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14377 // NaN the whole "src" will be copied because "dst" is zero. For all other
14378 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14379 // from "src", and all other bits are copied from 1.0.
14380 __ bsl(dst, __ T8B, one, src);
14381 %}
14382 ins_pipe(fp_uop_d);
14383 %}
14384
14385 instruct onspinwait() %{
14386 match(OnSpinWait);
14387 ins_cost(INSN_COST);
14388
14389 format %{ "onspinwait" %}
14390
14391 ins_encode %{
14392 __ spin_wait();
14393 %}
14394 ins_pipe(pipe_class_empty);
14395 %}
14396
14397 // ============================================================================
14398 // Logical Instructions
14399
14400 // Integer Logical Instructions
14401
14402 // And Instructions
14403
14404
14405 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14406 match(Set dst (AndI src1 src2));
14407
14408 format %{ "andw $dst, $src1, $src2\t# int" %}
14409
14410 ins_cost(INSN_COST);
14411 ins_encode %{
14412 __ andw(as_Register($dst$$reg),
14413 as_Register($src1$$reg),
14414 as_Register($src2$$reg));
14415 %}
14416
14417 ins_pipe(ialu_reg_reg);
14418 %}
14419
14420 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14421 match(Set dst (AndI src1 src2));
14422
14423 format %{ "andsw $dst, $src1, $src2\t# int" %}
14424
14425 ins_cost(INSN_COST);
14426 ins_encode %{
14427 __ andw(as_Register($dst$$reg),
14428 as_Register($src1$$reg),
14429 (uint64_t)($src2$$constant));
14430 %}
14431
14432 ins_pipe(ialu_reg_imm);
14433 %}
14434
14435 // Or Instructions
14436
14437 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14438 match(Set dst (OrI src1 src2));
14439
14440 format %{ "orrw $dst, $src1, $src2\t# int" %}
14441
14442 ins_cost(INSN_COST);
14443 ins_encode %{
14444 __ orrw(as_Register($dst$$reg),
14445 as_Register($src1$$reg),
14446 as_Register($src2$$reg));
14447 %}
14448
14449 ins_pipe(ialu_reg_reg);
14450 %}
14451
14452 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14453 match(Set dst (OrI src1 src2));
14454
14455 format %{ "orrw $dst, $src1, $src2\t# int" %}
14456
14457 ins_cost(INSN_COST);
14458 ins_encode %{
14459 __ orrw(as_Register($dst$$reg),
14460 as_Register($src1$$reg),
14461 (uint64_t)($src2$$constant));
14462 %}
14463
14464 ins_pipe(ialu_reg_imm);
14465 %}
14466
14467 // Xor Instructions
14468
14469 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14470 match(Set dst (XorI src1 src2));
14471
14472 format %{ "eorw $dst, $src1, $src2\t# int" %}
14473
14474 ins_cost(INSN_COST);
14475 ins_encode %{
14476 __ eorw(as_Register($dst$$reg),
14477 as_Register($src1$$reg),
14478 as_Register($src2$$reg));
14479 %}
14480
14481 ins_pipe(ialu_reg_reg);
14482 %}
14483
14484 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14485 match(Set dst (XorI src1 src2));
14486
14487 format %{ "eorw $dst, $src1, $src2\t# int" %}
14488
14489 ins_cost(INSN_COST);
14490 ins_encode %{
14491 __ eorw(as_Register($dst$$reg),
14492 as_Register($src1$$reg),
14493 (uint64_t)($src2$$constant));
14494 %}
14495
14496 ins_pipe(ialu_reg_imm);
14497 %}
14498
14499 // Long Logical Instructions
14500 // TODO
14501
14502 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14503 match(Set dst (AndL src1 src2));
14504
14505 format %{ "and $dst, $src1, $src2\t# int" %}
14506
14507 ins_cost(INSN_COST);
14508 ins_encode %{
14509 __ andr(as_Register($dst$$reg),
14510 as_Register($src1$$reg),
14511 as_Register($src2$$reg));
14512 %}
14513
14514 ins_pipe(ialu_reg_reg);
14515 %}
14516
14517 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14518 match(Set dst (AndL src1 src2));
14519
14520 format %{ "and $dst, $src1, $src2\t# int" %}
14521
14522 ins_cost(INSN_COST);
14523 ins_encode %{
14524 __ andr(as_Register($dst$$reg),
14525 as_Register($src1$$reg),
14526 (uint64_t)($src2$$constant));
14527 %}
14528
14529 ins_pipe(ialu_reg_imm);
14530 %}
14531
14532 // Or Instructions
14533
14534 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14535 match(Set dst (OrL src1 src2));
14536
14537 format %{ "orr $dst, $src1, $src2\t# int" %}
14538
14539 ins_cost(INSN_COST);
14540 ins_encode %{
14541 __ orr(as_Register($dst$$reg),
14542 as_Register($src1$$reg),
14543 as_Register($src2$$reg));
14544 %}
14545
14546 ins_pipe(ialu_reg_reg);
14547 %}
14548
14549 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14550 match(Set dst (OrL src1 src2));
14551
14552 format %{ "orr $dst, $src1, $src2\t# int" %}
14553
14554 ins_cost(INSN_COST);
14555 ins_encode %{
14556 __ orr(as_Register($dst$$reg),
14557 as_Register($src1$$reg),
14558 (uint64_t)($src2$$constant));
14559 %}
14560
14561 ins_pipe(ialu_reg_imm);
14562 %}
14563
14564 // Xor Instructions
14565
14566 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14567 match(Set dst (XorL src1 src2));
14568
14569 format %{ "eor $dst, $src1, $src2\t# int" %}
14570
14571 ins_cost(INSN_COST);
14572 ins_encode %{
14573 __ eor(as_Register($dst$$reg),
14574 as_Register($src1$$reg),
14575 as_Register($src2$$reg));
14576 %}
14577
14578 ins_pipe(ialu_reg_reg);
14579 %}
14580
14581 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14582 match(Set dst (XorL src1 src2));
14583
14584 ins_cost(INSN_COST);
14585 format %{ "eor $dst, $src1, $src2\t# int" %}
14586
14587 ins_encode %{
14588 __ eor(as_Register($dst$$reg),
14589 as_Register($src1$$reg),
14590 (uint64_t)($src2$$constant));
14591 %}
14592
14593 ins_pipe(ialu_reg_imm);
14594 %}
14595
14596 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14597 %{
14598 match(Set dst (ConvI2L src));
14599
14600 ins_cost(INSN_COST);
14601 format %{ "sxtw $dst, $src\t# i2l" %}
14602 ins_encode %{
14603 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14604 %}
14605 ins_pipe(ialu_reg_shift);
14606 %}
14607
14608 // this pattern occurs in bigmath arithmetic
14609 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14610 %{
14611 match(Set dst (AndL (ConvI2L src) mask));
14612
14613 ins_cost(INSN_COST);
14614 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14615 ins_encode %{
14616 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14617 %}
14618
14619 ins_pipe(ialu_reg_shift);
14620 %}
14621
14622 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14623 match(Set dst (ConvL2I src));
14624
14625 ins_cost(INSN_COST);
14626 format %{ "movw $dst, $src \t// l2i" %}
14627
14628 ins_encode %{
14629 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14630 %}
14631
14632 ins_pipe(ialu_reg);
14633 %}
14634
14635 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14636 %{
14637 match(Set dst (Conv2B src));
14638 effect(KILL cr);
14639
14640 format %{
14641 "cmpw $src, zr\n\t"
14642 "cset $dst, ne"
14643 %}
14644
14645 ins_encode %{
14646 __ cmpw(as_Register($src$$reg), zr);
14647 __ cset(as_Register($dst$$reg), Assembler::NE);
14648 %}
14649
14650 ins_pipe(ialu_reg);
14651 %}
14652
14653 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14654 %{
14655 match(Set dst (Conv2B src));
14656 effect(KILL cr);
14657
14658 format %{
14659 "cmp $src, zr\n\t"
14660 "cset $dst, ne"
14661 %}
14662
14663 ins_encode %{
14664 __ cmp(as_Register($src$$reg), zr);
14665 __ cset(as_Register($dst$$reg), Assembler::NE);
14666 %}
14667
14668 ins_pipe(ialu_reg);
14669 %}
14670
14671 instruct convD2F_reg(vRegF dst, vRegD src) %{
14672 match(Set dst (ConvD2F src));
14673
14674 ins_cost(INSN_COST * 5);
14675 format %{ "fcvtd $dst, $src \t// d2f" %}
14676
14677 ins_encode %{
14678 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14679 %}
14680
14681 ins_pipe(fp_d2f);
14682 %}
14683
14684 instruct convF2D_reg(vRegD dst, vRegF src) %{
14685 match(Set dst (ConvF2D src));
14686
14687 ins_cost(INSN_COST * 5);
14688 format %{ "fcvts $dst, $src \t// f2d" %}
14689
14690 ins_encode %{
14691 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14692 %}
14693
14694 ins_pipe(fp_f2d);
14695 %}
14696
14697 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14698 match(Set dst (ConvF2I src));
14699
14700 ins_cost(INSN_COST * 5);
14701 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14702
14703 ins_encode %{
14704 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14705 %}
14706
14707 ins_pipe(fp_f2i);
14708 %}
14709
14710 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14711 match(Set dst (ConvF2L src));
14712
14713 ins_cost(INSN_COST * 5);
14714 format %{ "fcvtzs $dst, $src \t// f2l" %}
14715
14716 ins_encode %{
14717 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14718 %}
14719
14720 ins_pipe(fp_f2l);
14721 %}
14722
14723 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14724 match(Set dst (ConvF2HF src));
14725 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14726 "smov $dst, $tmp\t# move result from $tmp to $dst"
14727 %}
14728 effect(TEMP tmp);
14729 ins_encode %{
14730 __ fcvtsh($tmp$$FloatRegister, $src$$FloatRegister);
14731 __ smov($dst$$Register, $tmp$$FloatRegister, __ H, 0);
14732 %}
14733 ins_pipe(pipe_slow);
14734 %}
14735
14736 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14737 match(Set dst (ConvHF2F src));
14738 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14739 "fcvt $dst, $tmp\t# convert half to single precision"
14740 %}
14741 effect(TEMP tmp);
14742 ins_encode %{
14743 __ mov($tmp$$FloatRegister, __ H, 0, $src$$Register);
14744 __ fcvths($dst$$FloatRegister, $tmp$$FloatRegister);
14745 %}
14746 ins_pipe(pipe_slow);
14747 %}
14748
14749 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14750 match(Set dst (ConvI2F src));
14751
14752 ins_cost(INSN_COST * 5);
14753 format %{ "scvtfws $dst, $src \t// i2f" %}
14754
14755 ins_encode %{
14756 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14757 %}
14758
14759 ins_pipe(fp_i2f);
14760 %}
14761
14762 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14763 match(Set dst (ConvL2F src));
14764
14765 ins_cost(INSN_COST * 5);
14766 format %{ "scvtfs $dst, $src \t// l2f" %}
14767
14768 ins_encode %{
14769 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14770 %}
14771
14772 ins_pipe(fp_l2f);
14773 %}
14774
14775 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14776 match(Set dst (ConvD2I src));
14777
14778 ins_cost(INSN_COST * 5);
14779 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14780
14781 ins_encode %{
14782 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14783 %}
14784
14785 ins_pipe(fp_d2i);
14786 %}
14787
14788 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14789 match(Set dst (ConvD2L src));
14790
14791 ins_cost(INSN_COST * 5);
14792 format %{ "fcvtzd $dst, $src \t// d2l" %}
14793
14794 ins_encode %{
14795 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14796 %}
14797
14798 ins_pipe(fp_d2l);
14799 %}
14800
14801 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14802 match(Set dst (ConvI2D src));
14803
14804 ins_cost(INSN_COST * 5);
14805 format %{ "scvtfwd $dst, $src \t// i2d" %}
14806
14807 ins_encode %{
14808 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14809 %}
14810
14811 ins_pipe(fp_i2d);
14812 %}
14813
14814 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14815 match(Set dst (ConvL2D src));
14816
14817 ins_cost(INSN_COST * 5);
14818 format %{ "scvtfd $dst, $src \t// l2d" %}
14819
14820 ins_encode %{
14821 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14822 %}
14823
14824 ins_pipe(fp_l2d);
14825 %}
14826
14827 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14828 %{
14829 match(Set dst (RoundD src));
14830 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14831 format %{ "java_round_double $dst,$src"%}
14832 ins_encode %{
14833 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14834 as_FloatRegister($ftmp$$reg));
14835 %}
14836 ins_pipe(pipe_slow);
14837 %}
14838
14839 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14840 %{
14841 match(Set dst (RoundF src));
14842 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14843 format %{ "java_round_float $dst,$src"%}
14844 ins_encode %{
14845 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14846 as_FloatRegister($ftmp$$reg));
14847 %}
14848 ins_pipe(pipe_slow);
14849 %}
14850
14851 // stack <-> reg and reg <-> reg shuffles with no conversion
14852
14853 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14854
14855 match(Set dst (MoveF2I src));
14856
14857 effect(DEF dst, USE src);
14858
14859 ins_cost(4 * INSN_COST);
14860
14861 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14862
14863 ins_encode %{
14864 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14865 %}
14866
14867 ins_pipe(iload_reg_reg);
14868
14869 %}
14870
14871 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14872
14873 match(Set dst (MoveI2F src));
14874
14875 effect(DEF dst, USE src);
14876
14877 ins_cost(4 * INSN_COST);
14878
14879 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14880
14881 ins_encode %{
14882 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14883 %}
14884
14885 ins_pipe(pipe_class_memory);
14886
14887 %}
14888
14889 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14890
14891 match(Set dst (MoveD2L src));
14892
14893 effect(DEF dst, USE src);
14894
14895 ins_cost(4 * INSN_COST);
14896
14897 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14898
14899 ins_encode %{
14900 __ ldr($dst$$Register, Address(sp, $src$$disp));
14901 %}
14902
14903 ins_pipe(iload_reg_reg);
14904
14905 %}
14906
14907 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14908
14909 match(Set dst (MoveL2D src));
14910
14911 effect(DEF dst, USE src);
14912
14913 ins_cost(4 * INSN_COST);
14914
14915 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14916
14917 ins_encode %{
14918 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14919 %}
14920
14921 ins_pipe(pipe_class_memory);
14922
14923 %}
14924
14925 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14926
14927 match(Set dst (MoveF2I src));
14928
14929 effect(DEF dst, USE src);
14930
14931 ins_cost(INSN_COST);
14932
14933 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14934
14935 ins_encode %{
14936 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14937 %}
14938
14939 ins_pipe(pipe_class_memory);
14940
14941 %}
14942
14943 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14944
14945 match(Set dst (MoveI2F src));
14946
14947 effect(DEF dst, USE src);
14948
14949 ins_cost(INSN_COST);
14950
14951 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14952
14953 ins_encode %{
14954 __ strw($src$$Register, Address(sp, $dst$$disp));
14955 %}
14956
14957 ins_pipe(istore_reg_reg);
14958
14959 %}
14960
14961 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14962
14963 match(Set dst (MoveD2L src));
14964
14965 effect(DEF dst, USE src);
14966
14967 ins_cost(INSN_COST);
14968
14969 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14970
14971 ins_encode %{
14972 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14973 %}
14974
14975 ins_pipe(pipe_class_memory);
14976
14977 %}
14978
14979 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14980
14981 match(Set dst (MoveL2D src));
14982
14983 effect(DEF dst, USE src);
14984
14985 ins_cost(INSN_COST);
14986
14987 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14988
14989 ins_encode %{
14990 __ str($src$$Register, Address(sp, $dst$$disp));
14991 %}
14992
14993 ins_pipe(istore_reg_reg);
14994
14995 %}
14996
14997 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14998
14999 match(Set dst (MoveF2I src));
15000
15001 effect(DEF dst, USE src);
15002
15003 ins_cost(INSN_COST);
15004
15005 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15006
15007 ins_encode %{
15008 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15009 %}
15010
15011 ins_pipe(fp_f2i);
15012
15013 %}
15014
15015 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15016
15017 match(Set dst (MoveI2F src));
15018
15019 effect(DEF dst, USE src);
15020
15021 ins_cost(INSN_COST);
15022
15023 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15024
15025 ins_encode %{
15026 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15027 %}
15028
15029 ins_pipe(fp_i2f);
15030
15031 %}
15032
15033 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15034
15035 match(Set dst (MoveD2L src));
15036
15037 effect(DEF dst, USE src);
15038
15039 ins_cost(INSN_COST);
15040
15041 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15042
15043 ins_encode %{
15044 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15045 %}
15046
15047 ins_pipe(fp_d2l);
15048
15049 %}
15050
15051 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15052
15053 match(Set dst (MoveL2D src));
15054
15055 effect(DEF dst, USE src);
15056
15057 ins_cost(INSN_COST);
15058
15059 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15060
15061 ins_encode %{
15062 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15063 %}
15064
15065 ins_pipe(fp_l2d);
15066
15067 %}
15068
15069 // ============================================================================
15070 // clearing of an array
15071
15072 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
15073 %{
15074 match(Set dummy (ClearArray (Binary cnt base) zero));
15075 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15076
15077 ins_cost(4 * INSN_COST);
15078 format %{ "ClearArray $cnt, $base" %}
15079
15080 ins_encode %{
15081 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15082 if (tpc == NULL) {
15083 ciEnv::current()->record_failure("CodeCache is full");
15084 return;
15085 }
15086 %}
15087
15088 ins_pipe(pipe_class_memory);
15089 %}
15090
15091 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
15092 %{
15093 predicate(((ClearArrayNode*)n)->word_copy_only());
15094 match(Set dummy (ClearArray (Binary cnt base) val));
15095 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15096
15097 ins_cost(4 * INSN_COST);
15098 format %{ "ClearArray $cnt, $base, $val" %}
15099
15100 ins_encode %{
15101 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
15102 %}
15103
15104 ins_pipe(pipe_class_memory);
15105 %}
15106
15107 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15108 %{
15109 predicate((uint64_t)n->in(2)->get_long()
15110 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
15111 && !((ClearArrayNode*)n)->word_copy_only());
15112 match(Set dummy (ClearArray cnt base));
15113 effect(TEMP temp, USE_KILL base, KILL cr);
15114
15115 ins_cost(4 * INSN_COST);
15116 format %{ "ClearArray $cnt, $base" %}
15117
15118 ins_encode %{
15119 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15120 if (tpc == NULL) {
15121 ciEnv::current()->record_failure("CodeCache is full");
15122 return;
15123 }
15124 %}
15125
15126 ins_pipe(pipe_class_memory);
15127 %}
15128
15129 // ============================================================================
15130 // Overflow Math Instructions
15131
15132 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15133 %{
15134 match(Set cr (OverflowAddI op1 op2));
15135
15136 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15137 ins_cost(INSN_COST);
15138 ins_encode %{
15139 __ cmnw($op1$$Register, $op2$$Register);
15140 %}
15141
15142 ins_pipe(icmp_reg_reg);
15143 %}
15144
15145 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15146 %{
15147 match(Set cr (OverflowAddI op1 op2));
15148
15149 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15150 ins_cost(INSN_COST);
15151 ins_encode %{
15152 __ cmnw($op1$$Register, $op2$$constant);
15153 %}
15154
15155 ins_pipe(icmp_reg_imm);
15156 %}
15157
15158 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15159 %{
15160 match(Set cr (OverflowAddL op1 op2));
15161
15162 format %{ "cmn $op1, $op2\t# overflow check long" %}
15163 ins_cost(INSN_COST);
15164 ins_encode %{
15165 __ cmn($op1$$Register, $op2$$Register);
15166 %}
15167
15168 ins_pipe(icmp_reg_reg);
15169 %}
15170
15171 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15172 %{
15173 match(Set cr (OverflowAddL op1 op2));
15174
15175 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15176 ins_cost(INSN_COST);
15177 ins_encode %{
15178 __ adds(zr, $op1$$Register, $op2$$constant);
15179 %}
15180
15181 ins_pipe(icmp_reg_imm);
15182 %}
15183
15184 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15185 %{
15186 match(Set cr (OverflowSubI op1 op2));
15187
15188 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15189 ins_cost(INSN_COST);
15190 ins_encode %{
15191 __ cmpw($op1$$Register, $op2$$Register);
15192 %}
15193
15194 ins_pipe(icmp_reg_reg);
15195 %}
15196
15197 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15198 %{
15199 match(Set cr (OverflowSubI op1 op2));
15200
15201 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15202 ins_cost(INSN_COST);
15203 ins_encode %{
15204 __ cmpw($op1$$Register, $op2$$constant);
15205 %}
15206
15207 ins_pipe(icmp_reg_imm);
15208 %}
15209
15210 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15211 %{
15212 match(Set cr (OverflowSubL op1 op2));
15213
15214 format %{ "cmp $op1, $op2\t# overflow check long" %}
15215 ins_cost(INSN_COST);
15216 ins_encode %{
15217 __ cmp($op1$$Register, $op2$$Register);
15218 %}
15219
15220 ins_pipe(icmp_reg_reg);
15221 %}
15222
15223 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15224 %{
15225 match(Set cr (OverflowSubL op1 op2));
15226
15227 format %{ "cmp $op1, $op2\t# overflow check long" %}
15228 ins_cost(INSN_COST);
15229 ins_encode %{
15230 __ subs(zr, $op1$$Register, $op2$$constant);
15231 %}
15232
15233 ins_pipe(icmp_reg_imm);
15234 %}
15235
15236 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15237 %{
15238 match(Set cr (OverflowSubI zero op1));
15239
15240 format %{ "cmpw zr, $op1\t# overflow check int" %}
15241 ins_cost(INSN_COST);
15242 ins_encode %{
15243 __ cmpw(zr, $op1$$Register);
15244 %}
15245
15246 ins_pipe(icmp_reg_imm);
15247 %}
15248
15249 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15250 %{
15251 match(Set cr (OverflowSubL zero op1));
15252
15253 format %{ "cmp zr, $op1\t# overflow check long" %}
15254 ins_cost(INSN_COST);
15255 ins_encode %{
15256 __ cmp(zr, $op1$$Register);
15257 %}
15258
15259 ins_pipe(icmp_reg_imm);
15260 %}
15261
15262 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15263 %{
15264 match(Set cr (OverflowMulI op1 op2));
15265
15266 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15267 "cmp rscratch1, rscratch1, sxtw\n\t"
15268 "movw rscratch1, #0x80000000\n\t"
15269 "cselw rscratch1, rscratch1, zr, NE\n\t"
15270 "cmpw rscratch1, #1" %}
15271 ins_cost(5 * INSN_COST);
15272 ins_encode %{
15273 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15274 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15275 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15276 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15277 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15278 %}
15279
15280 ins_pipe(pipe_slow);
15281 %}
15282
15283 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15284 %{
15285 match(If cmp (OverflowMulI op1 op2));
15286 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15287 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15288 effect(USE labl, KILL cr);
15289
15290 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15291 "cmp rscratch1, rscratch1, sxtw\n\t"
15292 "b$cmp $labl" %}
15293 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15294 ins_encode %{
15295 Label* L = $labl$$label;
15296 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15297 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15298 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15299 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15300 %}
15301
15302 ins_pipe(pipe_serial);
15303 %}
15304
15305 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15306 %{
15307 match(Set cr (OverflowMulL op1 op2));
15308
15309 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15310 "smulh rscratch2, $op1, $op2\n\t"
15311 "cmp rscratch2, rscratch1, ASR #63\n\t"
15312 "movw rscratch1, #0x80000000\n\t"
15313 "cselw rscratch1, rscratch1, zr, NE\n\t"
15314 "cmpw rscratch1, #1" %}
15315 ins_cost(6 * INSN_COST);
15316 ins_encode %{
15317 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15318 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15319 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15320 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15321 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15322 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15323 %}
15324
15325 ins_pipe(pipe_slow);
15326 %}
15327
15328 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15329 %{
15330 match(If cmp (OverflowMulL op1 op2));
15331 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15332 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15333 effect(USE labl, KILL cr);
15334
15335 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15336 "smulh rscratch2, $op1, $op2\n\t"
15337 "cmp rscratch2, rscratch1, ASR #63\n\t"
15338 "b$cmp $labl" %}
15339 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15340 ins_encode %{
15341 Label* L = $labl$$label;
15342 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15343 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15344 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15345 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15346 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15347 %}
15348
15349 ins_pipe(pipe_serial);
15350 %}
15351
15352 // ============================================================================
15353 // Compare Instructions
15354
15355 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15356 %{
15357 match(Set cr (CmpI op1 op2));
15358
15359 effect(DEF cr, USE op1, USE op2);
15360
15361 ins_cost(INSN_COST);
15362 format %{ "cmpw $op1, $op2" %}
15363
15364 ins_encode(aarch64_enc_cmpw(op1, op2));
15365
15366 ins_pipe(icmp_reg_reg);
15367 %}
15368
15369 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15370 %{
15371 match(Set cr (CmpI op1 zero));
15372
15373 effect(DEF cr, USE op1);
15374
15375 ins_cost(INSN_COST);
15376 format %{ "cmpw $op1, 0" %}
15377
15378 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15379
15380 ins_pipe(icmp_reg_imm);
15381 %}
15382
15383 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15384 %{
15385 match(Set cr (CmpI op1 op2));
15386
15387 effect(DEF cr, USE op1);
15388
15389 ins_cost(INSN_COST);
15390 format %{ "cmpw $op1, $op2" %}
15391
15392 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15393
15394 ins_pipe(icmp_reg_imm);
15395 %}
15396
15397 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15398 %{
15399 match(Set cr (CmpI op1 op2));
15400
15401 effect(DEF cr, USE op1);
15402
15403 ins_cost(INSN_COST * 2);
15404 format %{ "cmpw $op1, $op2" %}
15405
15406 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15407
15408 ins_pipe(icmp_reg_imm);
15409 %}
15410
15411 // Unsigned compare Instructions; really, same as signed compare
15412 // except it should only be used to feed an If or a CMovI which takes a
15413 // cmpOpU.
15414
15415 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15416 %{
15417 match(Set cr (CmpU op1 op2));
15418
15419 effect(DEF cr, USE op1, USE op2);
15420
15421 ins_cost(INSN_COST);
15422 format %{ "cmpw $op1, $op2\t# unsigned" %}
15423
15424 ins_encode(aarch64_enc_cmpw(op1, op2));
15425
15426 ins_pipe(icmp_reg_reg);
15427 %}
15428
15429 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15430 %{
15431 match(Set cr (CmpU op1 zero));
15432
15433 effect(DEF cr, USE op1);
15434
15435 ins_cost(INSN_COST);
15436 format %{ "cmpw $op1, #0\t# unsigned" %}
15437
15438 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15439
15440 ins_pipe(icmp_reg_imm);
15441 %}
15442
15443 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15444 %{
15445 match(Set cr (CmpU op1 op2));
15446
15447 effect(DEF cr, USE op1);
15448
15449 ins_cost(INSN_COST);
15450 format %{ "cmpw $op1, $op2\t# unsigned" %}
15451
15452 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15453
15454 ins_pipe(icmp_reg_imm);
15455 %}
15456
15457 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15458 %{
15459 match(Set cr (CmpU op1 op2));
15460
15461 effect(DEF cr, USE op1);
15462
15463 ins_cost(INSN_COST * 2);
15464 format %{ "cmpw $op1, $op2\t# unsigned" %}
15465
15466 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15467
15468 ins_pipe(icmp_reg_imm);
15469 %}
15470
15471 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15472 %{
15473 match(Set cr (CmpL op1 op2));
15474
15475 effect(DEF cr, USE op1, USE op2);
15476
15477 ins_cost(INSN_COST);
15478 format %{ "cmp $op1, $op2" %}
15479
15480 ins_encode(aarch64_enc_cmp(op1, op2));
15481
15482 ins_pipe(icmp_reg_reg);
15483 %}
15484
15485 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15486 %{
15487 match(Set cr (CmpL op1 zero));
15488
15489 effect(DEF cr, USE op1);
15490
15491 ins_cost(INSN_COST);
15492 format %{ "tst $op1" %}
15493
15494 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15495
15496 ins_pipe(icmp_reg_imm);
15497 %}
15498
15499 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15500 %{
15501 match(Set cr (CmpL op1 op2));
15502
15503 effect(DEF cr, USE op1);
15504
15505 ins_cost(INSN_COST);
15506 format %{ "cmp $op1, $op2" %}
15507
15508 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15509
15510 ins_pipe(icmp_reg_imm);
15511 %}
15512
15513 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15514 %{
15515 match(Set cr (CmpL op1 op2));
15516
15517 effect(DEF cr, USE op1);
15518
15519 ins_cost(INSN_COST * 2);
15520 format %{ "cmp $op1, $op2" %}
15521
15522 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15523
15524 ins_pipe(icmp_reg_imm);
15525 %}
15526
15527 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15528 %{
15529 match(Set cr (CmpUL op1 op2));
15530
15531 effect(DEF cr, USE op1, USE op2);
15532
15533 ins_cost(INSN_COST);
15534 format %{ "cmp $op1, $op2" %}
15535
15536 ins_encode(aarch64_enc_cmp(op1, op2));
15537
15538 ins_pipe(icmp_reg_reg);
15539 %}
15540
15541 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15542 %{
15543 match(Set cr (CmpUL op1 zero));
15544
15545 effect(DEF cr, USE op1);
15546
15547 ins_cost(INSN_COST);
15548 format %{ "tst $op1" %}
15549
15550 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15551
15552 ins_pipe(icmp_reg_imm);
15553 %}
15554
15555 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15556 %{
15557 match(Set cr (CmpUL op1 op2));
15558
15559 effect(DEF cr, USE op1);
15560
15561 ins_cost(INSN_COST);
15562 format %{ "cmp $op1, $op2" %}
15563
15564 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15565
15566 ins_pipe(icmp_reg_imm);
15567 %}
15568
15569 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15570 %{
15571 match(Set cr (CmpUL op1 op2));
15572
15573 effect(DEF cr, USE op1);
15574
15575 ins_cost(INSN_COST * 2);
15576 format %{ "cmp $op1, $op2" %}
15577
15578 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15579
15580 ins_pipe(icmp_reg_imm);
15581 %}
15582
15583 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15584 %{
15585 match(Set cr (CmpP op1 op2));
15586
15587 effect(DEF cr, USE op1, USE op2);
15588
15589 ins_cost(INSN_COST);
15590 format %{ "cmp $op1, $op2\t // ptr" %}
15591
15592 ins_encode(aarch64_enc_cmpp(op1, op2));
15593
15594 ins_pipe(icmp_reg_reg);
15595 %}
15596
15597 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15598 %{
15599 match(Set cr (CmpN op1 op2));
15600
15601 effect(DEF cr, USE op1, USE op2);
15602
15603 ins_cost(INSN_COST);
15604 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15605
15606 ins_encode(aarch64_enc_cmpn(op1, op2));
15607
15608 ins_pipe(icmp_reg_reg);
15609 %}
15610
15611 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15612 %{
15613 match(Set cr (CmpP op1 zero));
15614
15615 effect(DEF cr, USE op1, USE zero);
15616
15617 ins_cost(INSN_COST);
15618 format %{ "cmp $op1, 0\t // ptr" %}
15619
15620 ins_encode(aarch64_enc_testp(op1));
15621
15622 ins_pipe(icmp_reg_imm);
15623 %}
15624
15625 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15626 %{
15627 match(Set cr (CmpN op1 zero));
15628
15629 effect(DEF cr, USE op1, USE zero);
15630
15631 ins_cost(INSN_COST);
15632 format %{ "cmp $op1, 0\t // compressed ptr" %}
15633
15634 ins_encode(aarch64_enc_testn(op1));
15635
15636 ins_pipe(icmp_reg_imm);
15637 %}
15638
15639 // FP comparisons
15640 //
15641 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15642 // using normal cmpOp. See declaration of rFlagsReg for details.
15643
15644 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15645 %{
15646 match(Set cr (CmpF src1 src2));
15647
15648 ins_cost(3 * INSN_COST);
15649 format %{ "fcmps $src1, $src2" %}
15650
15651 ins_encode %{
15652 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15653 %}
15654
15655 ins_pipe(pipe_class_compare);
15656 %}
15657
15658 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15659 %{
15660 match(Set cr (CmpF src1 src2));
15661
15662 ins_cost(3 * INSN_COST);
15663 format %{ "fcmps $src1, 0.0" %}
15664
15665 ins_encode %{
15666 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15667 %}
15668
15669 ins_pipe(pipe_class_compare);
15670 %}
15671 // FROM HERE
15672
15673 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15674 %{
15675 match(Set cr (CmpD src1 src2));
15676
15677 ins_cost(3 * INSN_COST);
15678 format %{ "fcmpd $src1, $src2" %}
15679
15680 ins_encode %{
15681 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15682 %}
15683
15684 ins_pipe(pipe_class_compare);
15685 %}
15686
15687 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15688 %{
15689 match(Set cr (CmpD src1 src2));
15690
15691 ins_cost(3 * INSN_COST);
15692 format %{ "fcmpd $src1, 0.0" %}
15693
15694 ins_encode %{
15695 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15696 %}
15697
15698 ins_pipe(pipe_class_compare);
15699 %}
15700
15701 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15702 %{
15703 match(Set dst (CmpF3 src1 src2));
15704 effect(KILL cr);
15705
15706 ins_cost(5 * INSN_COST);
15707 format %{ "fcmps $src1, $src2\n\t"
15708 "csinvw($dst, zr, zr, eq\n\t"
15709 "csnegw($dst, $dst, $dst, lt)"
15710 %}
15711
15712 ins_encode %{
15713 Label done;
15714 FloatRegister s1 = as_FloatRegister($src1$$reg);
15715 FloatRegister s2 = as_FloatRegister($src2$$reg);
15716 Register d = as_Register($dst$$reg);
15717 __ fcmps(s1, s2);
15718 // installs 0 if EQ else -1
15719 __ csinvw(d, zr, zr, Assembler::EQ);
15720 // keeps -1 if less or unordered else installs 1
15721 __ csnegw(d, d, d, Assembler::LT);
15722 __ bind(done);
15723 %}
15724
15725 ins_pipe(pipe_class_default);
15726
15727 %}
15728
15729 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15730 %{
15731 match(Set dst (CmpD3 src1 src2));
15732 effect(KILL cr);
15733
15734 ins_cost(5 * INSN_COST);
15735 format %{ "fcmpd $src1, $src2\n\t"
15736 "csinvw($dst, zr, zr, eq\n\t"
15737 "csnegw($dst, $dst, $dst, lt)"
15738 %}
15739
15740 ins_encode %{
15741 Label done;
15742 FloatRegister s1 = as_FloatRegister($src1$$reg);
15743 FloatRegister s2 = as_FloatRegister($src2$$reg);
15744 Register d = as_Register($dst$$reg);
15745 __ fcmpd(s1, s2);
15746 // installs 0 if EQ else -1
15747 __ csinvw(d, zr, zr, Assembler::EQ);
15748 // keeps -1 if less or unordered else installs 1
15749 __ csnegw(d, d, d, Assembler::LT);
15750 __ bind(done);
15751 %}
15752 ins_pipe(pipe_class_default);
15753
15754 %}
15755
15756 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15757 %{
15758 match(Set dst (CmpF3 src1 zero));
15759 effect(KILL cr);
15760
15761 ins_cost(5 * INSN_COST);
15762 format %{ "fcmps $src1, 0.0\n\t"
15763 "csinvw($dst, zr, zr, eq\n\t"
15764 "csnegw($dst, $dst, $dst, lt)"
15765 %}
15766
15767 ins_encode %{
15768 Label done;
15769 FloatRegister s1 = as_FloatRegister($src1$$reg);
15770 Register d = as_Register($dst$$reg);
15771 __ fcmps(s1, 0.0);
15772 // installs 0 if EQ else -1
15773 __ csinvw(d, zr, zr, Assembler::EQ);
15774 // keeps -1 if less or unordered else installs 1
15775 __ csnegw(d, d, d, Assembler::LT);
15776 __ bind(done);
15777 %}
15778
15779 ins_pipe(pipe_class_default);
15780
15781 %}
15782
15783 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15784 %{
15785 match(Set dst (CmpD3 src1 zero));
15786 effect(KILL cr);
15787
15788 ins_cost(5 * INSN_COST);
15789 format %{ "fcmpd $src1, 0.0\n\t"
15790 "csinvw($dst, zr, zr, eq\n\t"
15791 "csnegw($dst, $dst, $dst, lt)"
15792 %}
15793
15794 ins_encode %{
15795 Label done;
15796 FloatRegister s1 = as_FloatRegister($src1$$reg);
15797 Register d = as_Register($dst$$reg);
15798 __ fcmpd(s1, 0.0);
15799 // installs 0 if EQ else -1
15800 __ csinvw(d, zr, zr, Assembler::EQ);
15801 // keeps -1 if less or unordered else installs 1
15802 __ csnegw(d, d, d, Assembler::LT);
15803 __ bind(done);
15804 %}
15805 ins_pipe(pipe_class_default);
15806
15807 %}
15808
15809 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15810 %{
15811 match(Set dst (CmpLTMask p q));
15812 effect(KILL cr);
15813
15814 ins_cost(3 * INSN_COST);
15815
15816 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15817 "csetw $dst, lt\n\t"
15818 "subw $dst, zr, $dst"
15819 %}
15820
15821 ins_encode %{
15822 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15823 __ csetw(as_Register($dst$$reg), Assembler::LT);
15824 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15825 %}
15826
15827 ins_pipe(ialu_reg_reg);
15828 %}
15829
15830 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15831 %{
15832 match(Set dst (CmpLTMask src zero));
15833 effect(KILL cr);
15834
15835 ins_cost(INSN_COST);
15836
15837 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15838
15839 ins_encode %{
15840 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15841 %}
15842
15843 ins_pipe(ialu_reg_shift);
15844 %}
15845
15846 // ============================================================================
15847 // Max and Min
15848
15849 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15850 %{
15851 effect( DEF dst, USE src1, USE src2, USE cr );
15852
15853 ins_cost(INSN_COST * 2);
15854 format %{ "cselw $dst, $src1, $src2 lt\t" %}
15855
15856 ins_encode %{
15857 __ cselw(as_Register($dst$$reg),
15858 as_Register($src1$$reg),
15859 as_Register($src2$$reg),
15860 Assembler::LT);
15861 %}
15862
15863 ins_pipe(icond_reg_reg);
15864 %}
15865
15866 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15867 %{
15868 match(Set dst (MinI src1 src2));
15869 ins_cost(INSN_COST * 3);
15870
15871 expand %{
15872 rFlagsReg cr;
15873 compI_reg_reg(cr, src1, src2);
15874 cmovI_reg_reg_lt(dst, src1, src2, cr);
15875 %}
15876
15877 %}
15878 // FROM HERE
15879
15880 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15881 %{
15882 effect( DEF dst, USE src1, USE src2, USE cr );
15883
15884 ins_cost(INSN_COST * 2);
15885 format %{ "cselw $dst, $src1, $src2 gt\t" %}
15886
15887 ins_encode %{
15888 __ cselw(as_Register($dst$$reg),
15889 as_Register($src1$$reg),
15890 as_Register($src2$$reg),
15891 Assembler::GT);
15892 %}
15893
15894 ins_pipe(icond_reg_reg);
15895 %}
15896
15897 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15898 %{
15899 match(Set dst (MaxI src1 src2));
15900 ins_cost(INSN_COST * 3);
15901 expand %{
15902 rFlagsReg cr;
15903 compI_reg_reg(cr, src1, src2);
15904 cmovI_reg_reg_gt(dst, src1, src2, cr);
15905 %}
15906 %}
15907
15908 // ============================================================================
15909 // Branch Instructions
15910
15911 // Direct Branch.
15912 instruct branch(label lbl)
15913 %{
15914 match(Goto);
15915
15916 effect(USE lbl);
15917
15918 ins_cost(BRANCH_COST);
15919 format %{ "b $lbl" %}
15920
15921 ins_encode(aarch64_enc_b(lbl));
15922
15923 ins_pipe(pipe_branch);
15924 %}
15925
15926 // Conditional Near Branch
15927 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15928 %{
15929 // Same match rule as `branchConFar'.
15930 match(If cmp cr);
15931
15932 effect(USE lbl);
15933
15934 ins_cost(BRANCH_COST);
15935 // If set to 1 this indicates that the current instruction is a
15936 // short variant of a long branch. This avoids using this
15937 // instruction in first-pass matching. It will then only be used in
15938 // the `Shorten_branches' pass.
15939 // ins_short_branch(1);
15940 format %{ "b$cmp $lbl" %}
15941
15942 ins_encode(aarch64_enc_br_con(cmp, lbl));
15943
15944 ins_pipe(pipe_branch_cond);
15945 %}
15946
15947 // Conditional Near Branch Unsigned
15948 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15949 %{
15950 // Same match rule as `branchConFar'.
15951 match(If cmp cr);
15952
15953 effect(USE lbl);
15954
15955 ins_cost(BRANCH_COST);
15956 // If set to 1 this indicates that the current instruction is a
15957 // short variant of a long branch. This avoids using this
15958 // instruction in first-pass matching. It will then only be used in
15959 // the `Shorten_branches' pass.
15960 // ins_short_branch(1);
15961 format %{ "b$cmp $lbl\t# unsigned" %}
15962
15963 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15964
15965 ins_pipe(pipe_branch_cond);
15966 %}
15967
15968 // Make use of CBZ and CBNZ. These instructions, as well as being
15969 // shorter than (cmp; branch), have the additional benefit of not
15970 // killing the flags.
15971
15972 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15973 match(If cmp (CmpI op1 op2));
15974 effect(USE labl);
15975
15976 ins_cost(BRANCH_COST);
15977 format %{ "cbw$cmp $op1, $labl" %}
15978 ins_encode %{
15979 Label* L = $labl$$label;
15980 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15981 if (cond == Assembler::EQ)
15982 __ cbzw($op1$$Register, *L);
15983 else
15984 __ cbnzw($op1$$Register, *L);
15985 %}
15986 ins_pipe(pipe_cmp_branch);
15987 %}
15988
15989 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15990 match(If cmp (CmpL op1 op2));
15991 effect(USE labl);
15992
15993 ins_cost(BRANCH_COST);
15994 format %{ "cb$cmp $op1, $labl" %}
15995 ins_encode %{
15996 Label* L = $labl$$label;
15997 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15998 if (cond == Assembler::EQ)
15999 __ cbz($op1$$Register, *L);
16000 else
16001 __ cbnz($op1$$Register, *L);
16002 %}
16003 ins_pipe(pipe_cmp_branch);
16004 %}
16005
16006 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16007 match(If cmp (CmpP op1 op2));
16008 effect(USE labl);
16009
16010 ins_cost(BRANCH_COST);
16011 format %{ "cb$cmp $op1, $labl" %}
16012 ins_encode %{
16013 Label* L = $labl$$label;
16014 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16015 if (cond == Assembler::EQ)
16016 __ cbz($op1$$Register, *L);
16017 else
16018 __ cbnz($op1$$Register, *L);
16019 %}
16020 ins_pipe(pipe_cmp_branch);
16021 %}
16022
16023 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16024 match(If cmp (CmpN op1 op2));
16025 effect(USE labl);
16026
16027 ins_cost(BRANCH_COST);
16028 format %{ "cbw$cmp $op1, $labl" %}
16029 ins_encode %{
16030 Label* L = $labl$$label;
16031 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16032 if (cond == Assembler::EQ)
16033 __ cbzw($op1$$Register, *L);
16034 else
16035 __ cbnzw($op1$$Register, *L);
16036 %}
16037 ins_pipe(pipe_cmp_branch);
16038 %}
16039
16040 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16041 match(If cmp (CmpP (DecodeN oop) zero));
16042 effect(USE labl);
16043
16044 ins_cost(BRANCH_COST);
16045 format %{ "cb$cmp $oop, $labl" %}
16046 ins_encode %{
16047 Label* L = $labl$$label;
16048 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16049 if (cond == Assembler::EQ)
16050 __ cbzw($oop$$Register, *L);
16051 else
16052 __ cbnzw($oop$$Register, *L);
16053 %}
16054 ins_pipe(pipe_cmp_branch);
16055 %}
16056
16057 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16058 match(If cmp (CmpU op1 op2));
16059 effect(USE labl);
16060
16061 ins_cost(BRANCH_COST);
16062 format %{ "cbw$cmp $op1, $labl" %}
16063 ins_encode %{
16064 Label* L = $labl$$label;
16065 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16066 if (cond == Assembler::EQ || cond == Assembler::LS)
16067 __ cbzw($op1$$Register, *L);
16068 else
16069 __ cbnzw($op1$$Register, *L);
16070 %}
16071 ins_pipe(pipe_cmp_branch);
16072 %}
16073
16074 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16075 match(If cmp (CmpUL op1 op2));
16076 effect(USE labl);
16077
16078 ins_cost(BRANCH_COST);
16079 format %{ "cb$cmp $op1, $labl" %}
16080 ins_encode %{
16081 Label* L = $labl$$label;
16082 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16083 if (cond == Assembler::EQ || cond == Assembler::LS)
16084 __ cbz($op1$$Register, *L);
16085 else
16086 __ cbnz($op1$$Register, *L);
16087 %}
16088 ins_pipe(pipe_cmp_branch);
16089 %}
16090
16091 // Test bit and Branch
16092
16093 // Patterns for short (< 32KiB) variants
16094 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16095 match(If cmp (CmpL op1 op2));
16096 effect(USE labl);
16097
16098 ins_cost(BRANCH_COST);
16099 format %{ "cb$cmp $op1, $labl # long" %}
16100 ins_encode %{
16101 Label* L = $labl$$label;
16102 Assembler::Condition cond =
16103 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16104 __ tbr(cond, $op1$$Register, 63, *L);
16105 %}
16106 ins_pipe(pipe_cmp_branch);
16107 ins_short_branch(1);
16108 %}
16109
16110 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16111 match(If cmp (CmpI op1 op2));
16112 effect(USE labl);
16113
16114 ins_cost(BRANCH_COST);
16115 format %{ "cb$cmp $op1, $labl # int" %}
16116 ins_encode %{
16117 Label* L = $labl$$label;
16118 Assembler::Condition cond =
16119 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16120 __ tbr(cond, $op1$$Register, 31, *L);
16121 %}
16122 ins_pipe(pipe_cmp_branch);
16123 ins_short_branch(1);
16124 %}
16125
16126 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16127 match(If cmp (CmpL (AndL op1 op2) op3));
16128 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16129 effect(USE labl);
16130
16131 ins_cost(BRANCH_COST);
16132 format %{ "tb$cmp $op1, $op2, $labl" %}
16133 ins_encode %{
16134 Label* L = $labl$$label;
16135 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16136 int bit = exact_log2_long($op2$$constant);
16137 __ tbr(cond, $op1$$Register, bit, *L);
16138 %}
16139 ins_pipe(pipe_cmp_branch);
16140 ins_short_branch(1);
16141 %}
16142
16143 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16144 match(If cmp (CmpI (AndI op1 op2) op3));
16145 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16146 effect(USE labl);
16147
16148 ins_cost(BRANCH_COST);
16149 format %{ "tb$cmp $op1, $op2, $labl" %}
16150 ins_encode %{
16151 Label* L = $labl$$label;
16152 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16153 int bit = exact_log2((juint)$op2$$constant);
16154 __ tbr(cond, $op1$$Register, bit, *L);
16155 %}
16156 ins_pipe(pipe_cmp_branch);
16157 ins_short_branch(1);
16158 %}
16159
16160 // And far variants
16161 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16162 match(If cmp (CmpL op1 op2));
16163 effect(USE labl);
16164
16165 ins_cost(BRANCH_COST);
16166 format %{ "cb$cmp $op1, $labl # long" %}
16167 ins_encode %{
16168 Label* L = $labl$$label;
16169 Assembler::Condition cond =
16170 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16171 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16172 %}
16173 ins_pipe(pipe_cmp_branch);
16174 %}
16175
16176 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16177 match(If cmp (CmpI op1 op2));
16178 effect(USE labl);
16179
16180 ins_cost(BRANCH_COST);
16181 format %{ "cb$cmp $op1, $labl # int" %}
16182 ins_encode %{
16183 Label* L = $labl$$label;
16184 Assembler::Condition cond =
16185 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16186 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16187 %}
16188 ins_pipe(pipe_cmp_branch);
16189 %}
16190
16191 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16192 match(If cmp (CmpL (AndL op1 op2) op3));
16193 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16194 effect(USE labl);
16195
16196 ins_cost(BRANCH_COST);
16197 format %{ "tb$cmp $op1, $op2, $labl" %}
16198 ins_encode %{
16199 Label* L = $labl$$label;
16200 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16201 int bit = exact_log2_long($op2$$constant);
16202 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16203 %}
16204 ins_pipe(pipe_cmp_branch);
16205 %}
16206
16207 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16208 match(If cmp (CmpI (AndI op1 op2) op3));
16209 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16210 effect(USE labl);
16211
16212 ins_cost(BRANCH_COST);
16213 format %{ "tb$cmp $op1, $op2, $labl" %}
16214 ins_encode %{
16215 Label* L = $labl$$label;
16216 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16217 int bit = exact_log2((juint)$op2$$constant);
16218 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16219 %}
16220 ins_pipe(pipe_cmp_branch);
16221 %}
16222
16223 // Test bits
16224
16225 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16226 match(Set cr (CmpL (AndL op1 op2) op3));
16227 predicate(Assembler::operand_valid_for_logical_immediate
16228 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16229
16230 ins_cost(INSN_COST);
16231 format %{ "tst $op1, $op2 # long" %}
16232 ins_encode %{
16233 __ tst($op1$$Register, $op2$$constant);
16234 %}
16235 ins_pipe(ialu_reg_reg);
16236 %}
16237
16238 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16239 match(Set cr (CmpI (AndI op1 op2) op3));
16240 predicate(Assembler::operand_valid_for_logical_immediate
16241 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16242
16243 ins_cost(INSN_COST);
16244 format %{ "tst $op1, $op2 # int" %}
16245 ins_encode %{
16246 __ tstw($op1$$Register, $op2$$constant);
16247 %}
16248 ins_pipe(ialu_reg_reg);
16249 %}
16250
16251 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16252 match(Set cr (CmpL (AndL op1 op2) op3));
16253
16254 ins_cost(INSN_COST);
16255 format %{ "tst $op1, $op2 # long" %}
16256 ins_encode %{
16257 __ tst($op1$$Register, $op2$$Register);
16258 %}
16259 ins_pipe(ialu_reg_reg);
16260 %}
16261
16262 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16263 match(Set cr (CmpI (AndI op1 op2) op3));
16264
16265 ins_cost(INSN_COST);
16266 format %{ "tstw $op1, $op2 # int" %}
16267 ins_encode %{
16268 __ tstw($op1$$Register, $op2$$Register);
16269 %}
16270 ins_pipe(ialu_reg_reg);
16271 %}
16272
16273
16274 // Conditional Far Branch
16275 // Conditional Far Branch Unsigned
16276 // TODO: fixme
16277
16278 // counted loop end branch near
16279 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16280 %{
16281 match(CountedLoopEnd cmp cr);
16282
16283 effect(USE lbl);
16284
16285 ins_cost(BRANCH_COST);
16286 // short variant.
16287 // ins_short_branch(1);
16288 format %{ "b$cmp $lbl \t// counted loop end" %}
16289
16290 ins_encode(aarch64_enc_br_con(cmp, lbl));
16291
16292 ins_pipe(pipe_branch);
16293 %}
16294
16295 // counted loop end branch near Unsigned
16296 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16297 %{
16298 match(CountedLoopEnd cmp cr);
16299
16300 effect(USE lbl);
16301
16302 ins_cost(BRANCH_COST);
16303 // short variant.
16304 // ins_short_branch(1);
16305 format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16306
16307 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16308
16309 ins_pipe(pipe_branch);
16310 %}
16311
16312 // counted loop end branch far
16313 // counted loop end branch far unsigned
16314 // TODO: fixme
16315
16316 // ============================================================================
16317 // inlined locking and unlocking
16318
16319 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16320 %{
16321 match(Set cr (FastLock object box));
16322 effect(TEMP tmp, TEMP tmp2);
16323
16324 // TODO
16325 // identify correct cost
16326 ins_cost(5 * INSN_COST);
16327 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16328
16329 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16330
16331 ins_pipe(pipe_serial);
16332 %}
16333
16334 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16335 %{
16336 match(Set cr (FastUnlock object box));
16337 effect(TEMP tmp, TEMP tmp2);
16338
16339 ins_cost(5 * INSN_COST);
16340 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16341
16342 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16343
16344 ins_pipe(pipe_serial);
16345 %}
16346
16347
16348 // ============================================================================
16349 // Safepoint Instructions
16350
16351 // TODO
16352 // provide a near and far version of this code
16353
16354 instruct safePoint(rFlagsReg cr, iRegP poll)
16355 %{
16356 match(SafePoint poll);
16357 effect(KILL cr);
16358
16359 format %{
16360 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16361 %}
16362 ins_encode %{
16363 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16364 %}
16365 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16366 %}
16367
16368
16369 // ============================================================================
16370 // Procedure Call/Return Instructions
16371
16372 // Call Java Static Instruction
16373
16374 instruct CallStaticJavaDirect(method meth)
16375 %{
16376 match(CallStaticJava);
16377
16378 effect(USE meth);
16379
16380 ins_cost(CALL_COST);
16381
16382 format %{ "call,static $meth \t// ==> " %}
16383
16384 ins_encode(aarch64_enc_java_static_call(meth),
16385 aarch64_enc_call_epilog);
16386
16387 ins_pipe(pipe_class_call);
16388 %}
16389
16390 // TO HERE
16391
16392 // Call Java Dynamic Instruction
16393 instruct CallDynamicJavaDirect(method meth)
16394 %{
16395 match(CallDynamicJava);
16396
16397 effect(USE meth);
16398
16399 ins_cost(CALL_COST);
16400
16401 format %{ "CALL,dynamic $meth \t// ==> " %}
16402
16403 ins_encode(aarch64_enc_java_dynamic_call(meth),
16404 aarch64_enc_call_epilog);
16405
16406 ins_pipe(pipe_class_call);
16407 %}
16408
16409 // Call Runtime Instruction
16410
16411 instruct CallRuntimeDirect(method meth)
16412 %{
16413 match(CallRuntime);
16414
16415 effect(USE meth);
16416
16417 ins_cost(CALL_COST);
16418
16419 format %{ "CALL, runtime $meth" %}
16420
16421 ins_encode( aarch64_enc_java_to_runtime(meth) );
16422
16423 ins_pipe(pipe_class_call);
16424 %}
16425
16426 // Call Runtime Instruction
16427
16428 instruct CallLeafDirect(method meth)
16429 %{
16430 match(CallLeaf);
16431
16432 effect(USE meth);
16433
16434 ins_cost(CALL_COST);
16435
16436 format %{ "CALL, runtime leaf $meth" %}
16437
16438 ins_encode( aarch64_enc_java_to_runtime(meth) );
16439
16440 ins_pipe(pipe_class_call);
16441 %}
16442
16443 // Call Runtime Instruction
16444
16445 // entry point is null, target holds the address to call
16446 instruct CallLeafNoFPIndirect(iRegP target)
16447 %{
16448 predicate(n->as_Call()->entry_point() == NULL);
16449
16450 match(CallLeafNoFP target);
16451
16452 ins_cost(CALL_COST);
16453
16454 format %{ "CALL, runtime leaf nofp indirect $target" %}
16455
16456 ins_encode %{
16457 __ blr($target$$Register);
16458 %}
16459
16460 ins_pipe(pipe_class_call);
16461 %}
16462
16463 instruct CallLeafNoFPDirect(method meth)
16464 %{
16465 predicate(n->as_Call()->entry_point() != NULL);
16466
16467 match(CallLeafNoFP);
16468
16469 effect(USE meth);
16470
16471 ins_cost(CALL_COST);
16472
16473 format %{ "CALL, runtime leaf nofp $meth" %}
16474
16475 ins_encode( aarch64_enc_java_to_runtime(meth) );
16476
16477 ins_pipe(pipe_class_call);
16478 %}
16479
16480 // Tail Call; Jump from runtime stub to Java code.
16481 // Also known as an 'interprocedural jump'.
16482 // Target of jump will eventually return to caller.
16483 // TailJump below removes the return address.
16484 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16485 %{
16486 match(TailCall jump_target method_ptr);
16487
16488 ins_cost(CALL_COST);
16489
16490 format %{ "br $jump_target\t# $method_ptr holds method" %}
16491
16492 ins_encode(aarch64_enc_tail_call(jump_target));
16493
16494 ins_pipe(pipe_class_call);
16495 %}
16496
16497 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16498 %{
16499 match(TailJump jump_target ex_oop);
16500
16501 ins_cost(CALL_COST);
16502
16503 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16504
16505 ins_encode(aarch64_enc_tail_jmp(jump_target));
16506
16507 ins_pipe(pipe_class_call);
16508 %}
16509
16510 // Create exception oop: created by stack-crawling runtime code.
16511 // Created exception is now available to this handler, and is setup
16512 // just prior to jumping to this handler. No code emitted.
16513 // TODO check
16514 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16515 instruct CreateException(iRegP_R0 ex_oop)
16516 %{
16517 match(Set ex_oop (CreateEx));
16518
16519 format %{ " -- \t// exception oop; no code emitted" %}
16520
16521 size(0);
16522
16523 ins_encode( /*empty*/ );
16524
16525 ins_pipe(pipe_class_empty);
16526 %}
16527
16528 // Rethrow exception: The exception oop will come in the first
16529 // argument position. Then JUMP (not call) to the rethrow stub code.
16530 instruct RethrowException() %{
16531 match(Rethrow);
16532 ins_cost(CALL_COST);
16533
16534 format %{ "b rethrow_stub" %}
16535
16536 ins_encode( aarch64_enc_rethrow() );
16537
16538 ins_pipe(pipe_class_call);
16539 %}
16540
16541
16542 // Return Instruction
16543 // epilog node loads ret address into lr as part of frame pop
16544 instruct Ret()
16545 %{
16546 match(Return);
16547
16548 format %{ "ret\t// return register" %}
16549
16550 ins_encode( aarch64_enc_ret() );
16551
16552 ins_pipe(pipe_branch);
16553 %}
16554
16555 // Die now.
16556 instruct ShouldNotReachHere() %{
16557 match(Halt);
16558
16559 ins_cost(CALL_COST);
16560 format %{ "ShouldNotReachHere" %}
16561
16562 ins_encode %{
16563 if (is_reachable()) {
16564 __ stop(_halt_reason);
16565 }
16566 %}
16567
16568 ins_pipe(pipe_class_default);
16569 %}
16570
16571 // ============================================================================
16572 // Partial Subtype Check
16573 //
16574 // superklass array for an instance of the superklass. Set a hidden
16575 // internal cache on a hit (cache is checked with exposed code in
16576 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16577 // encoding ALSO sets flags.
16578
16579 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16580 %{
16581 match(Set result (PartialSubtypeCheck sub super));
16582 effect(KILL cr, KILL temp);
16583
16584 ins_cost(1100); // slightly larger than the next version
16585 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16586
16587 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16588
16589 opcode(0x1); // Force zero of result reg on hit
16590
16591 ins_pipe(pipe_class_memory);
16592 %}
16593
16594 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16595 %{
16596 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16597 effect(KILL temp, KILL result);
16598
16599 ins_cost(1100); // slightly larger than the next version
16600 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16601
16602 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16603
16604 opcode(0x0); // Don't zero result reg on hit
16605
16606 ins_pipe(pipe_class_memory);
16607 %}
16608
16609 // Intrisics for String.compareTo()
16610
16611 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16612 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16613 %{
16614 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16615 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16616 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16617
16618 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16619 ins_encode %{
16620 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16621 __ string_compare($str1$$Register, $str2$$Register,
16622 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16623 $tmp1$$Register, $tmp2$$Register,
16624 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16625 %}
16626 ins_pipe(pipe_class_memory);
16627 %}
16628
16629 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16630 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16631 %{
16632 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16633 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16634 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16635
16636 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16637 ins_encode %{
16638 __ string_compare($str1$$Register, $str2$$Register,
16639 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16640 $tmp1$$Register, $tmp2$$Register,
16641 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16642 %}
16643 ins_pipe(pipe_class_memory);
16644 %}
16645
16646 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16647 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16648 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16649 %{
16650 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16651 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16652 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16653 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16654
16655 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16656 ins_encode %{
16657 __ string_compare($str1$$Register, $str2$$Register,
16658 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16659 $tmp1$$Register, $tmp2$$Register,
16660 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16661 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16662 %}
16663 ins_pipe(pipe_class_memory);
16664 %}
16665
16666 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16667 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16668 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16669 %{
16670 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16671 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16672 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16673 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16674
16675 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16676 ins_encode %{
16677 __ string_compare($str1$$Register, $str2$$Register,
16678 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16679 $tmp1$$Register, $tmp2$$Register,
16680 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16681 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16682 %}
16683 ins_pipe(pipe_class_memory);
16684 %}
16685
16686 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16687 // these string_compare variants as NEON register type for convenience so that the prototype of
16688 // string_compare can be shared with all variants.
16689
16690 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16691 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16692 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16693 pRegGov_P1 pgtmp2, rFlagsReg cr)
16694 %{
16695 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16696 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16697 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16698 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16699
16700 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16701 ins_encode %{
16702 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16703 __ string_compare($str1$$Register, $str2$$Register,
16704 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16705 $tmp1$$Register, $tmp2$$Register,
16706 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16707 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16708 StrIntrinsicNode::LL);
16709 %}
16710 ins_pipe(pipe_class_memory);
16711 %}
16712
16713 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16714 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16715 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16716 pRegGov_P1 pgtmp2, rFlagsReg cr)
16717 %{
16718 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16719 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16720 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16721 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16722
16723 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16724 ins_encode %{
16725 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16726 __ string_compare($str1$$Register, $str2$$Register,
16727 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16728 $tmp1$$Register, $tmp2$$Register,
16729 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16730 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16731 StrIntrinsicNode::LU);
16732 %}
16733 ins_pipe(pipe_class_memory);
16734 %}
16735
16736 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16737 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16738 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16739 pRegGov_P1 pgtmp2, rFlagsReg cr)
16740 %{
16741 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16742 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16743 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16744 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16745
16746 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16747 ins_encode %{
16748 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16749 __ string_compare($str1$$Register, $str2$$Register,
16750 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16751 $tmp1$$Register, $tmp2$$Register,
16752 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16753 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16754 StrIntrinsicNode::UL);
16755 %}
16756 ins_pipe(pipe_class_memory);
16757 %}
16758
16759 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16760 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16761 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16762 pRegGov_P1 pgtmp2, rFlagsReg cr)
16763 %{
16764 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16765 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16766 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16767 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16768
16769 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16770 ins_encode %{
16771 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16772 __ string_compare($str1$$Register, $str2$$Register,
16773 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16774 $tmp1$$Register, $tmp2$$Register,
16775 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16776 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16777 StrIntrinsicNode::UU);
16778 %}
16779 ins_pipe(pipe_class_memory);
16780 %}
16781
16782 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16783 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16784 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16785 %{
16786 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16787 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16788 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16789 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16790 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16791
16792 ins_encode %{
16793 __ string_indexof($str1$$Register, $str2$$Register,
16794 $cnt1$$Register, $cnt2$$Register,
16795 $tmp1$$Register, $tmp2$$Register,
16796 $tmp3$$Register, $tmp4$$Register,
16797 $tmp5$$Register, $tmp6$$Register,
16798 -1, $result$$Register, StrIntrinsicNode::UU);
16799 %}
16800 ins_pipe(pipe_class_memory);
16801 %}
16802
16803 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16804 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16805 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16806 %{
16807 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16808 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16809 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16810 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16811 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16812
16813 ins_encode %{
16814 __ string_indexof($str1$$Register, $str2$$Register,
16815 $cnt1$$Register, $cnt2$$Register,
16816 $tmp1$$Register, $tmp2$$Register,
16817 $tmp3$$Register, $tmp4$$Register,
16818 $tmp5$$Register, $tmp6$$Register,
16819 -1, $result$$Register, StrIntrinsicNode::LL);
16820 %}
16821 ins_pipe(pipe_class_memory);
16822 %}
16823
16824 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16825 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16826 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16827 %{
16828 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16829 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16830 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16831 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16832 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16833
16834 ins_encode %{
16835 __ string_indexof($str1$$Register, $str2$$Register,
16836 $cnt1$$Register, $cnt2$$Register,
16837 $tmp1$$Register, $tmp2$$Register,
16838 $tmp3$$Register, $tmp4$$Register,
16839 $tmp5$$Register, $tmp6$$Register,
16840 -1, $result$$Register, StrIntrinsicNode::UL);
16841 %}
16842 ins_pipe(pipe_class_memory);
16843 %}
16844
16845 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16846 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16847 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16848 %{
16849 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16850 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16851 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16852 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16853 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16854
16855 ins_encode %{
16856 int icnt2 = (int)$int_cnt2$$constant;
16857 __ string_indexof($str1$$Register, $str2$$Register,
16858 $cnt1$$Register, zr,
16859 $tmp1$$Register, $tmp2$$Register,
16860 $tmp3$$Register, $tmp4$$Register, zr, zr,
16861 icnt2, $result$$Register, StrIntrinsicNode::UU);
16862 %}
16863 ins_pipe(pipe_class_memory);
16864 %}
16865
16866 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16867 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16868 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16869 %{
16870 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16871 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16872 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16873 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16874 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16875
16876 ins_encode %{
16877 int icnt2 = (int)$int_cnt2$$constant;
16878 __ string_indexof($str1$$Register, $str2$$Register,
16879 $cnt1$$Register, zr,
16880 $tmp1$$Register, $tmp2$$Register,
16881 $tmp3$$Register, $tmp4$$Register, zr, zr,
16882 icnt2, $result$$Register, StrIntrinsicNode::LL);
16883 %}
16884 ins_pipe(pipe_class_memory);
16885 %}
16886
16887 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16888 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16889 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16890 %{
16891 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16892 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16893 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16894 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16895 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16896
16897 ins_encode %{
16898 int icnt2 = (int)$int_cnt2$$constant;
16899 __ string_indexof($str1$$Register, $str2$$Register,
16900 $cnt1$$Register, zr,
16901 $tmp1$$Register, $tmp2$$Register,
16902 $tmp3$$Register, $tmp4$$Register, zr, zr,
16903 icnt2, $result$$Register, StrIntrinsicNode::UL);
16904 %}
16905 ins_pipe(pipe_class_memory);
16906 %}
16907
16908 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16909 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16910 iRegINoSp tmp3, rFlagsReg cr)
16911 %{
16912 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16913 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16914 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16915 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16916
16917 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16918
16919 ins_encode %{
16920 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16921 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16922 $tmp3$$Register);
16923 %}
16924 ins_pipe(pipe_class_memory);
16925 %}
16926
16927 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16928 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16929 iRegINoSp tmp3, rFlagsReg cr)
16930 %{
16931 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16932 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16933 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16934 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16935
16936 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16937
16938 ins_encode %{
16939 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16940 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16941 $tmp3$$Register);
16942 %}
16943 ins_pipe(pipe_class_memory);
16944 %}
16945
16946 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16947 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16948 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16949 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16950 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16951 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16952 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16953 ins_encode %{
16954 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16955 $result$$Register, $ztmp1$$FloatRegister,
16956 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16957 $ptmp$$PRegister, true /* isL */);
16958 %}
16959 ins_pipe(pipe_class_memory);
16960 %}
16961
16962 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16963 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16964 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16965 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16966 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16967 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16968 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16969 ins_encode %{
16970 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16971 $result$$Register, $ztmp1$$FloatRegister,
16972 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16973 $ptmp$$PRegister, false /* isL */);
16974 %}
16975 ins_pipe(pipe_class_memory);
16976 %}
16977
16978 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16979 iRegI_R0 result, rFlagsReg cr)
16980 %{
16981 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16982 match(Set result (StrEquals (Binary str1 str2) cnt));
16983 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16984
16985 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16986 ins_encode %{
16987 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16988 __ string_equals($str1$$Register, $str2$$Register,
16989 $result$$Register, $cnt$$Register, 1);
16990 %}
16991 ins_pipe(pipe_class_memory);
16992 %}
16993
16994 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16995 iRegI_R0 result, rFlagsReg cr)
16996 %{
16997 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16998 match(Set result (StrEquals (Binary str1 str2) cnt));
16999 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17000
17001 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17002 ins_encode %{
17003 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17004 __ string_equals($str1$$Register, $str2$$Register,
17005 $result$$Register, $cnt$$Register, 2);
17006 %}
17007 ins_pipe(pipe_class_memory);
17008 %}
17009
17010 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17011 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17012 iRegP_R10 tmp, rFlagsReg cr)
17013 %{
17014 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17015 match(Set result (AryEq ary1 ary2));
17016 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17017
17018 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
17019 ins_encode %{
17020 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17021 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17022 $result$$Register, $tmp$$Register, 1);
17023 if (tpc == NULL) {
17024 ciEnv::current()->record_failure("CodeCache is full");
17025 return;
17026 }
17027 %}
17028 ins_pipe(pipe_class_memory);
17029 %}
17030
17031 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17032 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17033 iRegP_R10 tmp, rFlagsReg cr)
17034 %{
17035 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17036 match(Set result (AryEq ary1 ary2));
17037 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17038
17039 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
17040 ins_encode %{
17041 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17042 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17043 $result$$Register, $tmp$$Register, 2);
17044 if (tpc == NULL) {
17045 ciEnv::current()->record_failure("CodeCache is full");
17046 return;
17047 }
17048 %}
17049 ins_pipe(pipe_class_memory);
17050 %}
17051
17052 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17053 %{
17054 match(Set result (CountPositives ary1 len));
17055 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17056 format %{ "count positives byte[] $ary1,$len -> $result" %}
17057 ins_encode %{
17058 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17059 if (tpc == NULL) {
17060 ciEnv::current()->record_failure("CodeCache is full");
17061 return;
17062 }
17063 %}
17064 ins_pipe( pipe_slow );
17065 %}
17066
17067 // fast char[] to byte[] compression
17068 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17069 vRegD_V0 tmp1, vRegD_V1 tmp2,
17070 vRegD_V2 tmp3, vRegD_V3 tmp4,
17071 iRegI_R0 result, rFlagsReg cr)
17072 %{
17073 match(Set result (StrCompressedCopy src (Binary dst len)));
17074 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
17075 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17076
17077 format %{ "String Compress $src,$dst,$len -> $result // KILL $src,$dst" %}
17078 ins_encode %{
17079 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17080 $result$$Register,
17081 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17082 $tmp3$$FloatRegister, $tmp4$$FloatRegister);
17083 %}
17084 ins_pipe(pipe_slow);
17085 %}
17086
17087 // fast byte[] to char[] inflation
17088 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17089 vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17090 %{
17091 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17092 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17093
17094 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
17095 ins_encode %{
17096 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17097 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17098 $tmp3$$FloatRegister, $tmp4$$Register);
17099 if (tpc == NULL) {
17100 ciEnv::current()->record_failure("CodeCache is full");
17101 return;
17102 }
17103 %}
17104 ins_pipe(pipe_class_memory);
17105 %}
17106
17107 // encode char[] to byte[] in ISO_8859_1
17108 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17109 vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17110 vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17111 iRegI_R0 result, rFlagsReg cr)
17112 %{
17113 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17114 match(Set result (EncodeISOArray src (Binary dst len)));
17115 effect(USE_KILL src, USE_KILL dst, USE len,
17116 KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17117
17118 format %{ "Encode ISO array $src,$dst,$len -> $result" %}
17119 ins_encode %{
17120 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17121 $result$$Register, false,
17122 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17123 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17124 %}
17125 ins_pipe(pipe_class_memory);
17126 %}
17127
17128 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17129 vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17130 vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17131 iRegI_R0 result, rFlagsReg cr)
17132 %{
17133 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17134 match(Set result (EncodeISOArray src (Binary dst len)));
17135 effect(USE_KILL src, USE_KILL dst, USE len,
17136 KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17137
17138 format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
17139 ins_encode %{
17140 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17141 $result$$Register, true,
17142 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17143 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17144 %}
17145 ins_pipe(pipe_class_memory);
17146 %}
17147
17148 // ============================================================================
17149 // This name is KNOWN by the ADLC and cannot be changed.
17150 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17151 // for this guy.
17152 instruct tlsLoadP(thread_RegP dst)
17153 %{
17154 match(Set dst (ThreadLocal));
17155
17156 ins_cost(0);
17157
17158 format %{ " -- \t// $dst=Thread::current(), empty" %}
17159
17160 size(0);
17161
17162 ins_encode( /*empty*/ );
17163
17164 ins_pipe(pipe_class_empty);
17165 %}
17166
17167 //----------PEEPHOLE RULES-----------------------------------------------------
17168 // These must follow all instruction definitions as they use the names
17169 // defined in the instructions definitions.
17170 //
17171 // peepmatch ( root_instr_name [preceding_instruction]* );
17172 //
17173 // peepconstraint %{
17174 // (instruction_number.operand_name relational_op instruction_number.operand_name
17175 // [, ...] );
17176 // // instruction numbers are zero-based using left to right order in peepmatch
17177 //
17178 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17179 // // provide an instruction_number.operand_name for each operand that appears
17180 // // in the replacement instruction's match rule
17181 //
17182 // ---------VM FLAGS---------------------------------------------------------
17183 //
17184 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17185 //
17186 // Each peephole rule is given an identifying number starting with zero and
17187 // increasing by one in the order seen by the parser. An individual peephole
17188 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17189 // on the command-line.
17190 //
17191 // ---------CURRENT LIMITATIONS----------------------------------------------
17192 //
17193 // Only match adjacent instructions in same basic block
17194 // Only equality constraints
17195 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17196 // Only one replacement instruction
17197 //
17198 // ---------EXAMPLE----------------------------------------------------------
17199 //
17200 // // pertinent parts of existing instructions in architecture description
17201 // instruct movI(iRegINoSp dst, iRegI src)
17202 // %{
17203 // match(Set dst (CopyI src));
17204 // %}
17205 //
17206 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17207 // %{
17208 // match(Set dst (AddI dst src));
17209 // effect(KILL cr);
17210 // %}
17211 //
17212 // // Change (inc mov) to lea
17213 // peephole %{
17214 // // increment preceded by register-register move
17215 // peepmatch ( incI_iReg movI );
17216 // // require that the destination register of the increment
17217 // // match the destination register of the move
17218 // peepconstraint ( 0.dst == 1.dst );
17219 // // construct a replacement instruction that sets
17220 // // the destination to ( move's source register + one )
17221 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17222 // %}
17223 //
17224
17225 // Implementation no longer uses movX instructions since
17226 // machine-independent system no longer uses CopyX nodes.
17227 //
17228 // peephole
17229 // %{
17230 // peepmatch (incI_iReg movI);
17231 // peepconstraint (0.dst == 1.dst);
17232 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17233 // %}
17234
17235 // peephole
17236 // %{
17237 // peepmatch (decI_iReg movI);
17238 // peepconstraint (0.dst == 1.dst);
17239 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17240 // %}
17241
17242 // peephole
17243 // %{
17244 // peepmatch (addI_iReg_imm movI);
17245 // peepconstraint (0.dst == 1.dst);
17246 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17247 // %}
17248
17249 // peephole
17250 // %{
17251 // peepmatch (incL_iReg movL);
17252 // peepconstraint (0.dst == 1.dst);
17253 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17254 // %}
17255
17256 // peephole
17257 // %{
17258 // peepmatch (decL_iReg movL);
17259 // peepconstraint (0.dst == 1.dst);
17260 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17261 // %}
17262
17263 // peephole
17264 // %{
17265 // peepmatch (addL_iReg_imm movL);
17266 // peepconstraint (0.dst == 1.dst);
17267 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17268 // %}
17269
17270 // peephole
17271 // %{
17272 // peepmatch (addP_iReg_imm movP);
17273 // peepconstraint (0.dst == 1.dst);
17274 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17275 // %}
17276
17277 // // Change load of spilled value to only a spill
17278 // instruct storeI(memory mem, iRegI src)
17279 // %{
17280 // match(Set mem (StoreI mem src));
17281 // %}
17282 //
17283 // instruct loadI(iRegINoSp dst, memory mem)
17284 // %{
17285 // match(Set dst (LoadI mem));
17286 // %}
17287 //
17288
17289 //----------SMARTSPILL RULES---------------------------------------------------
17290 // These must follow all instruction definitions as they use the names
17291 // defined in the instructions definitions.
17292
17293 // Local Variables:
17294 // mode: c++
17295 // End: