1 //
2 // Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all float registers
698 reg_class float_reg(
699 V0,
700 V1,
701 V2,
702 V3,
703 V4,
704 V5,
705 V6,
706 V7,
707 V8,
708 V9,
709 V10,
710 V11,
711 V12,
712 V13,
713 V14,
714 V15,
715 V16,
716 V17,
717 V18,
718 V19,
719 V20,
720 V21,
721 V22,
722 V23,
723 V24,
724 V25,
725 V26,
726 V27,
727 V28,
728 V29,
729 V30,
730 V31
731 );
732
733 // Double precision float registers have virtual `high halves' that
734 // are needed by the allocator.
735 // Class for all double registers
736 reg_class double_reg(
737 V0, V0_H,
738 V1, V1_H,
739 V2, V2_H,
740 V3, V3_H,
741 V4, V4_H,
742 V5, V5_H,
743 V6, V6_H,
744 V7, V7_H,
745 V8, V8_H,
746 V9, V9_H,
747 V10, V10_H,
748 V11, V11_H,
749 V12, V12_H,
750 V13, V13_H,
751 V14, V14_H,
752 V15, V15_H,
753 V16, V16_H,
754 V17, V17_H,
755 V18, V18_H,
756 V19, V19_H,
757 V20, V20_H,
758 V21, V21_H,
759 V22, V22_H,
760 V23, V23_H,
761 V24, V24_H,
762 V25, V25_H,
763 V26, V26_H,
764 V27, V27_H,
765 V28, V28_H,
766 V29, V29_H,
767 V30, V30_H,
768 V31, V31_H
769 );
770
771 // Class for all SVE vector registers.
772 reg_class vectora_reg (
773 V0, V0_H, V0_J, V0_K,
774 V1, V1_H, V1_J, V1_K,
775 V2, V2_H, V2_J, V2_K,
776 V3, V3_H, V3_J, V3_K,
777 V4, V4_H, V4_J, V4_K,
778 V5, V5_H, V5_J, V5_K,
779 V6, V6_H, V6_J, V6_K,
780 V7, V7_H, V7_J, V7_K,
781 V8, V8_H, V8_J, V8_K,
782 V9, V9_H, V9_J, V9_K,
783 V10, V10_H, V10_J, V10_K,
784 V11, V11_H, V11_J, V11_K,
785 V12, V12_H, V12_J, V12_K,
786 V13, V13_H, V13_J, V13_K,
787 V14, V14_H, V14_J, V14_K,
788 V15, V15_H, V15_J, V15_K,
789 V16, V16_H, V16_J, V16_K,
790 V17, V17_H, V17_J, V17_K,
791 V18, V18_H, V18_J, V18_K,
792 V19, V19_H, V19_J, V19_K,
793 V20, V20_H, V20_J, V20_K,
794 V21, V21_H, V21_J, V21_K,
795 V22, V22_H, V22_J, V22_K,
796 V23, V23_H, V23_J, V23_K,
797 V24, V24_H, V24_J, V24_K,
798 V25, V25_H, V25_J, V25_K,
799 V26, V26_H, V26_J, V26_K,
800 V27, V27_H, V27_J, V27_K,
801 V28, V28_H, V28_J, V28_K,
802 V29, V29_H, V29_J, V29_K,
803 V30, V30_H, V30_J, V30_K,
804 V31, V31_H, V31_J, V31_K,
805 );
806
807 // Class for all 64bit vector registers
808 reg_class vectord_reg(
809 V0, V0_H,
810 V1, V1_H,
811 V2, V2_H,
812 V3, V3_H,
813 V4, V4_H,
814 V5, V5_H,
815 V6, V6_H,
816 V7, V7_H,
817 V8, V8_H,
818 V9, V9_H,
819 V10, V10_H,
820 V11, V11_H,
821 V12, V12_H,
822 V13, V13_H,
823 V14, V14_H,
824 V15, V15_H,
825 V16, V16_H,
826 V17, V17_H,
827 V18, V18_H,
828 V19, V19_H,
829 V20, V20_H,
830 V21, V21_H,
831 V22, V22_H,
832 V23, V23_H,
833 V24, V24_H,
834 V25, V25_H,
835 V26, V26_H,
836 V27, V27_H,
837 V28, V28_H,
838 V29, V29_H,
839 V30, V30_H,
840 V31, V31_H
841 );
842
843 // Class for all 128bit vector registers
844 reg_class vectorx_reg(
845 V0, V0_H, V0_J, V0_K,
846 V1, V1_H, V1_J, V1_K,
847 V2, V2_H, V2_J, V2_K,
848 V3, V3_H, V3_J, V3_K,
849 V4, V4_H, V4_J, V4_K,
850 V5, V5_H, V5_J, V5_K,
851 V6, V6_H, V6_J, V6_K,
852 V7, V7_H, V7_J, V7_K,
853 V8, V8_H, V8_J, V8_K,
854 V9, V9_H, V9_J, V9_K,
855 V10, V10_H, V10_J, V10_K,
856 V11, V11_H, V11_J, V11_K,
857 V12, V12_H, V12_J, V12_K,
858 V13, V13_H, V13_J, V13_K,
859 V14, V14_H, V14_J, V14_K,
860 V15, V15_H, V15_J, V15_K,
861 V16, V16_H, V16_J, V16_K,
862 V17, V17_H, V17_J, V17_K,
863 V18, V18_H, V18_J, V18_K,
864 V19, V19_H, V19_J, V19_K,
865 V20, V20_H, V20_J, V20_K,
866 V21, V21_H, V21_J, V21_K,
867 V22, V22_H, V22_J, V22_K,
868 V23, V23_H, V23_J, V23_K,
869 V24, V24_H, V24_J, V24_K,
870 V25, V25_H, V25_J, V25_K,
871 V26, V26_H, V26_J, V26_K,
872 V27, V27_H, V27_J, V27_K,
873 V28, V28_H, V28_J, V28_K,
874 V29, V29_H, V29_J, V29_K,
875 V30, V30_H, V30_J, V30_K,
876 V31, V31_H, V31_J, V31_K
877 );
878
879 // Class for 128 bit register v0
880 reg_class v0_reg(
881 V0, V0_H
882 );
883
884 // Class for 128 bit register v1
885 reg_class v1_reg(
886 V1, V1_H
887 );
888
889 // Class for 128 bit register v2
890 reg_class v2_reg(
891 V2, V2_H
892 );
893
894 // Class for 128 bit register v3
895 reg_class v3_reg(
896 V3, V3_H
897 );
898
899 // Class for 128 bit register v4
900 reg_class v4_reg(
901 V4, V4_H
902 );
903
904 // Class for 128 bit register v5
905 reg_class v5_reg(
906 V5, V5_H
907 );
908
909 // Class for 128 bit register v6
910 reg_class v6_reg(
911 V6, V6_H
912 );
913
914 // Class for 128 bit register v7
915 reg_class v7_reg(
916 V7, V7_H
917 );
918
919 // Class for 128 bit register v8
920 reg_class v8_reg(
921 V8, V8_H
922 );
923
924 // Class for 128 bit register v9
925 reg_class v9_reg(
926 V9, V9_H
927 );
928
929 // Class for 128 bit register v10
930 reg_class v10_reg(
931 V10, V10_H
932 );
933
934 // Class for 128 bit register v11
935 reg_class v11_reg(
936 V11, V11_H
937 );
938
939 // Class for 128 bit register v12
940 reg_class v12_reg(
941 V12, V12_H
942 );
943
944 // Class for 128 bit register v13
945 reg_class v13_reg(
946 V13, V13_H
947 );
948
949 // Class for 128 bit register v14
950 reg_class v14_reg(
951 V14, V14_H
952 );
953
954 // Class for 128 bit register v15
955 reg_class v15_reg(
956 V15, V15_H
957 );
958
959 // Class for 128 bit register v16
960 reg_class v16_reg(
961 V16, V16_H
962 );
963
964 // Class for 128 bit register v17
965 reg_class v17_reg(
966 V17, V17_H
967 );
968
969 // Class for 128 bit register v18
970 reg_class v18_reg(
971 V18, V18_H
972 );
973
974 // Class for 128 bit register v19
975 reg_class v19_reg(
976 V19, V19_H
977 );
978
979 // Class for 128 bit register v20
980 reg_class v20_reg(
981 V20, V20_H
982 );
983
984 // Class for 128 bit register v21
985 reg_class v21_reg(
986 V21, V21_H
987 );
988
989 // Class for 128 bit register v22
990 reg_class v22_reg(
991 V22, V22_H
992 );
993
994 // Class for 128 bit register v23
995 reg_class v23_reg(
996 V23, V23_H
997 );
998
999 // Class for 128 bit register v24
1000 reg_class v24_reg(
1001 V24, V24_H
1002 );
1003
1004 // Class for 128 bit register v25
1005 reg_class v25_reg(
1006 V25, V25_H
1007 );
1008
1009 // Class for 128 bit register v26
1010 reg_class v26_reg(
1011 V26, V26_H
1012 );
1013
1014 // Class for 128 bit register v27
1015 reg_class v27_reg(
1016 V27, V27_H
1017 );
1018
1019 // Class for 128 bit register v28
1020 reg_class v28_reg(
1021 V28, V28_H
1022 );
1023
1024 // Class for 128 bit register v29
1025 reg_class v29_reg(
1026 V29, V29_H
1027 );
1028
1029 // Class for 128 bit register v30
1030 reg_class v30_reg(
1031 V30, V30_H
1032 );
1033
1034 // Class for 128 bit register v31
1035 reg_class v31_reg(
1036 V31, V31_H
1037 );
1038
1039 // Class for all SVE predicate registers.
1040 reg_class pr_reg (
1041 P0,
1042 P1,
1043 P2,
1044 P3,
1045 P4,
1046 P5,
1047 P6,
1048 // P7, non-allocatable, preserved with all elements preset to TRUE.
1049 P8,
1050 P9,
1051 P10,
1052 P11,
1053 P12,
1054 P13,
1055 P14,
1056 P15
1057 );
1058
1059 // Class for SVE governing predicate registers, which are used
1060 // to determine the active elements of a predicated instruction.
1061 reg_class gov_pr (
1062 P0,
1063 P1,
1064 P2,
1065 P3,
1066 P4,
1067 P5,
1068 P6,
1069 // P7, non-allocatable, preserved with all elements preset to TRUE.
1070 );
1071
1072 reg_class p0_reg(P0);
1073 reg_class p1_reg(P1);
1074
1075 // Singleton class for condition codes
1076 reg_class int_flags(RFLAGS);
1077
1078 %}
1079
1080 //----------DEFINITION BLOCK---------------------------------------------------
1081 // Define name --> value mappings to inform the ADLC of an integer valued name
1082 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1083 // Format:
1084 // int_def <name> ( <int_value>, <expression>);
1085 // Generated Code in ad_<arch>.hpp
1086 // #define <name> (<expression>)
1087 // // value == <int_value>
1088 // Generated code in ad_<arch>.cpp adlc_verification()
1089 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1090 //
1091
1092 // we follow the ppc-aix port in using a simple cost model which ranks
1093 // register operations as cheap, memory ops as more expensive and
1094 // branches as most expensive. the first two have a low as well as a
1095 // normal cost. huge cost appears to be a way of saying don't do
1096 // something
1097
1098 definitions %{
1099 // The default cost (of a register move instruction).
1100 int_def INSN_COST ( 100, 100);
1101 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1102 int_def CALL_COST ( 200, 2 * INSN_COST);
1103 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1104 %}
1105
1106
1107 //----------SOURCE BLOCK-------------------------------------------------------
1108 // This is a block of C++ code which provides values, functions, and
1109 // definitions necessary in the rest of the architecture description
1110
1111 source_hpp %{
1112
1113 #include "asm/macroAssembler.hpp"
1114 #include "gc/shared/barrierSetAssembler.hpp"
1115 #include "gc/shared/cardTable.hpp"
1116 #include "gc/shared/cardTableBarrierSet.hpp"
1117 #include "gc/shared/collectedHeap.hpp"
1118 #include "opto/addnode.hpp"
1119 #include "opto/convertnode.hpp"
1120 #include "runtime/objectMonitor.hpp"
1121
1122 extern RegMask _ANY_REG32_mask;
1123 extern RegMask _ANY_REG_mask;
1124 extern RegMask _PTR_REG_mask;
1125 extern RegMask _NO_SPECIAL_REG32_mask;
1126 extern RegMask _NO_SPECIAL_REG_mask;
1127 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1128
1129 class CallStubImpl {
1130
1131 //--------------------------------------------------------------
1132 //---< Used for optimization in Compile::shorten_branches >---
1133 //--------------------------------------------------------------
1134
1135 public:
1136 // Size of call trampoline stub.
1137 static uint size_call_trampoline() {
1138 return 0; // no call trampolines on this platform
1139 }
1140
1141 // number of relocations needed by a call trampoline stub
1142 static uint reloc_call_trampoline() {
1143 return 0; // no call trampolines on this platform
1144 }
1145 };
1146
1147 class HandlerImpl {
1148
1149 public:
1150
1151 static int emit_exception_handler(CodeBuffer &cbuf);
1152 static int emit_deopt_handler(CodeBuffer& cbuf);
1153
1154 static uint size_exception_handler() {
1155 return MacroAssembler::far_codestub_branch_size();
1156 }
1157
1158 static uint size_deopt_handler() {
1159 // count one adr and one far branch instruction
1160 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1161 }
1162 };
1163
1164 class Node::PD {
1165 public:
1166 enum NodeFlags {
1167 _last_flag = Node::_last_flag
1168 };
1169 };
1170
1171 bool is_CAS(int opcode, bool maybe_volatile);
1172
1173 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1174
1175 bool unnecessary_acquire(const Node *barrier);
1176 bool needs_acquiring_load(const Node *load);
1177
1178 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1179
1180 bool unnecessary_release(const Node *barrier);
1181 bool unnecessary_volatile(const Node *barrier);
1182 bool needs_releasing_store(const Node *store);
1183
1184 // predicate controlling translation of CompareAndSwapX
1185 bool needs_acquiring_load_exclusive(const Node *load);
1186
1187 // predicate controlling addressing modes
1188 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1189
1190 // Convert BootTest condition to Assembler condition.
1191 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1192 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1193 %}
1194
1195 source %{
1196
1197 // Derived RegMask with conditionally allocatable registers
1198
1199 void PhaseOutput::pd_perform_mach_node_analysis() {
1200 }
1201
1202 int MachNode::pd_alignment_required() const {
1203 return 1;
1204 }
1205
1206 int MachNode::compute_padding(int current_offset) const {
1207 return 0;
1208 }
1209
1210 RegMask _ANY_REG32_mask;
1211 RegMask _ANY_REG_mask;
1212 RegMask _PTR_REG_mask;
1213 RegMask _NO_SPECIAL_REG32_mask;
1214 RegMask _NO_SPECIAL_REG_mask;
1215 RegMask _NO_SPECIAL_PTR_REG_mask;
1216
1217 void reg_mask_init() {
1218 // We derive below RegMask(s) from the ones which are auto-generated from
1219 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1220 // registers conditionally reserved.
1221
1222 _ANY_REG32_mask = _ALL_REG32_mask;
1223 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1224
1225 _ANY_REG_mask = _ALL_REG_mask;
1226
1227 _PTR_REG_mask = _ALL_REG_mask;
1228
1229 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1230 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1231
1232 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1233 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1234
1235 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1236 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1237
1238 // r27 is not allocatable when compressed oops is on and heapbase is not
1239 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1240 if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
1241 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1242 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1243 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1244 }
1245
1246 // r29 is not allocatable when PreserveFramePointer is on
1247 if (PreserveFramePointer) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1251 }
1252 }
1253
1254 // Optimizaton of volatile gets and puts
1255 // -------------------------------------
1256 //
1257 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1258 // use to implement volatile reads and writes. For a volatile read
1259 // we simply need
1260 //
1261 // ldar<x>
1262 //
1263 // and for a volatile write we need
1264 //
1265 // stlr<x>
1266 //
1267 // Alternatively, we can implement them by pairing a normal
1268 // load/store with a memory barrier. For a volatile read we need
1269 //
1270 // ldr<x>
1271 // dmb ishld
1272 //
1273 // for a volatile write
1274 //
1275 // dmb ish
1276 // str<x>
1277 // dmb ish
1278 //
1279 // We can also use ldaxr and stlxr to implement compare and swap CAS
1280 // sequences. These are normally translated to an instruction
1281 // sequence like the following
1282 //
1283 // dmb ish
1284 // retry:
1285 // ldxr<x> rval raddr
1286 // cmp rval rold
1287 // b.ne done
1288 // stlxr<x> rval, rnew, rold
1289 // cbnz rval retry
1290 // done:
1291 // cset r0, eq
1292 // dmb ishld
1293 //
1294 // Note that the exclusive store is already using an stlxr
1295 // instruction. That is required to ensure visibility to other
1296 // threads of the exclusive write (assuming it succeeds) before that
1297 // of any subsequent writes.
1298 //
1299 // The following instruction sequence is an improvement on the above
1300 //
1301 // retry:
1302 // ldaxr<x> rval raddr
1303 // cmp rval rold
1304 // b.ne done
1305 // stlxr<x> rval, rnew, rold
1306 // cbnz rval retry
1307 // done:
1308 // cset r0, eq
1309 //
1310 // We don't need the leading dmb ish since the stlxr guarantees
1311 // visibility of prior writes in the case that the swap is
1312 // successful. Crucially we don't have to worry about the case where
1313 // the swap is not successful since no valid program should be
1314 // relying on visibility of prior changes by the attempting thread
1315 // in the case where the CAS fails.
1316 //
1317 // Similarly, we don't need the trailing dmb ishld if we substitute
1318 // an ldaxr instruction since that will provide all the guarantees we
1319 // require regarding observation of changes made by other threads
1320 // before any change to the CAS address observed by the load.
1321 //
1322 // In order to generate the desired instruction sequence we need to
1323 // be able to identify specific 'signature' ideal graph node
1324 // sequences which i) occur as a translation of a volatile reads or
1325 // writes or CAS operations and ii) do not occur through any other
1326 // translation or graph transformation. We can then provide
1327 // alternative aldc matching rules which translate these node
1328 // sequences to the desired machine code sequences. Selection of the
1329 // alternative rules can be implemented by predicates which identify
1330 // the relevant node sequences.
1331 //
1332 // The ideal graph generator translates a volatile read to the node
1333 // sequence
1334 //
1335 // LoadX[mo_acquire]
1336 // MemBarAcquire
1337 //
1338 // As a special case when using the compressed oops optimization we
1339 // may also see this variant
1340 //
1341 // LoadN[mo_acquire]
1342 // DecodeN
1343 // MemBarAcquire
1344 //
1345 // A volatile write is translated to the node sequence
1346 //
1347 // MemBarRelease
1348 // StoreX[mo_release] {CardMark}-optional
1349 // MemBarVolatile
1350 //
1351 // n.b. the above node patterns are generated with a strict
1352 // 'signature' configuration of input and output dependencies (see
1353 // the predicates below for exact details). The card mark may be as
1354 // simple as a few extra nodes or, in a few GC configurations, may
1355 // include more complex control flow between the leading and
1356 // trailing memory barriers. However, whatever the card mark
1357 // configuration these signatures are unique to translated volatile
1358 // reads/stores -- they will not appear as a result of any other
1359 // bytecode translation or inlining nor as a consequence of
1360 // optimizing transforms.
1361 //
1362 // We also want to catch inlined unsafe volatile gets and puts and
1363 // be able to implement them using either ldar<x>/stlr<x> or some
1364 // combination of ldr<x>/stlr<x> and dmb instructions.
1365 //
1366 // Inlined unsafe volatiles puts manifest as a minor variant of the
1367 // normal volatile put node sequence containing an extra cpuorder
1368 // membar
1369 //
1370 // MemBarRelease
1371 // MemBarCPUOrder
1372 // StoreX[mo_release] {CardMark}-optional
1373 // MemBarCPUOrder
1374 // MemBarVolatile
1375 //
1376 // n.b. as an aside, a cpuorder membar is not itself subject to
1377 // matching and translation by adlc rules. However, the rule
1378 // predicates need to detect its presence in order to correctly
1379 // select the desired adlc rules.
1380 //
1381 // Inlined unsafe volatile gets manifest as a slightly different
1382 // node sequence to a normal volatile get because of the
1383 // introduction of some CPUOrder memory barriers to bracket the
1384 // Load. However, but the same basic skeleton of a LoadX feeding a
1385 // MemBarAcquire, possibly through an optional DecodeN, is still
1386 // present
1387 //
1388 // MemBarCPUOrder
1389 // || \\
1390 // MemBarCPUOrder LoadX[mo_acquire]
1391 // || |
1392 // || {DecodeN} optional
1393 // || /
1394 // MemBarAcquire
1395 //
1396 // In this case the acquire membar does not directly depend on the
1397 // load. However, we can be sure that the load is generated from an
1398 // inlined unsafe volatile get if we see it dependent on this unique
1399 // sequence of membar nodes. Similarly, given an acquire membar we
1400 // can know that it was added because of an inlined unsafe volatile
1401 // get if it is fed and feeds a cpuorder membar and if its feed
1402 // membar also feeds an acquiring load.
1403 //
1404 // Finally an inlined (Unsafe) CAS operation is translated to the
1405 // following ideal graph
1406 //
1407 // MemBarRelease
1408 // MemBarCPUOrder
1409 // CompareAndSwapX {CardMark}-optional
1410 // MemBarCPUOrder
1411 // MemBarAcquire
1412 //
1413 // So, where we can identify these volatile read and write
1414 // signatures we can choose to plant either of the above two code
1415 // sequences. For a volatile read we can simply plant a normal
1416 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1417 // also choose to inhibit translation of the MemBarAcquire and
1418 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1419 //
1420 // When we recognise a volatile store signature we can choose to
1421 // plant at a dmb ish as a translation for the MemBarRelease, a
1422 // normal str<x> and then a dmb ish for the MemBarVolatile.
1423 // Alternatively, we can inhibit translation of the MemBarRelease
1424 // and MemBarVolatile and instead plant a simple stlr<x>
1425 // instruction.
1426 //
1427 // when we recognise a CAS signature we can choose to plant a dmb
1428 // ish as a translation for the MemBarRelease, the conventional
1429 // macro-instruction sequence for the CompareAndSwap node (which
1430 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1431 // Alternatively, we can elide generation of the dmb instructions
1432 // and plant the alternative CompareAndSwap macro-instruction
1433 // sequence (which uses ldaxr<x>).
1434 //
1435 // Of course, the above only applies when we see these signature
1436 // configurations. We still want to plant dmb instructions in any
1437 // other cases where we may see a MemBarAcquire, MemBarRelease or
1438 // MemBarVolatile. For example, at the end of a constructor which
1439 // writes final/volatile fields we will see a MemBarRelease
1440 // instruction and this needs a 'dmb ish' lest we risk the
1441 // constructed object being visible without making the
1442 // final/volatile field writes visible.
1443 //
1444 // n.b. the translation rules below which rely on detection of the
1445 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1446 // If we see anything other than the signature configurations we
1447 // always just translate the loads and stores to ldr<x> and str<x>
1448 // and translate acquire, release and volatile membars to the
1449 // relevant dmb instructions.
1450 //
1451
1452 // is_CAS(int opcode, bool maybe_volatile)
1453 //
1454 // return true if opcode is one of the possible CompareAndSwapX
1455 // values otherwise false.
1456
1457 bool is_CAS(int opcode, bool maybe_volatile)
1458 {
1459 switch(opcode) {
1460 // We handle these
1461 case Op_CompareAndSwapI:
1462 case Op_CompareAndSwapL:
1463 case Op_CompareAndSwapP:
1464 case Op_CompareAndSwapN:
1465 case Op_ShenandoahCompareAndSwapP:
1466 case Op_ShenandoahCompareAndSwapN:
1467 case Op_CompareAndSwapB:
1468 case Op_CompareAndSwapS:
1469 case Op_GetAndSetI:
1470 case Op_GetAndSetL:
1471 case Op_GetAndSetP:
1472 case Op_GetAndSetN:
1473 case Op_GetAndAddI:
1474 case Op_GetAndAddL:
1475 return true;
1476 case Op_CompareAndExchangeI:
1477 case Op_CompareAndExchangeN:
1478 case Op_CompareAndExchangeB:
1479 case Op_CompareAndExchangeS:
1480 case Op_CompareAndExchangeL:
1481 case Op_CompareAndExchangeP:
1482 case Op_WeakCompareAndSwapB:
1483 case Op_WeakCompareAndSwapS:
1484 case Op_WeakCompareAndSwapI:
1485 case Op_WeakCompareAndSwapL:
1486 case Op_WeakCompareAndSwapP:
1487 case Op_WeakCompareAndSwapN:
1488 case Op_ShenandoahWeakCompareAndSwapP:
1489 case Op_ShenandoahWeakCompareAndSwapN:
1490 case Op_ShenandoahCompareAndExchangeP:
1491 case Op_ShenandoahCompareAndExchangeN:
1492 return maybe_volatile;
1493 default:
1494 return false;
1495 }
1496 }
1497
1498 // helper to determine the maximum number of Phi nodes we may need to
1499 // traverse when searching from a card mark membar for the merge mem
1500 // feeding a trailing membar or vice versa
1501
1502 // predicates controlling emit of ldr<x>/ldar<x>
1503
1504 bool unnecessary_acquire(const Node *barrier)
1505 {
1506 assert(barrier->is_MemBar(), "expecting a membar");
1507
1508 MemBarNode* mb = barrier->as_MemBar();
1509
1510 if (mb->trailing_load()) {
1511 return true;
1512 }
1513
1514 if (mb->trailing_load_store()) {
1515 Node* load_store = mb->in(MemBarNode::Precedent);
1516 assert(load_store->is_LoadStore(), "unexpected graph shape");
1517 return is_CAS(load_store->Opcode(), true);
1518 }
1519
1520 return false;
1521 }
1522
1523 bool needs_acquiring_load(const Node *n)
1524 {
1525 assert(n->is_Load(), "expecting a load");
1526 LoadNode *ld = n->as_Load();
1527 return ld->is_acquire();
1528 }
1529
1530 bool unnecessary_release(const Node *n)
1531 {
1532 assert((n->is_MemBar() &&
1533 n->Opcode() == Op_MemBarRelease),
1534 "expecting a release membar");
1535
1536 MemBarNode *barrier = n->as_MemBar();
1537 if (!barrier->leading()) {
1538 return false;
1539 } else {
1540 Node* trailing = barrier->trailing_membar();
1541 MemBarNode* trailing_mb = trailing->as_MemBar();
1542 assert(trailing_mb->trailing(), "Not a trailing membar?");
1543 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1544
1545 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1546 if (mem->is_Store()) {
1547 assert(mem->as_Store()->is_release(), "");
1548 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1549 return true;
1550 } else {
1551 assert(mem->is_LoadStore(), "");
1552 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1553 return is_CAS(mem->Opcode(), true);
1554 }
1555 }
1556 return false;
1557 }
1558
1559 bool unnecessary_volatile(const Node *n)
1560 {
1561 // assert n->is_MemBar();
1562 MemBarNode *mbvol = n->as_MemBar();
1563
1564 bool release = mbvol->trailing_store();
1565 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1566 #ifdef ASSERT
1567 if (release) {
1568 Node* leading = mbvol->leading_membar();
1569 assert(leading->Opcode() == Op_MemBarRelease, "");
1570 assert(leading->as_MemBar()->leading_store(), "");
1571 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1572 }
1573 #endif
1574
1575 return release;
1576 }
1577
1578 // predicates controlling emit of str<x>/stlr<x>
1579
1580 bool needs_releasing_store(const Node *n)
1581 {
1582 // assert n->is_Store();
1583 StoreNode *st = n->as_Store();
1584 return st->trailing_membar() != NULL;
1585 }
1586
1587 // predicate controlling translation of CAS
1588 //
1589 // returns true if CAS needs to use an acquiring load otherwise false
1590
1591 bool needs_acquiring_load_exclusive(const Node *n)
1592 {
1593 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1594 LoadStoreNode* ldst = n->as_LoadStore();
1595 if (is_CAS(n->Opcode(), false)) {
1596 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1597 } else {
1598 return ldst->trailing_membar() != NULL;
1599 }
1600
1601 // so we can just return true here
1602 return true;
1603 }
1604
1605 #define __ _masm.
1606
1607 // advance declarations for helper functions to convert register
1608 // indices to register objects
1609
1610 // the ad file has to provide implementations of certain methods
1611 // expected by the generic code
1612 //
1613 // REQUIRED FUNCTIONALITY
1614
1615 //=============================================================================
1616
1617 // !!!!! Special hack to get all types of calls to specify the byte offset
1618 // from the start of the call to the point where the return address
1619 // will point.
1620
1621 int MachCallStaticJavaNode::ret_addr_offset()
1622 {
1623 // call should be a simple bl
1624 int off = 4;
1625 return off;
1626 }
1627
1628 int MachCallDynamicJavaNode::ret_addr_offset()
1629 {
1630 return 16; // movz, movk, movk, bl
1631 }
1632
1633 int MachCallRuntimeNode::ret_addr_offset() {
1634 // for generated stubs the call will be
1635 // bl(addr)
1636 // or with far branches
1637 // bl(trampoline_stub)
1638 // for real runtime callouts it will be six instructions
1639 // see aarch64_enc_java_to_runtime
1640 // adr(rscratch2, retaddr)
1641 // lea(rscratch1, RuntimeAddress(addr)
1642 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1643 // blr(rscratch1)
1644 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1645 if (cb) {
1646 return 1 * NativeInstruction::instruction_size;
1647 } else {
1648 return 6 * NativeInstruction::instruction_size;
1649 }
1650 }
1651
1652 //=============================================================================
1653
1654 #ifndef PRODUCT
1655 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1656 st->print("BREAKPOINT");
1657 }
1658 #endif
1659
1660 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1661 C2_MacroAssembler _masm(&cbuf);
1662 __ brk(0);
1663 }
1664
1665 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1666 return MachNode::size(ra_);
1667 }
1668
1669 //=============================================================================
1670
1671 #ifndef PRODUCT
1672 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1673 st->print("nop \t# %d bytes pad for loops and calls", _count);
1674 }
1675 #endif
1676
1677 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1678 C2_MacroAssembler _masm(&cbuf);
1679 for (int i = 0; i < _count; i++) {
1680 __ nop();
1681 }
1682 }
1683
1684 uint MachNopNode::size(PhaseRegAlloc*) const {
1685 return _count * NativeInstruction::instruction_size;
1686 }
1687
1688 //=============================================================================
1689 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1690
1691 int ConstantTable::calculate_table_base_offset() const {
1692 return 0; // absolute addressing, no offset
1693 }
1694
1695 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1696 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1697 ShouldNotReachHere();
1698 }
1699
1700 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1701 // Empty encoding
1702 }
1703
1704 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1705 return 0;
1706 }
1707
1708 #ifndef PRODUCT
1709 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1710 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1711 }
1712 #endif
1713
1714 #ifndef PRODUCT
1715 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1716 Compile* C = ra_->C;
1717
1718 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1719
1720 if (C->output()->need_stack_bang(framesize))
1721 st->print("# stack bang size=%d\n\t", framesize);
1722
1723 if (VM_Version::use_rop_protection()) {
1724 st->print("ldr zr, [lr]\n\t");
1725 st->print("pacia lr, rfp\n\t");
1726 }
1727 if (framesize < ((1 << 9) + 2 * wordSize)) {
1728 st->print("sub sp, sp, #%d\n\t", framesize);
1729 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1730 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1731 } else {
1732 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1733 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1734 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1735 st->print("sub sp, sp, rscratch1");
1736 }
1737 if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1738 st->print("\n\t");
1739 st->print("ldr rscratch1, [guard]\n\t");
1740 st->print("dmb ishld\n\t");
1741 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1742 st->print("cmp rscratch1, rscratch2\n\t");
1743 st->print("b.eq skip");
1744 st->print("\n\t");
1745 st->print("blr #nmethod_entry_barrier_stub\n\t");
1746 st->print("b skip\n\t");
1747 st->print("guard: int\n\t");
1748 st->print("\n\t");
1749 st->print("skip:\n\t");
1750 }
1751 }
1752 #endif
1753
1754 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1755 Compile* C = ra_->C;
1756 C2_MacroAssembler _masm(&cbuf);
1757
1758 // n.b. frame size includes space for return pc and rfp
1759 const int framesize = C->output()->frame_size_in_bytes();
1760
1761 // insert a nop at the start of the prolog so we can patch in a
1762 // branch if we need to invalidate the method later
1763 __ nop();
1764
1765 if (C->clinit_barrier_on_entry()) {
1766 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1767
1768 Label L_skip_barrier;
1769
1770 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1771 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1772 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1773 __ bind(L_skip_barrier);
1774 }
1775
1776 if (C->max_vector_size() > 0) {
1777 __ reinitialize_ptrue();
1778 }
1779
1780 int bangsize = C->output()->bang_size_in_bytes();
1781 if (C->output()->need_stack_bang(bangsize))
1782 __ generate_stack_overflow_check(bangsize);
1783
1784 __ build_frame(framesize);
1785
1786 if (C->stub_function() == NULL) {
1787 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1788 if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1789 // Dummy labels for just measuring the code size
1790 Label dummy_slow_path;
1791 Label dummy_continuation;
1792 Label dummy_guard;
1793 Label* slow_path = &dummy_slow_path;
1794 Label* continuation = &dummy_continuation;
1795 Label* guard = &dummy_guard;
1796 if (!Compile::current()->output()->in_scratch_emit_size()) {
1797 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1798 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1799 Compile::current()->output()->add_stub(stub);
1800 slow_path = &stub->entry();
1801 continuation = &stub->continuation();
1802 guard = &stub->guard();
1803 }
1804 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1805 bs->nmethod_entry_barrier(&_masm, slow_path, continuation, guard);
1806 }
1807 }
1808
1809 if (VerifyStackAtCalls) {
1810 Unimplemented();
1811 }
1812
1813 C->output()->set_frame_complete(cbuf.insts_size());
1814
1815 if (C->has_mach_constant_base_node()) {
1816 // NOTE: We set the table base offset here because users might be
1817 // emitted before MachConstantBaseNode.
1818 ConstantTable& constant_table = C->output()->constant_table();
1819 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1820 }
1821 }
1822
1823 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1824 {
1825 return MachNode::size(ra_); // too many variables; just compute it
1826 // the hard way
1827 }
1828
1829 int MachPrologNode::reloc() const
1830 {
1831 return 0;
1832 }
1833
1834 //=============================================================================
1835
1836 #ifndef PRODUCT
1837 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1838 Compile* C = ra_->C;
1839 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1840
1841 st->print("# pop frame %d\n\t",framesize);
1842
1843 if (framesize == 0) {
1844 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1845 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1846 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1847 st->print("add sp, sp, #%d\n\t", framesize);
1848 } else {
1849 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1850 st->print("add sp, sp, rscratch1\n\t");
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 }
1853 if (VM_Version::use_rop_protection()) {
1854 st->print("autia lr, rfp\n\t");
1855 st->print("ldr zr, [lr]\n\t");
1856 }
1857
1858 if (do_polling() && C->is_method_compilation()) {
1859 st->print("# test polling word\n\t");
1860 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1861 st->print("cmp sp, rscratch1\n\t");
1862 st->print("bhi #slow_path");
1863 }
1864 }
1865 #endif
1866
1867 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1868 Compile* C = ra_->C;
1869 C2_MacroAssembler _masm(&cbuf);
1870 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1871
1872 __ remove_frame(framesize);
1873
1874 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1875 __ reserved_stack_check();
1876 }
1877
1878 if (do_polling() && C->is_method_compilation()) {
1879 Label dummy_label;
1880 Label* code_stub = &dummy_label;
1881 if (!C->output()->in_scratch_emit_size()) {
1882 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1883 C->output()->add_stub(stub);
1884 code_stub = &stub->entry();
1885 }
1886 __ relocate(relocInfo::poll_return_type);
1887 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1888 }
1889 }
1890
1891 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1892 // Variable size. Determine dynamically.
1893 return MachNode::size(ra_);
1894 }
1895
1896 int MachEpilogNode::reloc() const {
1897 // Return number of relocatable values contained in this instruction.
1898 return 1; // 1 for polling page.
1899 }
1900
1901 const Pipeline * MachEpilogNode::pipeline() const {
1902 return MachNode::pipeline_class();
1903 }
1904
1905 //=============================================================================
1906
1907 // Figure out which register class each belongs in: rc_int, rc_float or
1908 // rc_stack.
1909 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
1910
1911 static enum RC rc_class(OptoReg::Name reg) {
1912
1913 if (reg == OptoReg::Bad) {
1914 return rc_bad;
1915 }
1916
1917 // we have 32 int registers * 2 halves
1918 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1919
1920 if (reg < slots_of_int_registers) {
1921 return rc_int;
1922 }
1923
1924 // we have 32 float register * 8 halves
1925 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1926 if (reg < slots_of_int_registers + slots_of_float_registers) {
1927 return rc_float;
1928 }
1929
1930 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1931 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1932 return rc_predicate;
1933 }
1934
1935 // Between predicate regs & stack is the flags.
1936 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1937
1938 return rc_stack;
1939 }
1940
1941 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1942 Compile* C = ra_->C;
1943
1944 // Get registers to move.
1945 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1946 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1947 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1948 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1949
1950 enum RC src_hi_rc = rc_class(src_hi);
1951 enum RC src_lo_rc = rc_class(src_lo);
1952 enum RC dst_hi_rc = rc_class(dst_hi);
1953 enum RC dst_lo_rc = rc_class(dst_lo);
1954
1955 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1956
1957 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1958 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1959 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1960 "expected aligned-adjacent pairs");
1961 }
1962
1963 if (src_lo == dst_lo && src_hi == dst_hi) {
1964 return 0; // Self copy, no move.
1965 }
1966
1967 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1968 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1969 int src_offset = ra_->reg2offset(src_lo);
1970 int dst_offset = ra_->reg2offset(dst_lo);
1971
1972 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1973 uint ireg = ideal_reg();
1974 if (ireg == Op_VecA && cbuf) {
1975 C2_MacroAssembler _masm(cbuf);
1976 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1977 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1978 // stack->stack
1979 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1980 sve_vector_reg_size_in_bytes);
1981 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1982 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1983 sve_vector_reg_size_in_bytes);
1984 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1985 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1986 sve_vector_reg_size_in_bytes);
1987 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1988 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1989 as_FloatRegister(Matcher::_regEncode[src_lo]),
1990 as_FloatRegister(Matcher::_regEncode[src_lo]));
1991 } else {
1992 ShouldNotReachHere();
1993 }
1994 } else if (cbuf) {
1995 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1996 C2_MacroAssembler _masm(cbuf);
1997 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1998 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1999 // stack->stack
2000 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2001 if (ireg == Op_VecD) {
2002 __ unspill(rscratch1, true, src_offset);
2003 __ spill(rscratch1, true, dst_offset);
2004 } else {
2005 __ spill_copy128(src_offset, dst_offset);
2006 }
2007 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2008 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2009 ireg == Op_VecD ? __ T8B : __ T16B,
2010 as_FloatRegister(Matcher::_regEncode[src_lo]));
2011 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2012 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2013 ireg == Op_VecD ? __ D : __ Q,
2014 ra_->reg2offset(dst_lo));
2015 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2016 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 ireg == Op_VecD ? __ D : __ Q,
2018 ra_->reg2offset(src_lo));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 }
2023 } else if (cbuf) {
2024 C2_MacroAssembler _masm(cbuf);
2025 switch (src_lo_rc) {
2026 case rc_int:
2027 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2028 if (is64) {
2029 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2030 as_Register(Matcher::_regEncode[src_lo]));
2031 } else {
2032 C2_MacroAssembler _masm(cbuf);
2033 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2034 as_Register(Matcher::_regEncode[src_lo]));
2035 }
2036 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2037 if (is64) {
2038 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2039 as_Register(Matcher::_regEncode[src_lo]));
2040 } else {
2041 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2042 as_Register(Matcher::_regEncode[src_lo]));
2043 }
2044 } else { // gpr --> stack spill
2045 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2046 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2047 }
2048 break;
2049 case rc_float:
2050 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2051 if (is64) {
2052 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2053 as_FloatRegister(Matcher::_regEncode[src_lo]));
2054 } else {
2055 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2056 as_FloatRegister(Matcher::_regEncode[src_lo]));
2057 }
2058 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2059 if (is64) {
2060 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2061 as_FloatRegister(Matcher::_regEncode[src_lo]));
2062 } else {
2063 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2064 as_FloatRegister(Matcher::_regEncode[src_lo]));
2065 }
2066 } else { // fpr --> stack spill
2067 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2068 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2069 is64 ? __ D : __ S, dst_offset);
2070 }
2071 break;
2072 case rc_stack:
2073 if (dst_lo_rc == rc_int) { // stack --> gpr load
2074 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2075 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2076 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2077 is64 ? __ D : __ S, src_offset);
2078 } else if (dst_lo_rc == rc_predicate) {
2079 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2080 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2081 } else { // stack --> stack copy
2082 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2083 if (ideal_reg() == Op_RegVectMask) {
2084 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2085 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2086 } else {
2087 __ unspill(rscratch1, is64, src_offset);
2088 __ spill(rscratch1, is64, dst_offset);
2089 }
2090 }
2091 break;
2092 case rc_predicate:
2093 if (dst_lo_rc == rc_predicate) {
2094 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2095 } else if (dst_lo_rc == rc_stack) {
2096 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2097 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2098 } else {
2099 assert(false, "bad src and dst rc_class combination.");
2100 ShouldNotReachHere();
2101 }
2102 break;
2103 default:
2104 assert(false, "bad rc_class for spill");
2105 ShouldNotReachHere();
2106 }
2107 }
2108
2109 if (st) {
2110 st->print("spill ");
2111 if (src_lo_rc == rc_stack) {
2112 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2113 } else {
2114 st->print("%s -> ", Matcher::regName[src_lo]);
2115 }
2116 if (dst_lo_rc == rc_stack) {
2117 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2118 } else {
2119 st->print("%s", Matcher::regName[dst_lo]);
2120 }
2121 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2122 int vsize = 0;
2123 switch (ideal_reg()) {
2124 case Op_VecD:
2125 vsize = 64;
2126 break;
2127 case Op_VecX:
2128 vsize = 128;
2129 break;
2130 case Op_VecA:
2131 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2132 break;
2133 default:
2134 assert(false, "bad register type for spill");
2135 ShouldNotReachHere();
2136 }
2137 st->print("\t# vector spill size = %d", vsize);
2138 } else if (ideal_reg() == Op_RegVectMask) {
2139 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2140 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2141 st->print("\t# predicate spill size = %d", vsize);
2142 } else {
2143 st->print("\t# spill size = %d", is64 ? 64 : 32);
2144 }
2145 }
2146
2147 return 0;
2148
2149 }
2150
2151 #ifndef PRODUCT
2152 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2153 if (!ra_)
2154 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2155 else
2156 implementation(NULL, ra_, false, st);
2157 }
2158 #endif
2159
2160 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2161 implementation(&cbuf, ra_, false, NULL);
2162 }
2163
2164 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2165 return MachNode::size(ra_);
2166 }
2167
2168 //=============================================================================
2169
2170 #ifndef PRODUCT
2171 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2172 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2173 int reg = ra_->get_reg_first(this);
2174 st->print("add %s, rsp, #%d]\t# box lock",
2175 Matcher::regName[reg], offset);
2176 }
2177 #endif
2178
2179 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2180 C2_MacroAssembler _masm(&cbuf);
2181
2182 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2183 int reg = ra_->get_encode(this);
2184
2185 // This add will handle any 24-bit signed offset. 24 bits allows an
2186 // 8 megabyte stack frame.
2187 __ add(as_Register(reg), sp, offset);
2188 }
2189
2190 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2191 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2192 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2193
2194 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2195 return NativeInstruction::instruction_size;
2196 } else {
2197 return 2 * NativeInstruction::instruction_size;
2198 }
2199 }
2200
2201 //=============================================================================
2202
2203 #ifndef PRODUCT
2204 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2205 {
2206 st->print_cr("# MachUEPNode");
2207 if (UseCompressedClassPointers) {
2208 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2209 if (CompressedKlassPointers::shift() != 0) {
2210 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2211 }
2212 } else {
2213 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2214 }
2215 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2216 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2217 }
2218 #endif
2219
2220 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2221 {
2222 // This is the unverified entry point.
2223 C2_MacroAssembler _masm(&cbuf);
2224
2225 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2226 Label skip;
2227 // TODO
2228 // can we avoid this skip and still use a reloc?
2229 __ br(Assembler::EQ, skip);
2230 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2231 __ bind(skip);
2232 }
2233
2234 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2235 {
2236 return MachNode::size(ra_);
2237 }
2238
2239 // REQUIRED EMIT CODE
2240
2241 //=============================================================================
2242
2243 // Emit exception handler code.
2244 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2245 {
2246 // mov rscratch1 #exception_blob_entry_point
2247 // br rscratch1
2248 // Note that the code buffer's insts_mark is always relative to insts.
2249 // That's why we must use the macroassembler to generate a handler.
2250 C2_MacroAssembler _masm(&cbuf);
2251 address base = __ start_a_stub(size_exception_handler());
2252 if (base == NULL) {
2253 ciEnv::current()->record_failure("CodeCache is full");
2254 return 0; // CodeBuffer::expand failed
2255 }
2256 int offset = __ offset();
2257 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2258 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2259 __ end_a_stub();
2260 return offset;
2261 }
2262
2263 // Emit deopt handler code.
2264 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2265 {
2266 // Note that the code buffer's insts_mark is always relative to insts.
2267 // That's why we must use the macroassembler to generate a handler.
2268 C2_MacroAssembler _masm(&cbuf);
2269 address base = __ start_a_stub(size_deopt_handler());
2270 if (base == NULL) {
2271 ciEnv::current()->record_failure("CodeCache is full");
2272 return 0; // CodeBuffer::expand failed
2273 }
2274 int offset = __ offset();
2275
2276 __ adr(lr, __ pc());
2277 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2278
2279 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2280 __ end_a_stub();
2281 return offset;
2282 }
2283
2284 // REQUIRED MATCHER CODE
2285
2286 //=============================================================================
2287
2288 bool Matcher::match_rule_supported(int opcode) {
2289 if (!has_match_rule(opcode))
2290 return false;
2291
2292 switch (opcode) {
2293 case Op_OnSpinWait:
2294 return VM_Version::supports_on_spin_wait();
2295 case Op_CacheWB:
2296 case Op_CacheWBPreSync:
2297 case Op_CacheWBPostSync:
2298 if (!VM_Version::supports_data_cache_line_flush()) {
2299 return false;
2300 }
2301 break;
2302 case Op_ExpandBits:
2303 case Op_CompressBits:
2304 if (!VM_Version::supports_svebitperm()) {
2305 return false;
2306 }
2307 break;
2308 case Op_FmaF:
2309 case Op_FmaD:
2310 case Op_FmaVF:
2311 case Op_FmaVD:
2312 if (!UseFMA) {
2313 return false;
2314 }
2315 break;
2316 }
2317
2318 return true; // Per default match rules are supported.
2319 }
2320
2321 const RegMask* Matcher::predicate_reg_mask(void) {
2322 return &_PR_REG_mask;
2323 }
2324
2325 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2326 return new TypeVectMask(elemTy, length);
2327 }
2328
2329 // Vector calling convention not yet implemented.
2330 bool Matcher::supports_vector_calling_convention(void) {
2331 return false;
2332 }
2333
2334 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2335 Unimplemented();
2336 return OptoRegPair(0, 0);
2337 }
2338
2339 // Is this branch offset short enough that a short branch can be used?
2340 //
2341 // NOTE: If the platform does not provide any short branch variants, then
2342 // this method should return false for offset 0.
2343 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2344 // The passed offset is relative to address of the branch.
2345
2346 return (-32768 <= offset && offset < 32768);
2347 }
2348
2349 // Vector width in bytes.
2350 int Matcher::vector_width_in_bytes(BasicType bt) {
2351 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2352 int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
2353 // Minimum 2 values in vector
2354 if (size < 2*type2aelembytes(bt)) size = 0;
2355 // But never < 4
2356 if (size < 4) size = 0;
2357 return size;
2358 }
2359
2360 // Limits on vector size (number of elements) loaded into vector.
2361 int Matcher::max_vector_size(const BasicType bt) {
2362 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2363 }
2364
2365 int Matcher::min_vector_size(const BasicType bt) {
2366 int max_size = max_vector_size(bt);
2367 // Limit the min vector size to 8 bytes.
2368 int size = 8 / type2aelembytes(bt);
2369 if (bt == T_BYTE) {
2370 // To support vector api shuffle/rearrange.
2371 size = 4;
2372 } else if (bt == T_BOOLEAN) {
2373 // To support vector api load/store mask.
2374 size = 2;
2375 }
2376 if (size < 2) size = 2;
2377 return MIN2(size, max_size);
2378 }
2379
2380 int Matcher::superword_max_vector_size(const BasicType bt) {
2381 return Matcher::max_vector_size(bt);
2382 }
2383
2384 // Actual max scalable vector register length.
2385 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2386 return Matcher::max_vector_size(bt);
2387 }
2388
2389 // Vector ideal reg.
2390 uint Matcher::vector_ideal_reg(int len) {
2391 if (UseSVE > 0 && 16 < len && len <= 256) {
2392 return Op_VecA;
2393 }
2394 switch(len) {
2395 // For 16-bit/32-bit mask vector, reuse VecD.
2396 case 2:
2397 case 4:
2398 case 8: return Op_VecD;
2399 case 16: return Op_VecX;
2400 }
2401 ShouldNotReachHere();
2402 return 0;
2403 }
2404
2405 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2406 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2407 switch (ideal_reg) {
2408 case Op_VecA: return new vecAOper();
2409 case Op_VecD: return new vecDOper();
2410 case Op_VecX: return new vecXOper();
2411 }
2412 ShouldNotReachHere();
2413 return NULL;
2414 }
2415
2416 bool Matcher::is_reg2reg_move(MachNode* m) {
2417 return false;
2418 }
2419
2420 bool Matcher::is_generic_vector(MachOper* opnd) {
2421 return opnd->opcode() == VREG;
2422 }
2423
2424 // Return whether or not this register is ever used as an argument.
2425 // This function is used on startup to build the trampoline stubs in
2426 // generateOptoStub. Registers not mentioned will be killed by the VM
2427 // call in the trampoline, and arguments in those registers not be
2428 // available to the callee.
2429 bool Matcher::can_be_java_arg(int reg)
2430 {
2431 return
2432 reg == R0_num || reg == R0_H_num ||
2433 reg == R1_num || reg == R1_H_num ||
2434 reg == R2_num || reg == R2_H_num ||
2435 reg == R3_num || reg == R3_H_num ||
2436 reg == R4_num || reg == R4_H_num ||
2437 reg == R5_num || reg == R5_H_num ||
2438 reg == R6_num || reg == R6_H_num ||
2439 reg == R7_num || reg == R7_H_num ||
2440 reg == V0_num || reg == V0_H_num ||
2441 reg == V1_num || reg == V1_H_num ||
2442 reg == V2_num || reg == V2_H_num ||
2443 reg == V3_num || reg == V3_H_num ||
2444 reg == V4_num || reg == V4_H_num ||
2445 reg == V5_num || reg == V5_H_num ||
2446 reg == V6_num || reg == V6_H_num ||
2447 reg == V7_num || reg == V7_H_num;
2448 }
2449
2450 bool Matcher::is_spillable_arg(int reg)
2451 {
2452 return can_be_java_arg(reg);
2453 }
2454
2455 uint Matcher::int_pressure_limit()
2456 {
2457 // JDK-8183543: When taking the number of available registers as int
2458 // register pressure threshold, the jtreg test:
2459 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2460 // failed due to C2 compilation failure with
2461 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2462 //
2463 // A derived pointer is live at CallNode and then is flagged by RA
2464 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2465 // derived pointers and lastly fail to spill after reaching maximum
2466 // number of iterations. Lowering the default pressure threshold to
2467 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2468 // a high register pressure area of the code so that split_DEF can
2469 // generate DefinitionSpillCopy for the derived pointer.
2470 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2471 if (!PreserveFramePointer) {
2472 // When PreserveFramePointer is off, frame pointer is allocatable,
2473 // but different from other SOC registers, it is excluded from
2474 // fatproj's mask because its save type is No-Save. Decrease 1 to
2475 // ensure high pressure at fatproj when PreserveFramePointer is off.
2476 // See check_pressure_at_fatproj().
2477 default_int_pressure_threshold--;
2478 }
2479 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2480 }
2481
2482 uint Matcher::float_pressure_limit()
2483 {
2484 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2485 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2486 }
2487
2488 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2489 return false;
2490 }
2491
2492 RegMask Matcher::divI_proj_mask() {
2493 ShouldNotReachHere();
2494 return RegMask();
2495 }
2496
2497 // Register for MODI projection of divmodI.
2498 RegMask Matcher::modI_proj_mask() {
2499 ShouldNotReachHere();
2500 return RegMask();
2501 }
2502
2503 // Register for DIVL projection of divmodL.
2504 RegMask Matcher::divL_proj_mask() {
2505 ShouldNotReachHere();
2506 return RegMask();
2507 }
2508
2509 // Register for MODL projection of divmodL.
2510 RegMask Matcher::modL_proj_mask() {
2511 ShouldNotReachHere();
2512 return RegMask();
2513 }
2514
2515 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2516 return FP_REG_mask();
2517 }
2518
2519 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2520 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2521 Node* u = addp->fast_out(i);
2522 if (u->is_LoadStore()) {
2523 // On AArch64, LoadStoreNodes (i.e. compare and swap
2524 // instructions) only take register indirect as an operand, so
2525 // any attempt to use an AddPNode as an input to a LoadStoreNode
2526 // must fail.
2527 return false;
2528 }
2529 if (u->is_Mem()) {
2530 int opsize = u->as_Mem()->memory_size();
2531 assert(opsize > 0, "unexpected memory operand size");
2532 if (u->as_Mem()->memory_size() != (1<<shift)) {
2533 return false;
2534 }
2535 }
2536 }
2537 return true;
2538 }
2539
2540 // Convert BootTest condition to Assembler condition.
2541 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2542 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2543 Assembler::Condition result;
2544 switch(cond) {
2545 case BoolTest::eq:
2546 result = Assembler::EQ; break;
2547 case BoolTest::ne:
2548 result = Assembler::NE; break;
2549 case BoolTest::le:
2550 result = Assembler::LE; break;
2551 case BoolTest::ge:
2552 result = Assembler::GE; break;
2553 case BoolTest::lt:
2554 result = Assembler::LT; break;
2555 case BoolTest::gt:
2556 result = Assembler::GT; break;
2557 case BoolTest::ule:
2558 result = Assembler::LS; break;
2559 case BoolTest::uge:
2560 result = Assembler::HS; break;
2561 case BoolTest::ult:
2562 result = Assembler::LO; break;
2563 case BoolTest::ugt:
2564 result = Assembler::HI; break;
2565 case BoolTest::overflow:
2566 result = Assembler::VS; break;
2567 case BoolTest::no_overflow:
2568 result = Assembler::VC; break;
2569 default:
2570 ShouldNotReachHere();
2571 return Assembler::Condition(-1);
2572 }
2573
2574 // Check conversion
2575 if (cond & BoolTest::unsigned_compare) {
2576 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2577 } else {
2578 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2579 }
2580
2581 return result;
2582 }
2583
2584 // Binary src (Replicate con)
2585 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2586 if (n == NULL || m == NULL) {
2587 return false;
2588 }
2589
2590 if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
2591 return false;
2592 }
2593
2594 Node* imm_node = m->in(1);
2595 if (!imm_node->is_Con()) {
2596 return false;
2597 }
2598
2599 const Type* t = imm_node->bottom_type();
2600 if (!(t->isa_int() || t->isa_long())) {
2601 return false;
2602 }
2603
2604 switch (n->Opcode()) {
2605 case Op_AndV:
2606 case Op_OrV:
2607 case Op_XorV: {
2608 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2609 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2610 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2611 }
2612 case Op_AddVB:
2613 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2614 case Op_AddVS:
2615 case Op_AddVI:
2616 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2617 case Op_AddVL:
2618 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2619 default:
2620 return false;
2621 }
2622 }
2623
2624 // (XorV src (Replicate m1))
2625 // (XorVMask src (MaskAll m1))
2626 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2627 if (n != NULL && m != NULL) {
2628 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2629 VectorNode::is_all_ones_vector(m);
2630 }
2631 return false;
2632 }
2633
2634 // Should the matcher clone input 'm' of node 'n'?
2635 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2636 if (is_vshift_con_pattern(n, m) ||
2637 is_vector_bitwise_not_pattern(n, m) ||
2638 is_valid_sve_arith_imm_pattern(n, m)) {
2639 mstack.push(m, Visit);
2640 return true;
2641 }
2642 return false;
2643 }
2644
2645 // Should the Matcher clone shifts on addressing modes, expecting them
2646 // to be subsumed into complex addressing expressions or compute them
2647 // into registers?
2648 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2649 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2650 return true;
2651 }
2652
2653 Node *off = m->in(AddPNode::Offset);
2654 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2655 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2656 // Are there other uses besides address expressions?
2657 !is_visited(off)) {
2658 address_visited.set(off->_idx); // Flag as address_visited
2659 mstack.push(off->in(2), Visit);
2660 Node *conv = off->in(1);
2661 if (conv->Opcode() == Op_ConvI2L &&
2662 // Are there other uses besides address expressions?
2663 !is_visited(conv)) {
2664 address_visited.set(conv->_idx); // Flag as address_visited
2665 mstack.push(conv->in(1), Pre_Visit);
2666 } else {
2667 mstack.push(conv, Pre_Visit);
2668 }
2669 address_visited.test_set(m->_idx); // Flag as address_visited
2670 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2671 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2672 return true;
2673 } else if (off->Opcode() == Op_ConvI2L &&
2674 // Are there other uses besides address expressions?
2675 !is_visited(off)) {
2676 address_visited.test_set(m->_idx); // Flag as address_visited
2677 address_visited.set(off->_idx); // Flag as address_visited
2678 mstack.push(off->in(1), Pre_Visit);
2679 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2680 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2681 return true;
2682 }
2683 return false;
2684 }
2685
2686 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2687 C2_MacroAssembler _masm(&cbuf); \
2688 { \
2689 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2690 guarantee(DISP == 0, "mode not permitted for volatile"); \
2691 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2692 __ INSN(REG, as_Register(BASE)); \
2693 }
2694
2695
2696 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2697 {
2698 Address::extend scale;
2699
2700 // Hooboy, this is fugly. We need a way to communicate to the
2701 // encoder that the index needs to be sign extended, so we have to
2702 // enumerate all the cases.
2703 switch (opcode) {
2704 case INDINDEXSCALEDI2L:
2705 case INDINDEXSCALEDI2LN:
2706 case INDINDEXI2L:
2707 case INDINDEXI2LN:
2708 scale = Address::sxtw(size);
2709 break;
2710 default:
2711 scale = Address::lsl(size);
2712 }
2713
2714 if (index == -1) {
2715 return Address(base, disp);
2716 } else {
2717 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2718 return Address(base, as_Register(index), scale);
2719 }
2720 }
2721
2722
2723 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2724 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2725 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2726 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2727 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2728
2729 // Used for all non-volatile memory accesses. The use of
2730 // $mem->opcode() to discover whether this pattern uses sign-extended
2731 // offsets is something of a kludge.
2732 static void loadStore(C2_MacroAssembler masm, mem_insn insn,
2733 Register reg, int opcode,
2734 Register base, int index, int scale, int disp,
2735 int size_in_memory)
2736 {
2737 Address addr = mem2address(opcode, base, index, scale, disp);
2738 if (addr.getMode() == Address::base_plus_offset) {
2739 /* If we get an out-of-range offset it is a bug in the compiler,
2740 so we assert here. */
2741 assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
2742 "c2 compiler bug");
2743 /* Fix up any out-of-range offsets. */
2744 assert_different_registers(rscratch1, base);
2745 assert_different_registers(rscratch1, reg);
2746 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2747 }
2748 (masm.*insn)(reg, addr);
2749 }
2750
2751 static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
2752 FloatRegister reg, int opcode,
2753 Register base, int index, int size, int disp,
2754 int size_in_memory)
2755 {
2756 Address::extend scale;
2757
2758 switch (opcode) {
2759 case INDINDEXSCALEDI2L:
2760 case INDINDEXSCALEDI2LN:
2761 scale = Address::sxtw(size);
2762 break;
2763 default:
2764 scale = Address::lsl(size);
2765 }
2766
2767 if (index == -1) {
2768 /* If we get an out-of-range offset it is a bug in the compiler,
2769 so we assert here. */
2770 assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2771 /* Fix up any out-of-range offsets. */
2772 assert_different_registers(rscratch1, base);
2773 Address addr = Address(base, disp);
2774 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2775 (masm.*insn)(reg, addr);
2776 } else {
2777 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2778 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2779 }
2780 }
2781
2782 static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
2783 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2784 int opcode, Register base, int index, int size, int disp)
2785 {
2786 if (index == -1) {
2787 (masm.*insn)(reg, T, Address(base, disp));
2788 } else {
2789 assert(disp == 0, "unsupported address mode");
2790 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2791 }
2792 }
2793
2794 %}
2795
2796
2797
2798 //----------ENCODING BLOCK-----------------------------------------------------
2799 // This block specifies the encoding classes used by the compiler to
2800 // output byte streams. Encoding classes are parameterized macros
2801 // used by Machine Instruction Nodes in order to generate the bit
2802 // encoding of the instruction. Operands specify their base encoding
2803 // interface with the interface keyword. There are currently
2804 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2805 // COND_INTER. REG_INTER causes an operand to generate a function
2806 // which returns its register number when queried. CONST_INTER causes
2807 // an operand to generate a function which returns the value of the
2808 // constant when queried. MEMORY_INTER causes an operand to generate
2809 // four functions which return the Base Register, the Index Register,
2810 // the Scale Value, and the Offset Value of the operand when queried.
2811 // COND_INTER causes an operand to generate six functions which return
2812 // the encoding code (ie - encoding bits for the instruction)
2813 // associated with each basic boolean condition for a conditional
2814 // instruction.
2815 //
2816 // Instructions specify two basic values for encoding. Again, a
2817 // function is available to check if the constant displacement is an
2818 // oop. They use the ins_encode keyword to specify their encoding
2819 // classes (which must be a sequence of enc_class names, and their
2820 // parameters, specified in the encoding block), and they use the
2821 // opcode keyword to specify, in order, their primary, secondary, and
2822 // tertiary opcode. Only the opcode sections which a particular
2823 // instruction needs for encoding need to be specified.
2824 encode %{
2825 // Build emit functions for each basic byte or larger field in the
2826 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2827 // from C++ code in the enc_class source block. Emit functions will
2828 // live in the main source block for now. In future, we can
2829 // generalize this by adding a syntax that specifies the sizes of
2830 // fields in an order, so that the adlc can build the emit functions
2831 // automagically
2832
2833 // catch all for unimplemented encodings
2834 enc_class enc_unimplemented %{
2835 C2_MacroAssembler _masm(&cbuf);
2836 __ unimplemented("C2 catch all");
2837 %}
2838
2839 // BEGIN Non-volatile memory access
2840
2841 // This encoding class is generated automatically from ad_encode.m4.
2842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2843 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2844 Register dst_reg = as_Register($dst$$reg);
2845 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2846 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2847 %}
2848
2849 // This encoding class is generated automatically from ad_encode.m4.
2850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2851 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2852 Register dst_reg = as_Register($dst$$reg);
2853 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2854 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2855 %}
2856
2857 // This encoding class is generated automatically from ad_encode.m4.
2858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2859 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2860 Register dst_reg = as_Register($dst$$reg);
2861 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2862 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2863 %}
2864
2865 // This encoding class is generated automatically from ad_encode.m4.
2866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2867 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2868 Register dst_reg = as_Register($dst$$reg);
2869 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2870 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2871 %}
2872
2873 // This encoding class is generated automatically from ad_encode.m4.
2874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2875 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2876 Register dst_reg = as_Register($dst$$reg);
2877 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2878 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2879 %}
2880
2881 // This encoding class is generated automatically from ad_encode.m4.
2882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2883 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2884 Register dst_reg = as_Register($dst$$reg);
2885 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2886 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2887 %}
2888
2889 // This encoding class is generated automatically from ad_encode.m4.
2890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2891 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2892 Register dst_reg = as_Register($dst$$reg);
2893 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2894 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2895 %}
2896
2897 // This encoding class is generated automatically from ad_encode.m4.
2898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2899 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2900 Register dst_reg = as_Register($dst$$reg);
2901 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2902 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2903 %}
2904
2905 // This encoding class is generated automatically from ad_encode.m4.
2906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2907 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2908 Register dst_reg = as_Register($dst$$reg);
2909 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2910 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2911 %}
2912
2913 // This encoding class is generated automatically from ad_encode.m4.
2914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2915 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2916 Register dst_reg = as_Register($dst$$reg);
2917 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2918 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2919 %}
2920
2921 // This encoding class is generated automatically from ad_encode.m4.
2922 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2923 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2924 Register dst_reg = as_Register($dst$$reg);
2925 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2926 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2927 %}
2928
2929 // This encoding class is generated automatically from ad_encode.m4.
2930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2931 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2932 Register dst_reg = as_Register($dst$$reg);
2933 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2934 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2935 %}
2936
2937 // This encoding class is generated automatically from ad_encode.m4.
2938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2939 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2940 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2941 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2942 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2943 %}
2944
2945 // This encoding class is generated automatically from ad_encode.m4.
2946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2947 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2948 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2949 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2950 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2951 %}
2952
2953 // This encoding class is generated automatically from ad_encode.m4.
2954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2955 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2956 Register src_reg = as_Register($src$$reg);
2957 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2958 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2959 %}
2960
2961 // This encoding class is generated automatically from ad_encode.m4.
2962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2963 enc_class aarch64_enc_strb0(memory1 mem) %{
2964 C2_MacroAssembler _masm(&cbuf);
2965 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2966 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2967 %}
2968
2969 // This encoding class is generated automatically from ad_encode.m4.
2970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2971 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2972 Register src_reg = as_Register($src$$reg);
2973 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2974 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2975 %}
2976
2977 // This encoding class is generated automatically from ad_encode.m4.
2978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2979 enc_class aarch64_enc_strh0(memory2 mem) %{
2980 C2_MacroAssembler _masm(&cbuf);
2981 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2982 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2983 %}
2984
2985 // This encoding class is generated automatically from ad_encode.m4.
2986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2987 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2988 Register src_reg = as_Register($src$$reg);
2989 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2990 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2991 %}
2992
2993 // This encoding class is generated automatically from ad_encode.m4.
2994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2995 enc_class aarch64_enc_strw0(memory4 mem) %{
2996 C2_MacroAssembler _masm(&cbuf);
2997 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2998 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2999 %}
3000
3001 // This encoding class is generated automatically from ad_encode.m4.
3002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3003 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3004 Register src_reg = as_Register($src$$reg);
3005 // we sometimes get asked to store the stack pointer into the
3006 // current thread -- we cannot do that directly on AArch64
3007 if (src_reg == r31_sp) {
3008 C2_MacroAssembler _masm(&cbuf);
3009 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3010 __ mov(rscratch2, sp);
3011 src_reg = rscratch2;
3012 }
3013 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
3014 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3015 %}
3016
3017 // This encoding class is generated automatically from ad_encode.m4.
3018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3019 enc_class aarch64_enc_str0(memory8 mem) %{
3020 C2_MacroAssembler _masm(&cbuf);
3021 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
3022 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3023 %}
3024
3025 // This encoding class is generated automatically from ad_encode.m4.
3026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3027 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3028 FloatRegister src_reg = as_FloatRegister($src$$reg);
3029 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
3030 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3031 %}
3032
3033 // This encoding class is generated automatically from ad_encode.m4.
3034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3035 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3036 FloatRegister src_reg = as_FloatRegister($src$$reg);
3037 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
3038 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3039 %}
3040
3041 // This encoding class is generated automatically from ad_encode.m4.
3042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3043 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3044 C2_MacroAssembler _masm(&cbuf);
3045 __ membar(Assembler::StoreStore);
3046 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
3047 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3048 %}
3049
3050 // END Non-volatile memory access
3051
3052 // Vector loads and stores
3053 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3054 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3055 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3056 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3057 %}
3058
3059 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3060 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3061 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3062 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3063 %}
3064
3065 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3066 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3067 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3068 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3069 %}
3070
3071 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3072 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3073 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3074 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3075 %}
3076
3077 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3078 FloatRegister src_reg = as_FloatRegister($src$$reg);
3079 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
3080 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3081 %}
3082
3083 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3084 FloatRegister src_reg = as_FloatRegister($src$$reg);
3085 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
3086 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3087 %}
3088
3089 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3090 FloatRegister src_reg = as_FloatRegister($src$$reg);
3091 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
3092 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3093 %}
3094
3095 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3096 FloatRegister src_reg = as_FloatRegister($src$$reg);
3097 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
3098 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3099 %}
3100
3101 // volatile loads and stores
3102
3103 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3104 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3105 rscratch1, stlrb);
3106 %}
3107
3108 enc_class aarch64_enc_stlrb0(memory mem) %{
3109 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3110 rscratch1, stlrb);
3111 %}
3112
3113 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3114 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3115 rscratch1, stlrh);
3116 %}
3117
3118 enc_class aarch64_enc_stlrh0(memory mem) %{
3119 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3120 rscratch1, stlrh);
3121 %}
3122
3123 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3124 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3125 rscratch1, stlrw);
3126 %}
3127
3128 enc_class aarch64_enc_stlrw0(memory mem) %{
3129 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3130 rscratch1, stlrw);
3131 %}
3132
3133 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3134 Register dst_reg = as_Register($dst$$reg);
3135 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3136 rscratch1, ldarb);
3137 __ sxtbw(dst_reg, dst_reg);
3138 %}
3139
3140 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3141 Register dst_reg = as_Register($dst$$reg);
3142 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3143 rscratch1, ldarb);
3144 __ sxtb(dst_reg, dst_reg);
3145 %}
3146
3147 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3148 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3149 rscratch1, ldarb);
3150 %}
3151
3152 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3153 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3154 rscratch1, ldarb);
3155 %}
3156
3157 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3158 Register dst_reg = as_Register($dst$$reg);
3159 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, ldarh);
3161 __ sxthw(dst_reg, dst_reg);
3162 %}
3163
3164 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3165 Register dst_reg = as_Register($dst$$reg);
3166 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3167 rscratch1, ldarh);
3168 __ sxth(dst_reg, dst_reg);
3169 %}
3170
3171 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3172 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3173 rscratch1, ldarh);
3174 %}
3175
3176 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3177 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3178 rscratch1, ldarh);
3179 %}
3180
3181 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3182 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3183 rscratch1, ldarw);
3184 %}
3185
3186 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3187 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3188 rscratch1, ldarw);
3189 %}
3190
3191 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3192 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3193 rscratch1, ldar);
3194 %}
3195
3196 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3197 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3198 rscratch1, ldarw);
3199 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3200 %}
3201
3202 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3203 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3204 rscratch1, ldar);
3205 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3206 %}
3207
3208 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3209 Register src_reg = as_Register($src$$reg);
3210 // we sometimes get asked to store the stack pointer into the
3211 // current thread -- we cannot do that directly on AArch64
3212 if (src_reg == r31_sp) {
3213 C2_MacroAssembler _masm(&cbuf);
3214 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3215 __ mov(rscratch2, sp);
3216 src_reg = rscratch2;
3217 }
3218 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3219 rscratch1, stlr);
3220 %}
3221
3222 enc_class aarch64_enc_stlr0(memory mem) %{
3223 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3224 rscratch1, stlr);
3225 %}
3226
3227 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3228 {
3229 C2_MacroAssembler _masm(&cbuf);
3230 FloatRegister src_reg = as_FloatRegister($src$$reg);
3231 __ fmovs(rscratch2, src_reg);
3232 }
3233 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3234 rscratch1, stlrw);
3235 %}
3236
3237 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3238 {
3239 C2_MacroAssembler _masm(&cbuf);
3240 FloatRegister src_reg = as_FloatRegister($src$$reg);
3241 __ fmovd(rscratch2, src_reg);
3242 }
3243 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3244 rscratch1, stlr);
3245 %}
3246
3247 // synchronized read/update encodings
3248
3249 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3250 C2_MacroAssembler _masm(&cbuf);
3251 Register dst_reg = as_Register($dst$$reg);
3252 Register base = as_Register($mem$$base);
3253 int index = $mem$$index;
3254 int scale = $mem$$scale;
3255 int disp = $mem$$disp;
3256 if (index == -1) {
3257 if (disp != 0) {
3258 __ lea(rscratch1, Address(base, disp));
3259 __ ldaxr(dst_reg, rscratch1);
3260 } else {
3261 // TODO
3262 // should we ever get anything other than this case?
3263 __ ldaxr(dst_reg, base);
3264 }
3265 } else {
3266 Register index_reg = as_Register(index);
3267 if (disp == 0) {
3268 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3269 __ ldaxr(dst_reg, rscratch1);
3270 } else {
3271 __ lea(rscratch1, Address(base, disp));
3272 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3273 __ ldaxr(dst_reg, rscratch1);
3274 }
3275 }
3276 %}
3277
3278 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3279 C2_MacroAssembler _masm(&cbuf);
3280 Register src_reg = as_Register($src$$reg);
3281 Register base = as_Register($mem$$base);
3282 int index = $mem$$index;
3283 int scale = $mem$$scale;
3284 int disp = $mem$$disp;
3285 if (index == -1) {
3286 if (disp != 0) {
3287 __ lea(rscratch2, Address(base, disp));
3288 __ stlxr(rscratch1, src_reg, rscratch2);
3289 } else {
3290 // TODO
3291 // should we ever get anything other than this case?
3292 __ stlxr(rscratch1, src_reg, base);
3293 }
3294 } else {
3295 Register index_reg = as_Register(index);
3296 if (disp == 0) {
3297 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3298 __ stlxr(rscratch1, src_reg, rscratch2);
3299 } else {
3300 __ lea(rscratch2, Address(base, disp));
3301 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3302 __ stlxr(rscratch1, src_reg, rscratch2);
3303 }
3304 }
3305 __ cmpw(rscratch1, zr);
3306 %}
3307
3308 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3309 C2_MacroAssembler _masm(&cbuf);
3310 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3311 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3312 Assembler::xword, /*acquire*/ false, /*release*/ true,
3313 /*weak*/ false, noreg);
3314 %}
3315
3316 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3317 C2_MacroAssembler _masm(&cbuf);
3318 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3319 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3320 Assembler::word, /*acquire*/ false, /*release*/ true,
3321 /*weak*/ false, noreg);
3322 %}
3323
3324 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3325 C2_MacroAssembler _masm(&cbuf);
3326 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3327 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3328 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3329 /*weak*/ false, noreg);
3330 %}
3331
3332 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3333 C2_MacroAssembler _masm(&cbuf);
3334 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3335 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3336 Assembler::byte, /*acquire*/ false, /*release*/ true,
3337 /*weak*/ false, noreg);
3338 %}
3339
3340
3341 // The only difference between aarch64_enc_cmpxchg and
3342 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3343 // CompareAndSwap sequence to serve as a barrier on acquiring a
3344 // lock.
3345 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3346 C2_MacroAssembler _masm(&cbuf);
3347 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3348 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3349 Assembler::xword, /*acquire*/ true, /*release*/ true,
3350 /*weak*/ false, noreg);
3351 %}
3352
3353 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3354 C2_MacroAssembler _masm(&cbuf);
3355 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3356 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3357 Assembler::word, /*acquire*/ true, /*release*/ true,
3358 /*weak*/ false, noreg);
3359 %}
3360
3361 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3362 C2_MacroAssembler _masm(&cbuf);
3363 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3364 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3365 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3366 /*weak*/ false, noreg);
3367 %}
3368
3369 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3370 C2_MacroAssembler _masm(&cbuf);
3371 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3372 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3373 Assembler::byte, /*acquire*/ true, /*release*/ true,
3374 /*weak*/ false, noreg);
3375 %}
3376
3377 // auxiliary used for CompareAndSwapX to set result register
3378 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3379 C2_MacroAssembler _masm(&cbuf);
3380 Register res_reg = as_Register($res$$reg);
3381 __ cset(res_reg, Assembler::EQ);
3382 %}
3383
3384 // prefetch encodings
3385
3386 enc_class aarch64_enc_prefetchw(memory mem) %{
3387 C2_MacroAssembler _masm(&cbuf);
3388 Register base = as_Register($mem$$base);
3389 int index = $mem$$index;
3390 int scale = $mem$$scale;
3391 int disp = $mem$$disp;
3392 if (index == -1) {
3393 __ prfm(Address(base, disp), PSTL1KEEP);
3394 } else {
3395 Register index_reg = as_Register(index);
3396 if (disp == 0) {
3397 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3398 } else {
3399 __ lea(rscratch1, Address(base, disp));
3400 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3401 }
3402 }
3403 %}
3404
3405 /// mov envcodings
3406
3407 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3408 C2_MacroAssembler _masm(&cbuf);
3409 uint32_t con = (uint32_t)$src$$constant;
3410 Register dst_reg = as_Register($dst$$reg);
3411 if (con == 0) {
3412 __ movw(dst_reg, zr);
3413 } else {
3414 __ movw(dst_reg, con);
3415 }
3416 %}
3417
3418 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3419 C2_MacroAssembler _masm(&cbuf);
3420 Register dst_reg = as_Register($dst$$reg);
3421 uint64_t con = (uint64_t)$src$$constant;
3422 if (con == 0) {
3423 __ mov(dst_reg, zr);
3424 } else {
3425 __ mov(dst_reg, con);
3426 }
3427 %}
3428
3429 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3430 C2_MacroAssembler _masm(&cbuf);
3431 Register dst_reg = as_Register($dst$$reg);
3432 address con = (address)$src$$constant;
3433 if (con == NULL || con == (address)1) {
3434 ShouldNotReachHere();
3435 } else {
3436 relocInfo::relocType rtype = $src->constant_reloc();
3437 if (rtype == relocInfo::oop_type) {
3438 __ movoop(dst_reg, (jobject)con);
3439 } else if (rtype == relocInfo::metadata_type) {
3440 __ mov_metadata(dst_reg, (Metadata*)con);
3441 } else {
3442 assert(rtype == relocInfo::none, "unexpected reloc type");
3443 if (! __ is_valid_AArch64_address(con) ||
3444 con < (address)(uintptr_t)os::vm_page_size()) {
3445 __ mov(dst_reg, con);
3446 } else {
3447 uint64_t offset;
3448 __ adrp(dst_reg, con, offset);
3449 __ add(dst_reg, dst_reg, offset);
3450 }
3451 }
3452 }
3453 %}
3454
3455 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3456 C2_MacroAssembler _masm(&cbuf);
3457 Register dst_reg = as_Register($dst$$reg);
3458 __ mov(dst_reg, zr);
3459 %}
3460
3461 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3462 C2_MacroAssembler _masm(&cbuf);
3463 Register dst_reg = as_Register($dst$$reg);
3464 __ mov(dst_reg, (uint64_t)1);
3465 %}
3466
3467 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3468 C2_MacroAssembler _masm(&cbuf);
3469 __ load_byte_map_base($dst$$Register);
3470 %}
3471
3472 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3473 C2_MacroAssembler _masm(&cbuf);
3474 Register dst_reg = as_Register($dst$$reg);
3475 address con = (address)$src$$constant;
3476 if (con == NULL) {
3477 ShouldNotReachHere();
3478 } else {
3479 relocInfo::relocType rtype = $src->constant_reloc();
3480 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3481 __ set_narrow_oop(dst_reg, (jobject)con);
3482 }
3483 %}
3484
3485 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3486 C2_MacroAssembler _masm(&cbuf);
3487 Register dst_reg = as_Register($dst$$reg);
3488 __ mov(dst_reg, zr);
3489 %}
3490
3491 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3492 C2_MacroAssembler _masm(&cbuf);
3493 Register dst_reg = as_Register($dst$$reg);
3494 address con = (address)$src$$constant;
3495 if (con == NULL) {
3496 ShouldNotReachHere();
3497 } else {
3498 relocInfo::relocType rtype = $src->constant_reloc();
3499 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3500 __ set_narrow_klass(dst_reg, (Klass *)con);
3501 }
3502 %}
3503
3504 // arithmetic encodings
3505
3506 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3507 C2_MacroAssembler _masm(&cbuf);
3508 Register dst_reg = as_Register($dst$$reg);
3509 Register src_reg = as_Register($src1$$reg);
3510 int32_t con = (int32_t)$src2$$constant;
3511 // add has primary == 0, subtract has primary == 1
3512 if ($primary) { con = -con; }
3513 if (con < 0) {
3514 __ subw(dst_reg, src_reg, -con);
3515 } else {
3516 __ addw(dst_reg, src_reg, con);
3517 }
3518 %}
3519
3520 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3521 C2_MacroAssembler _masm(&cbuf);
3522 Register dst_reg = as_Register($dst$$reg);
3523 Register src_reg = as_Register($src1$$reg);
3524 int32_t con = (int32_t)$src2$$constant;
3525 // add has primary == 0, subtract has primary == 1
3526 if ($primary) { con = -con; }
3527 if (con < 0) {
3528 __ sub(dst_reg, src_reg, -con);
3529 } else {
3530 __ add(dst_reg, src_reg, con);
3531 }
3532 %}
3533
3534 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3535 C2_MacroAssembler _masm(&cbuf);
3536 Register dst_reg = as_Register($dst$$reg);
3537 Register src1_reg = as_Register($src1$$reg);
3538 Register src2_reg = as_Register($src2$$reg);
3539 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3540 %}
3541
3542 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3543 C2_MacroAssembler _masm(&cbuf);
3544 Register dst_reg = as_Register($dst$$reg);
3545 Register src1_reg = as_Register($src1$$reg);
3546 Register src2_reg = as_Register($src2$$reg);
3547 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3548 %}
3549
3550 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3551 C2_MacroAssembler _masm(&cbuf);
3552 Register dst_reg = as_Register($dst$$reg);
3553 Register src1_reg = as_Register($src1$$reg);
3554 Register src2_reg = as_Register($src2$$reg);
3555 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3556 %}
3557
3558 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3559 C2_MacroAssembler _masm(&cbuf);
3560 Register dst_reg = as_Register($dst$$reg);
3561 Register src1_reg = as_Register($src1$$reg);
3562 Register src2_reg = as_Register($src2$$reg);
3563 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3564 %}
3565
3566 // compare instruction encodings
3567
3568 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3569 C2_MacroAssembler _masm(&cbuf);
3570 Register reg1 = as_Register($src1$$reg);
3571 Register reg2 = as_Register($src2$$reg);
3572 __ cmpw(reg1, reg2);
3573 %}
3574
3575 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3576 C2_MacroAssembler _masm(&cbuf);
3577 Register reg = as_Register($src1$$reg);
3578 int32_t val = $src2$$constant;
3579 if (val >= 0) {
3580 __ subsw(zr, reg, val);
3581 } else {
3582 __ addsw(zr, reg, -val);
3583 }
3584 %}
3585
3586 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3587 C2_MacroAssembler _masm(&cbuf);
3588 Register reg1 = as_Register($src1$$reg);
3589 uint32_t val = (uint32_t)$src2$$constant;
3590 __ movw(rscratch1, val);
3591 __ cmpw(reg1, rscratch1);
3592 %}
3593
3594 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3595 C2_MacroAssembler _masm(&cbuf);
3596 Register reg1 = as_Register($src1$$reg);
3597 Register reg2 = as_Register($src2$$reg);
3598 __ cmp(reg1, reg2);
3599 %}
3600
3601 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3602 C2_MacroAssembler _masm(&cbuf);
3603 Register reg = as_Register($src1$$reg);
3604 int64_t val = $src2$$constant;
3605 if (val >= 0) {
3606 __ subs(zr, reg, val);
3607 } else if (val != -val) {
3608 __ adds(zr, reg, -val);
3609 } else {
3610 // aargh, Long.MIN_VALUE is a special case
3611 __ orr(rscratch1, zr, (uint64_t)val);
3612 __ subs(zr, reg, rscratch1);
3613 }
3614 %}
3615
3616 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3617 C2_MacroAssembler _masm(&cbuf);
3618 Register reg1 = as_Register($src1$$reg);
3619 uint64_t val = (uint64_t)$src2$$constant;
3620 __ mov(rscratch1, val);
3621 __ cmp(reg1, rscratch1);
3622 %}
3623
3624 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3625 C2_MacroAssembler _masm(&cbuf);
3626 Register reg1 = as_Register($src1$$reg);
3627 Register reg2 = as_Register($src2$$reg);
3628 __ cmp(reg1, reg2);
3629 %}
3630
3631 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3632 C2_MacroAssembler _masm(&cbuf);
3633 Register reg1 = as_Register($src1$$reg);
3634 Register reg2 = as_Register($src2$$reg);
3635 __ cmpw(reg1, reg2);
3636 %}
3637
3638 enc_class aarch64_enc_testp(iRegP src) %{
3639 C2_MacroAssembler _masm(&cbuf);
3640 Register reg = as_Register($src$$reg);
3641 __ cmp(reg, zr);
3642 %}
3643
3644 enc_class aarch64_enc_testn(iRegN src) %{
3645 C2_MacroAssembler _masm(&cbuf);
3646 Register reg = as_Register($src$$reg);
3647 __ cmpw(reg, zr);
3648 %}
3649
3650 enc_class aarch64_enc_b(label lbl) %{
3651 C2_MacroAssembler _masm(&cbuf);
3652 Label *L = $lbl$$label;
3653 __ b(*L);
3654 %}
3655
3656 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3657 C2_MacroAssembler _masm(&cbuf);
3658 Label *L = $lbl$$label;
3659 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3660 %}
3661
3662 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3663 C2_MacroAssembler _masm(&cbuf);
3664 Label *L = $lbl$$label;
3665 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3666 %}
3667
3668 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3669 %{
3670 Register sub_reg = as_Register($sub$$reg);
3671 Register super_reg = as_Register($super$$reg);
3672 Register temp_reg = as_Register($temp$$reg);
3673 Register result_reg = as_Register($result$$reg);
3674
3675 Label miss;
3676 C2_MacroAssembler _masm(&cbuf);
3677 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3678 NULL, &miss,
3679 /*set_cond_codes:*/ true);
3680 if ($primary) {
3681 __ mov(result_reg, zr);
3682 }
3683 __ bind(miss);
3684 %}
3685
3686 enc_class aarch64_enc_java_static_call(method meth) %{
3687 C2_MacroAssembler _masm(&cbuf);
3688
3689 address addr = (address)$meth$$method;
3690 address call;
3691 if (!_method) {
3692 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3693 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3694 if (call == NULL) {
3695 ciEnv::current()->record_failure("CodeCache is full");
3696 return;
3697 }
3698 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3699 // The NOP here is purely to ensure that eliding a call to
3700 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3701 __ nop();
3702 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3703 } else {
3704 int method_index = resolved_method_index(cbuf);
3705 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3706 : static_call_Relocation::spec(method_index);
3707 call = __ trampoline_call(Address(addr, rspec));
3708 if (call == NULL) {
3709 ciEnv::current()->record_failure("CodeCache is full");
3710 return;
3711 }
3712 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3713 // Calls of the same statically bound method can share
3714 // a stub to the interpreter.
3715 cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
3716 } else {
3717 // Emit stub for static call
3718 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
3719 if (stub == NULL) {
3720 ciEnv::current()->record_failure("CodeCache is full");
3721 return;
3722 }
3723 }
3724 }
3725
3726 __ post_call_nop();
3727
3728 // Only non uncommon_trap calls need to reinitialize ptrue.
3729 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3730 __ reinitialize_ptrue();
3731 }
3732 %}
3733
3734 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3735 C2_MacroAssembler _masm(&cbuf);
3736 int method_index = resolved_method_index(cbuf);
3737 address call = __ ic_call((address)$meth$$method, method_index);
3738 if (call == NULL) {
3739 ciEnv::current()->record_failure("CodeCache is full");
3740 return;
3741 }
3742 __ post_call_nop();
3743 if (Compile::current()->max_vector_size() > 0) {
3744 __ reinitialize_ptrue();
3745 }
3746 %}
3747
3748 enc_class aarch64_enc_call_epilog() %{
3749 C2_MacroAssembler _masm(&cbuf);
3750 if (VerifyStackAtCalls) {
3751 // Check that stack depth is unchanged: find majik cookie on stack
3752 __ call_Unimplemented();
3753 }
3754 %}
3755
3756 enc_class aarch64_enc_java_to_runtime(method meth) %{
3757 C2_MacroAssembler _masm(&cbuf);
3758
3759 // some calls to generated routines (arraycopy code) are scheduled
3760 // by C2 as runtime calls. if so we can call them using a br (they
3761 // will be in a reachable segment) otherwise we have to use a blr
3762 // which loads the absolute address into a register.
3763 address entry = (address)$meth$$method;
3764 CodeBlob *cb = CodeCache::find_blob(entry);
3765 if (cb) {
3766 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3767 if (call == NULL) {
3768 ciEnv::current()->record_failure("CodeCache is full");
3769 return;
3770 }
3771 __ post_call_nop();
3772 } else {
3773 Label retaddr;
3774 __ adr(rscratch2, retaddr);
3775 __ lea(rscratch1, RuntimeAddress(entry));
3776 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3777 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3778 __ blr(rscratch1);
3779 __ bind(retaddr);
3780 __ post_call_nop();
3781 __ add(sp, sp, 2 * wordSize);
3782 }
3783 if (Compile::current()->max_vector_size() > 0) {
3784 __ reinitialize_ptrue();
3785 }
3786 %}
3787
3788 enc_class aarch64_enc_rethrow() %{
3789 C2_MacroAssembler _masm(&cbuf);
3790 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3791 %}
3792
3793 enc_class aarch64_enc_ret() %{
3794 C2_MacroAssembler _masm(&cbuf);
3795 #ifdef ASSERT
3796 if (Compile::current()->max_vector_size() > 0) {
3797 __ verify_ptrue();
3798 }
3799 #endif
3800 __ ret(lr);
3801 %}
3802
3803 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3804 C2_MacroAssembler _masm(&cbuf);
3805 Register target_reg = as_Register($jump_target$$reg);
3806 __ br(target_reg);
3807 %}
3808
3809 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3810 C2_MacroAssembler _masm(&cbuf);
3811 Register target_reg = as_Register($jump_target$$reg);
3812 // exception oop should be in r0
3813 // ret addr has been popped into lr
3814 // callee expects it in r3
3815 __ mov(r3, lr);
3816 __ br(target_reg);
3817 %}
3818
3819 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3820 C2_MacroAssembler _masm(&cbuf);
3821 Register oop = as_Register($object$$reg);
3822 Register box = as_Register($box$$reg);
3823 Register disp_hdr = as_Register($tmp$$reg);
3824 Register tmp = as_Register($tmp2$$reg);
3825 Label cont;
3826 Label object_has_monitor;
3827 Label count, no_count;
3828
3829 assert_different_registers(oop, box, tmp, disp_hdr);
3830
3831 // Load markWord from object into displaced_header.
3832 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3833
3834 if (DiagnoseSyncOnValueBasedClasses != 0) {
3835 __ load_klass(tmp, oop);
3836 __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
3837 __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
3838 __ br(Assembler::NE, cont);
3839 }
3840
3841 // Check for existing monitor
3842 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3843
3844 if (LockingMode == LM_MONITOR) {
3845 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3846 __ b(cont);
3847 } else if (LockingMode == LM_LEGACY) {
3848 // Set tmp to be (markWord of object | UNLOCK_VALUE).
3849 __ orr(tmp, disp_hdr, markWord::unlocked_value);
3850
3851 // Initialize the box. (Must happen before we update the object mark!)
3852 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3853
3854 // Compare object markWord with an unlocked value (tmp) and if
3855 // equal exchange the stack address of our box with object markWord.
3856 // On failure disp_hdr contains the possibly locked markWord.
3857 __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3858 /*release*/ true, /*weak*/ false, disp_hdr);
3859 __ br(Assembler::EQ, cont);
3860
3861 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3862
3863 // If the compare-and-exchange succeeded, then we found an unlocked
3864 // object, will have now locked it will continue at label cont
3865
3866 // Check if the owner is self by comparing the value in the
3867 // markWord of object (disp_hdr) with the stack pointer.
3868 __ mov(rscratch1, sp);
3869 __ sub(disp_hdr, disp_hdr, rscratch1);
3870 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
3871 // If condition is true we are cont and hence we can store 0 as the
3872 // displaced header in the box, which indicates that it is a recursive lock.
3873 __ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
3874 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3875 __ b(cont);
3876 } else {
3877 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
3878 __ fast_lock(oop, disp_hdr, tmp, rscratch1, no_count);
3879 __ b(count);
3880 }
3881
3882 // Handle existing monitor.
3883 __ bind(object_has_monitor);
3884
3885 // The object's monitor m is unlocked iff m->owner == NULL,
3886 // otherwise m->owner may contain a thread or a stack address.
3887 //
3888 // Try to CAS m->owner from NULL to current thread.
3889 __ add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value));
3890 __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3891 /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
3892
3893 if (LockingMode != LM_LIGHTWEIGHT) {
3894 // Store a non-null value into the box to avoid looking like a re-entrant
3895 // lock. The fast-path monitor unlock code checks for
3896 // markWord::monitor_value so use markWord::unused_mark which has the
3897 // relevant bit set, and also matches ObjectSynchronizer::enter.
3898 __ mov(tmp, (address)markWord::unused_mark().value());
3899 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3900 }
3901 __ br(Assembler::EQ, cont); // CAS success means locking succeeded
3902
3903 __ cmp(rscratch1, rthread);
3904 __ br(Assembler::NE, cont); // Check for recursive locking
3905
3906 // Recursive lock case
3907 __ increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1);
3908 // flag == EQ still from the cmp above, checking if this is a reentrant lock
3909
3910 __ bind(cont);
3911 // flag == EQ indicates success
3912 // flag == NE indicates failure
3913 __ br(Assembler::NE, no_count);
3914
3915 __ bind(count);
3916 __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
3917
3918 __ bind(no_count);
3919 %}
3920
3921 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3922 C2_MacroAssembler _masm(&cbuf);
3923 Register oop = as_Register($object$$reg);
3924 Register box = as_Register($box$$reg);
3925 Register disp_hdr = as_Register($tmp$$reg);
3926 Register tmp = as_Register($tmp2$$reg);
3927 Label cont;
3928 Label object_has_monitor;
3929 Label count, no_count;
3930
3931 assert_different_registers(oop, box, tmp, disp_hdr);
3932
3933 if (LockingMode == LM_LEGACY) {
3934 // Find the lock address and load the displaced header from the stack.
3935 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3936
3937 // If the displaced header is 0, we have a recursive unlock.
3938 __ cmp(disp_hdr, zr);
3939 __ br(Assembler::EQ, cont);
3940 }
3941
3942 // Handle existing monitor.
3943 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3944 __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
3945
3946 if (LockingMode == LM_MONITOR) {
3947 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3948 __ b(cont);
3949 } else if (LockingMode == LM_LEGACY) {
3950 // Check if it is still a light weight lock, this is is true if we
3951 // see the stack address of the basicLock in the markWord of the
3952 // object.
3953
3954 __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
3955 /*release*/ true, /*weak*/ false, tmp);
3956 __ b(cont);
3957 } else {
3958 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
3959 __ fast_unlock(oop, tmp, box, disp_hdr, no_count);
3960 __ b(count);
3961 }
3962
3963 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3964
3965 // Handle existing monitor.
3966 __ bind(object_has_monitor);
3967 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
3968 __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
3969
3970 if (LockingMode == LM_LIGHTWEIGHT) {
3971 // If the owner is anonymous, we need to fix it -- in an outline stub.
3972 Register tmp2 = disp_hdr;
3973 __ ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset()));
3974 // We cannot use tbnz here, the target might be too far away and cannot
3975 // be encoded.
3976 __ tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER);
3977 C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
3978 Compile::current()->output()->add_stub(stub);
3979 __ br(Assembler::NE, stub->entry());
3980 __ bind(stub->continuation());
3981 }
3982
3983 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
3984
3985 Label notRecursive;
3986 __ cbz(disp_hdr, notRecursive);
3987
3988 // Recursive lock
3989 __ sub(disp_hdr, disp_hdr, 1u);
3990 __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
3991 __ cmp(disp_hdr, disp_hdr); // Sets flags for result
3992 __ b(cont);
3993
3994 __ bind(notRecursive);
3995 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset()));
3996 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
3997 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3998 __ cmp(rscratch1, zr); // Sets flags for result
3999 __ cbnz(rscratch1, cont);
4000 // need a release store here
4001 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset()));
4002 __ stlr(zr, tmp); // set unowned
4003
4004 __ bind(cont);
4005 // flag == EQ indicates success
4006 // flag == NE indicates failure
4007 __ br(Assembler::NE, no_count);
4008
4009 __ bind(count);
4010 __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
4011
4012 __ bind(no_count);
4013 %}
4014
4015 %}
4016
4017 //----------FRAME--------------------------------------------------------------
4018 // Definition of frame structure and management information.
4019 //
4020 // S T A C K L A Y O U T Allocators stack-slot number
4021 // | (to get allocators register number
4022 // G Owned by | | v add OptoReg::stack0())
4023 // r CALLER | |
4024 // o | +--------+ pad to even-align allocators stack-slot
4025 // w V | pad0 | numbers; owned by CALLER
4026 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
4027 // h ^ | in | 5
4028 // | | args | 4 Holes in incoming args owned by SELF
4029 // | | | | 3
4030 // | | +--------+
4031 // V | | old out| Empty on Intel, window on Sparc
4032 // | old |preserve| Must be even aligned.
4033 // | SP-+--------+----> Matcher::_old_SP, even aligned
4034 // | | in | 3 area for Intel ret address
4035 // Owned by |preserve| Empty on Sparc.
4036 // SELF +--------+
4037 // | | pad2 | 2 pad to align old SP
4038 // | +--------+ 1
4039 // | | locks | 0
4040 // | +--------+----> OptoReg::stack0(), even aligned
4041 // | | pad1 | 11 pad to align new SP
4042 // | +--------+
4043 // | | | 10
4044 // | | spills | 9 spills
4045 // V | | 8 (pad0 slot for callee)
4046 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
4047 // ^ | out | 7
4048 // | | args | 6 Holes in outgoing args owned by CALLEE
4049 // Owned by +--------+
4050 // CALLEE | new out| 6 Empty on Intel, window on Sparc
4051 // | new |preserve| Must be even-aligned.
4052 // | SP-+--------+----> Matcher::_new_SP, even aligned
4053 // | | |
4054 //
4055 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
4056 // known from SELF's arguments and the Java calling convention.
4057 // Region 6-7 is determined per call site.
4058 // Note 2: If the calling convention leaves holes in the incoming argument
4059 // area, those holes are owned by SELF. Holes in the outgoing area
4060 // are owned by the CALLEE. Holes should not be necessary in the
4061 // incoming area, as the Java calling convention is completely under
4062 // the control of the AD file. Doubles can be sorted and packed to
4063 // avoid holes. Holes in the outgoing arguments may be necessary for
4064 // varargs C calling conventions.
4065 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
4066 // even aligned with pad0 as needed.
4067 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
4068 // (the latter is true on Intel but is it false on AArch64?)
4069 // region 6-11 is even aligned; it may be padded out more so that
4070 // the region from SP to FP meets the minimum stack alignment.
4071 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
4072 // alignment. Region 11, pad1, may be dynamically extended so that
4073 // SP meets the minimum alignment.
4074
4075 frame %{
4076 // These three registers define part of the calling convention
4077 // between compiled code and the interpreter.
4078
4079 // Inline Cache Register or Method for I2C.
4080 inline_cache_reg(R12);
4081
4082 // Number of stack slots consumed by locking an object
4083 sync_stack_slots(2);
4084
4085 // Compiled code's Frame Pointer
4086 frame_pointer(R31);
4087
4088 // Interpreter stores its frame pointer in a register which is
4089 // stored to the stack by I2CAdaptors.
4090 // I2CAdaptors convert from interpreted java to compiled java.
4091 interpreter_frame_pointer(R29);
4092
4093 // Stack alignment requirement
4094 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4095
4096 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4097 // for calls to C. Supports the var-args backing area for register parms.
4098 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4099
4100 // The after-PROLOG location of the return address. Location of
4101 // return address specifies a type (REG or STACK) and a number
4102 // representing the register number (i.e. - use a register name) or
4103 // stack slot.
4104 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4105 // Otherwise, it is above the locks and verification slot and alignment word
4106 // TODO this may well be correct but need to check why that - 2 is there
4107 // ppc port uses 0 but we definitely need to allow for fixed_slots
4108 // which folds in the space used for monitors
4109 return_addr(STACK - 2 +
4110 align_up((Compile::current()->in_preserve_stack_slots() +
4111 Compile::current()->fixed_slots()),
4112 stack_alignment_in_slots()));
4113
4114 // Location of compiled Java return values. Same as C for now.
4115 return_value
4116 %{
4117 // TODO do we allow ideal_reg == Op_RegN???
4118 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4119 "only return normal values");
4120
4121 static const int lo[Op_RegL + 1] = { // enum name
4122 0, // Op_Node
4123 0, // Op_Set
4124 R0_num, // Op_RegN
4125 R0_num, // Op_RegI
4126 R0_num, // Op_RegP
4127 V0_num, // Op_RegF
4128 V0_num, // Op_RegD
4129 R0_num // Op_RegL
4130 };
4131
4132 static const int hi[Op_RegL + 1] = { // enum name
4133 0, // Op_Node
4134 0, // Op_Set
4135 OptoReg::Bad, // Op_RegN
4136 OptoReg::Bad, // Op_RegI
4137 R0_H_num, // Op_RegP
4138 OptoReg::Bad, // Op_RegF
4139 V0_H_num, // Op_RegD
4140 R0_H_num // Op_RegL
4141 };
4142
4143 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4144 %}
4145 %}
4146
4147 //----------ATTRIBUTES---------------------------------------------------------
4148 //----------Operand Attributes-------------------------------------------------
4149 op_attrib op_cost(1); // Required cost attribute
4150
4151 //----------Instruction Attributes---------------------------------------------
4152 ins_attrib ins_cost(INSN_COST); // Required cost attribute
4153 ins_attrib ins_size(32); // Required size attribute (in bits)
4154 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4155 // a non-matching short branch variant
4156 // of some long branch?
4157 ins_attrib ins_alignment(4); // Required alignment attribute (must
4158 // be a power of 2) specifies the
4159 // alignment that some part of the
4160 // instruction (not necessarily the
4161 // start) requires. If > 1, a
4162 // compute_padding() function must be
4163 // provided for the instruction
4164
4165 //----------OPERANDS-----------------------------------------------------------
4166 // Operand definitions must precede instruction definitions for correct parsing
4167 // in the ADLC because operands constitute user defined types which are used in
4168 // instruction definitions.
4169
4170 //----------Simple Operands----------------------------------------------------
4171
4172 // Integer operands 32 bit
4173 // 32 bit immediate
4174 operand immI()
4175 %{
4176 match(ConI);
4177
4178 op_cost(0);
4179 format %{ %}
4180 interface(CONST_INTER);
4181 %}
4182
4183 // 32 bit zero
4184 operand immI0()
4185 %{
4186 predicate(n->get_int() == 0);
4187 match(ConI);
4188
4189 op_cost(0);
4190 format %{ %}
4191 interface(CONST_INTER);
4192 %}
4193
4194 // 32 bit unit increment
4195 operand immI_1()
4196 %{
4197 predicate(n->get_int() == 1);
4198 match(ConI);
4199
4200 op_cost(0);
4201 format %{ %}
4202 interface(CONST_INTER);
4203 %}
4204
4205 // 32 bit unit decrement
4206 operand immI_M1()
4207 %{
4208 predicate(n->get_int() == -1);
4209 match(ConI);
4210
4211 op_cost(0);
4212 format %{ %}
4213 interface(CONST_INTER);
4214 %}
4215
4216 // Shift values for add/sub extension shift
4217 operand immIExt()
4218 %{
4219 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4220 match(ConI);
4221
4222 op_cost(0);
4223 format %{ %}
4224 interface(CONST_INTER);
4225 %}
4226
4227 operand immI_gt_1()
4228 %{
4229 predicate(n->get_int() > 1);
4230 match(ConI);
4231
4232 op_cost(0);
4233 format %{ %}
4234 interface(CONST_INTER);
4235 %}
4236
4237 operand immI_le_4()
4238 %{
4239 predicate(n->get_int() <= 4);
4240 match(ConI);
4241
4242 op_cost(0);
4243 format %{ %}
4244 interface(CONST_INTER);
4245 %}
4246
4247 operand immI_16()
4248 %{
4249 predicate(n->get_int() == 16);
4250 match(ConI);
4251
4252 op_cost(0);
4253 format %{ %}
4254 interface(CONST_INTER);
4255 %}
4256
4257 operand immI_24()
4258 %{
4259 predicate(n->get_int() == 24);
4260 match(ConI);
4261
4262 op_cost(0);
4263 format %{ %}
4264 interface(CONST_INTER);
4265 %}
4266
4267 operand immI_32()
4268 %{
4269 predicate(n->get_int() == 32);
4270 match(ConI);
4271
4272 op_cost(0);
4273 format %{ %}
4274 interface(CONST_INTER);
4275 %}
4276
4277 operand immI_48()
4278 %{
4279 predicate(n->get_int() == 48);
4280 match(ConI);
4281
4282 op_cost(0);
4283 format %{ %}
4284 interface(CONST_INTER);
4285 %}
4286
4287 operand immI_56()
4288 %{
4289 predicate(n->get_int() == 56);
4290 match(ConI);
4291
4292 op_cost(0);
4293 format %{ %}
4294 interface(CONST_INTER);
4295 %}
4296
4297 operand immI_63()
4298 %{
4299 predicate(n->get_int() == 63);
4300 match(ConI);
4301
4302 op_cost(0);
4303 format %{ %}
4304 interface(CONST_INTER);
4305 %}
4306
4307 operand immI_64()
4308 %{
4309 predicate(n->get_int() == 64);
4310 match(ConI);
4311
4312 op_cost(0);
4313 format %{ %}
4314 interface(CONST_INTER);
4315 %}
4316
4317 operand immI_255()
4318 %{
4319 predicate(n->get_int() == 255);
4320 match(ConI);
4321
4322 op_cost(0);
4323 format %{ %}
4324 interface(CONST_INTER);
4325 %}
4326
4327 operand immI_65535()
4328 %{
4329 predicate(n->get_int() == 65535);
4330 match(ConI);
4331
4332 op_cost(0);
4333 format %{ %}
4334 interface(CONST_INTER);
4335 %}
4336
4337 operand immI_positive()
4338 %{
4339 predicate(n->get_int() > 0);
4340 match(ConI);
4341
4342 op_cost(0);
4343 format %{ %}
4344 interface(CONST_INTER);
4345 %}
4346
4347 // BoolTest condition for signed compare
4348 operand immI_cmp_cond()
4349 %{
4350 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4351 match(ConI);
4352
4353 op_cost(0);
4354 format %{ %}
4355 interface(CONST_INTER);
4356 %}
4357
4358 // BoolTest condition for unsigned compare
4359 operand immI_cmpU_cond()
4360 %{
4361 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4362 match(ConI);
4363
4364 op_cost(0);
4365 format %{ %}
4366 interface(CONST_INTER);
4367 %}
4368
4369 operand immL_255()
4370 %{
4371 predicate(n->get_long() == 255L);
4372 match(ConL);
4373
4374 op_cost(0);
4375 format %{ %}
4376 interface(CONST_INTER);
4377 %}
4378
4379 operand immL_65535()
4380 %{
4381 predicate(n->get_long() == 65535L);
4382 match(ConL);
4383
4384 op_cost(0);
4385 format %{ %}
4386 interface(CONST_INTER);
4387 %}
4388
4389 operand immL_4294967295()
4390 %{
4391 predicate(n->get_long() == 4294967295L);
4392 match(ConL);
4393
4394 op_cost(0);
4395 format %{ %}
4396 interface(CONST_INTER);
4397 %}
4398
4399 operand immL_bitmask()
4400 %{
4401 predicate((n->get_long() != 0)
4402 && ((n->get_long() & 0xc000000000000000l) == 0)
4403 && is_power_of_2(n->get_long() + 1));
4404 match(ConL);
4405
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 operand immI_bitmask()
4412 %{
4413 predicate((n->get_int() != 0)
4414 && ((n->get_int() & 0xc0000000) == 0)
4415 && is_power_of_2(n->get_int() + 1));
4416 match(ConI);
4417
4418 op_cost(0);
4419 format %{ %}
4420 interface(CONST_INTER);
4421 %}
4422
4423 operand immL_positive_bitmaskI()
4424 %{
4425 predicate((n->get_long() != 0)
4426 && ((julong)n->get_long() < 0x80000000ULL)
4427 && is_power_of_2(n->get_long() + 1));
4428 match(ConL);
4429
4430 op_cost(0);
4431 format %{ %}
4432 interface(CONST_INTER);
4433 %}
4434
4435 // Scale values for scaled offset addressing modes (up to long but not quad)
4436 operand immIScale()
4437 %{
4438 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4439 match(ConI);
4440
4441 op_cost(0);
4442 format %{ %}
4443 interface(CONST_INTER);
4444 %}
4445
4446 // 26 bit signed offset -- for pc-relative branches
4447 operand immI26()
4448 %{
4449 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4450 match(ConI);
4451
4452 op_cost(0);
4453 format %{ %}
4454 interface(CONST_INTER);
4455 %}
4456
4457 // 19 bit signed offset -- for pc-relative loads
4458 operand immI19()
4459 %{
4460 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4461 match(ConI);
4462
4463 op_cost(0);
4464 format %{ %}
4465 interface(CONST_INTER);
4466 %}
4467
4468 // 5 bit signed integer
4469 operand immI5()
4470 %{
4471 predicate(Assembler::is_simm(n->get_int(), 5));
4472 match(ConI);
4473
4474 op_cost(0);
4475 format %{ %}
4476 interface(CONST_INTER);
4477 %}
4478
4479 // 7 bit unsigned integer
4480 operand immIU7()
4481 %{
4482 predicate(Assembler::is_uimm(n->get_int(), 7));
4483 match(ConI);
4484
4485 op_cost(0);
4486 format %{ %}
4487 interface(CONST_INTER);
4488 %}
4489
4490 // 12 bit unsigned offset -- for base plus immediate loads
4491 operand immIU12()
4492 %{
4493 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4494 match(ConI);
4495
4496 op_cost(0);
4497 format %{ %}
4498 interface(CONST_INTER);
4499 %}
4500
4501 operand immLU12()
4502 %{
4503 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4504 match(ConL);
4505
4506 op_cost(0);
4507 format %{ %}
4508 interface(CONST_INTER);
4509 %}
4510
4511 // Offset for scaled or unscaled immediate loads and stores
4512 operand immIOffset()
4513 %{
4514 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4515 match(ConI);
4516
4517 op_cost(0);
4518 format %{ %}
4519 interface(CONST_INTER);
4520 %}
4521
4522 operand immIOffset1()
4523 %{
4524 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4525 match(ConI);
4526
4527 op_cost(0);
4528 format %{ %}
4529 interface(CONST_INTER);
4530 %}
4531
4532 operand immIOffset2()
4533 %{
4534 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4535 match(ConI);
4536
4537 op_cost(0);
4538 format %{ %}
4539 interface(CONST_INTER);
4540 %}
4541
4542 operand immIOffset4()
4543 %{
4544 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4545 match(ConI);
4546
4547 op_cost(0);
4548 format %{ %}
4549 interface(CONST_INTER);
4550 %}
4551
4552 operand immIOffset8()
4553 %{
4554 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4555 match(ConI);
4556
4557 op_cost(0);
4558 format %{ %}
4559 interface(CONST_INTER);
4560 %}
4561
4562 operand immIOffset16()
4563 %{
4564 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4565 match(ConI);
4566
4567 op_cost(0);
4568 format %{ %}
4569 interface(CONST_INTER);
4570 %}
4571
4572 operand immLoffset()
4573 %{
4574 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4575 match(ConL);
4576
4577 op_cost(0);
4578 format %{ %}
4579 interface(CONST_INTER);
4580 %}
4581
4582 operand immLoffset1()
4583 %{
4584 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4585 match(ConL);
4586
4587 op_cost(0);
4588 format %{ %}
4589 interface(CONST_INTER);
4590 %}
4591
4592 operand immLoffset2()
4593 %{
4594 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4595 match(ConL);
4596
4597 op_cost(0);
4598 format %{ %}
4599 interface(CONST_INTER);
4600 %}
4601
4602 operand immLoffset4()
4603 %{
4604 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4605 match(ConL);
4606
4607 op_cost(0);
4608 format %{ %}
4609 interface(CONST_INTER);
4610 %}
4611
4612 operand immLoffset8()
4613 %{
4614 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4615 match(ConL);
4616
4617 op_cost(0);
4618 format %{ %}
4619 interface(CONST_INTER);
4620 %}
4621
4622 operand immLoffset16()
4623 %{
4624 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4625 match(ConL);
4626
4627 op_cost(0);
4628 format %{ %}
4629 interface(CONST_INTER);
4630 %}
4631
4632 // 5 bit signed long integer
4633 operand immL5()
4634 %{
4635 predicate(Assembler::is_simm(n->get_long(), 5));
4636 match(ConL);
4637
4638 op_cost(0);
4639 format %{ %}
4640 interface(CONST_INTER);
4641 %}
4642
4643 // 7 bit unsigned long integer
4644 operand immLU7()
4645 %{
4646 predicate(Assembler::is_uimm(n->get_long(), 7));
4647 match(ConL);
4648
4649 op_cost(0);
4650 format %{ %}
4651 interface(CONST_INTER);
4652 %}
4653
4654 // 8 bit signed value.
4655 operand immI8()
4656 %{
4657 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4658 match(ConI);
4659
4660 op_cost(0);
4661 format %{ %}
4662 interface(CONST_INTER);
4663 %}
4664
4665 // 8 bit signed value (simm8), or #simm8 LSL 8.
4666 operand immI8_shift8()
4667 %{
4668 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4669 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4670 match(ConI);
4671
4672 op_cost(0);
4673 format %{ %}
4674 interface(CONST_INTER);
4675 %}
4676
4677 // 8 bit signed value (simm8), or #simm8 LSL 8.
4678 operand immL8_shift8()
4679 %{
4680 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4681 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4682 match(ConL);
4683
4684 op_cost(0);
4685 format %{ %}
4686 interface(CONST_INTER);
4687 %}
4688
4689 // 8 bit integer valid for vector add sub immediate
4690 operand immBAddSubV()
4691 %{
4692 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4693 match(ConI);
4694
4695 op_cost(0);
4696 format %{ %}
4697 interface(CONST_INTER);
4698 %}
4699
4700 // 32 bit integer valid for add sub immediate
4701 operand immIAddSub()
4702 %{
4703 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4704 match(ConI);
4705 op_cost(0);
4706 format %{ %}
4707 interface(CONST_INTER);
4708 %}
4709
4710 // 32 bit integer valid for vector add sub immediate
4711 operand immIAddSubV()
4712 %{
4713 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4714 match(ConI);
4715
4716 op_cost(0);
4717 format %{ %}
4718 interface(CONST_INTER);
4719 %}
4720
4721 // 32 bit unsigned integer valid for logical immediate
4722
4723 operand immBLog()
4724 %{
4725 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4726 match(ConI);
4727
4728 op_cost(0);
4729 format %{ %}
4730 interface(CONST_INTER);
4731 %}
4732
4733 operand immSLog()
4734 %{
4735 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4736 match(ConI);
4737
4738 op_cost(0);
4739 format %{ %}
4740 interface(CONST_INTER);
4741 %}
4742
4743 operand immILog()
4744 %{
4745 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4746 match(ConI);
4747
4748 op_cost(0);
4749 format %{ %}
4750 interface(CONST_INTER);
4751 %}
4752
4753 // Integer operands 64 bit
4754 // 64 bit immediate
4755 operand immL()
4756 %{
4757 match(ConL);
4758
4759 op_cost(0);
4760 format %{ %}
4761 interface(CONST_INTER);
4762 %}
4763
4764 // 64 bit zero
4765 operand immL0()
4766 %{
4767 predicate(n->get_long() == 0);
4768 match(ConL);
4769
4770 op_cost(0);
4771 format %{ %}
4772 interface(CONST_INTER);
4773 %}
4774
4775 // 64 bit unit increment
4776 operand immL_1()
4777 %{
4778 predicate(n->get_long() == 1);
4779 match(ConL);
4780
4781 op_cost(0);
4782 format %{ %}
4783 interface(CONST_INTER);
4784 %}
4785
4786 // 64 bit unit decrement
4787 operand immL_M1()
4788 %{
4789 predicate(n->get_long() == -1);
4790 match(ConL);
4791
4792 op_cost(0);
4793 format %{ %}
4794 interface(CONST_INTER);
4795 %}
4796
4797 // 32 bit offset of pc in thread anchor
4798
4799 operand immL_pc_off()
4800 %{
4801 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4802 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4803 match(ConL);
4804
4805 op_cost(0);
4806 format %{ %}
4807 interface(CONST_INTER);
4808 %}
4809
4810 // 64 bit integer valid for add sub immediate
4811 operand immLAddSub()
4812 %{
4813 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4814 match(ConL);
4815 op_cost(0);
4816 format %{ %}
4817 interface(CONST_INTER);
4818 %}
4819
4820 // 64 bit integer valid for addv subv immediate
4821 operand immLAddSubV()
4822 %{
4823 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4824 match(ConL);
4825
4826 op_cost(0);
4827 format %{ %}
4828 interface(CONST_INTER);
4829 %}
4830
4831 // 64 bit integer valid for logical immediate
4832 operand immLLog()
4833 %{
4834 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4835 match(ConL);
4836 op_cost(0);
4837 format %{ %}
4838 interface(CONST_INTER);
4839 %}
4840
4841 // Long Immediate: low 32-bit mask
4842 operand immL_32bits()
4843 %{
4844 predicate(n->get_long() == 0xFFFFFFFFL);
4845 match(ConL);
4846 op_cost(0);
4847 format %{ %}
4848 interface(CONST_INTER);
4849 %}
4850
4851 // Pointer operands
4852 // Pointer Immediate
4853 operand immP()
4854 %{
4855 match(ConP);
4856
4857 op_cost(0);
4858 format %{ %}
4859 interface(CONST_INTER);
4860 %}
4861
4862 // NULL Pointer Immediate
4863 operand immP0()
4864 %{
4865 predicate(n->get_ptr() == 0);
4866 match(ConP);
4867
4868 op_cost(0);
4869 format %{ %}
4870 interface(CONST_INTER);
4871 %}
4872
4873 // Pointer Immediate One
4874 // this is used in object initialization (initial object header)
4875 operand immP_1()
4876 %{
4877 predicate(n->get_ptr() == 1);
4878 match(ConP);
4879
4880 op_cost(0);
4881 format %{ %}
4882 interface(CONST_INTER);
4883 %}
4884
4885 // Card Table Byte Map Base
4886 operand immByteMapBase()
4887 %{
4888 // Get base of card map
4889 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4890 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4891 match(ConP);
4892
4893 op_cost(0);
4894 format %{ %}
4895 interface(CONST_INTER);
4896 %}
4897
4898 // Pointer Immediate Minus One
4899 // this is used when we want to write the current PC to the thread anchor
4900 operand immP_M1()
4901 %{
4902 predicate(n->get_ptr() == -1);
4903 match(ConP);
4904
4905 op_cost(0);
4906 format %{ %}
4907 interface(CONST_INTER);
4908 %}
4909
4910 // Pointer Immediate Minus Two
4911 // this is used when we want to write the current PC to the thread anchor
4912 operand immP_M2()
4913 %{
4914 predicate(n->get_ptr() == -2);
4915 match(ConP);
4916
4917 op_cost(0);
4918 format %{ %}
4919 interface(CONST_INTER);
4920 %}
4921
4922 // Float and Double operands
4923 // Double Immediate
4924 operand immD()
4925 %{
4926 match(ConD);
4927 op_cost(0);
4928 format %{ %}
4929 interface(CONST_INTER);
4930 %}
4931
4932 // Double Immediate: +0.0d
4933 operand immD0()
4934 %{
4935 predicate(jlong_cast(n->getd()) == 0);
4936 match(ConD);
4937
4938 op_cost(0);
4939 format %{ %}
4940 interface(CONST_INTER);
4941 %}
4942
4943 // constant 'double +0.0'.
4944 operand immDPacked()
4945 %{
4946 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4947 match(ConD);
4948 op_cost(0);
4949 format %{ %}
4950 interface(CONST_INTER);
4951 %}
4952
4953 // Float Immediate
4954 operand immF()
4955 %{
4956 match(ConF);
4957 op_cost(0);
4958 format %{ %}
4959 interface(CONST_INTER);
4960 %}
4961
4962 // Float Immediate: +0.0f.
4963 operand immF0()
4964 %{
4965 predicate(jint_cast(n->getf()) == 0);
4966 match(ConF);
4967
4968 op_cost(0);
4969 format %{ %}
4970 interface(CONST_INTER);
4971 %}
4972
4973 //
4974 operand immFPacked()
4975 %{
4976 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4977 match(ConF);
4978 op_cost(0);
4979 format %{ %}
4980 interface(CONST_INTER);
4981 %}
4982
4983 // Narrow pointer operands
4984 // Narrow Pointer Immediate
4985 operand immN()
4986 %{
4987 match(ConN);
4988
4989 op_cost(0);
4990 format %{ %}
4991 interface(CONST_INTER);
4992 %}
4993
4994 // Narrow NULL Pointer Immediate
4995 operand immN0()
4996 %{
4997 predicate(n->get_narrowcon() == 0);
4998 match(ConN);
4999
5000 op_cost(0);
5001 format %{ %}
5002 interface(CONST_INTER);
5003 %}
5004
5005 operand immNKlass()
5006 %{
5007 match(ConNKlass);
5008
5009 op_cost(0);
5010 format %{ %}
5011 interface(CONST_INTER);
5012 %}
5013
5014 // Integer 32 bit Register Operands
5015 // Integer 32 bitRegister (excludes SP)
5016 operand iRegI()
5017 %{
5018 constraint(ALLOC_IN_RC(any_reg32));
5019 match(RegI);
5020 match(iRegINoSp);
5021 op_cost(0);
5022 format %{ %}
5023 interface(REG_INTER);
5024 %}
5025
5026 // Integer 32 bit Register not Special
5027 operand iRegINoSp()
5028 %{
5029 constraint(ALLOC_IN_RC(no_special_reg32));
5030 match(RegI);
5031 op_cost(0);
5032 format %{ %}
5033 interface(REG_INTER);
5034 %}
5035
5036 // Integer 64 bit Register Operands
5037 // Integer 64 bit Register (includes SP)
5038 operand iRegL()
5039 %{
5040 constraint(ALLOC_IN_RC(any_reg));
5041 match(RegL);
5042 match(iRegLNoSp);
5043 op_cost(0);
5044 format %{ %}
5045 interface(REG_INTER);
5046 %}
5047
5048 // Integer 64 bit Register not Special
5049 operand iRegLNoSp()
5050 %{
5051 constraint(ALLOC_IN_RC(no_special_reg));
5052 match(RegL);
5053 match(iRegL_R0);
5054 format %{ %}
5055 interface(REG_INTER);
5056 %}
5057
5058 // Pointer Register Operands
5059 // Pointer Register
5060 operand iRegP()
5061 %{
5062 constraint(ALLOC_IN_RC(ptr_reg));
5063 match(RegP);
5064 match(iRegPNoSp);
5065 match(iRegP_R0);
5066 //match(iRegP_R2);
5067 //match(iRegP_R4);
5068 match(iRegP_R5);
5069 match(thread_RegP);
5070 op_cost(0);
5071 format %{ %}
5072 interface(REG_INTER);
5073 %}
5074
5075 // Pointer 64 bit Register not Special
5076 operand iRegPNoSp()
5077 %{
5078 constraint(ALLOC_IN_RC(no_special_ptr_reg));
5079 match(RegP);
5080 // match(iRegP);
5081 // match(iRegP_R0);
5082 // match(iRegP_R2);
5083 // match(iRegP_R4);
5084 // match(iRegP_R5);
5085 // match(thread_RegP);
5086 op_cost(0);
5087 format %{ %}
5088 interface(REG_INTER);
5089 %}
5090
5091 // Pointer 64 bit Register R0 only
5092 operand iRegP_R0()
5093 %{
5094 constraint(ALLOC_IN_RC(r0_reg));
5095 match(RegP);
5096 // match(iRegP);
5097 match(iRegPNoSp);
5098 op_cost(0);
5099 format %{ %}
5100 interface(REG_INTER);
5101 %}
5102
5103 // Pointer 64 bit Register R1 only
5104 operand iRegP_R1()
5105 %{
5106 constraint(ALLOC_IN_RC(r1_reg));
5107 match(RegP);
5108 // match(iRegP);
5109 match(iRegPNoSp);
5110 op_cost(0);
5111 format %{ %}
5112 interface(REG_INTER);
5113 %}
5114
5115 // Pointer 64 bit Register R2 only
5116 operand iRegP_R2()
5117 %{
5118 constraint(ALLOC_IN_RC(r2_reg));
5119 match(RegP);
5120 // match(iRegP);
5121 match(iRegPNoSp);
5122 op_cost(0);
5123 format %{ %}
5124 interface(REG_INTER);
5125 %}
5126
5127 // Pointer 64 bit Register R3 only
5128 operand iRegP_R3()
5129 %{
5130 constraint(ALLOC_IN_RC(r3_reg));
5131 match(RegP);
5132 // match(iRegP);
5133 match(iRegPNoSp);
5134 op_cost(0);
5135 format %{ %}
5136 interface(REG_INTER);
5137 %}
5138
5139 // Pointer 64 bit Register R4 only
5140 operand iRegP_R4()
5141 %{
5142 constraint(ALLOC_IN_RC(r4_reg));
5143 match(RegP);
5144 // match(iRegP);
5145 match(iRegPNoSp);
5146 op_cost(0);
5147 format %{ %}
5148 interface(REG_INTER);
5149 %}
5150
5151 // Pointer 64 bit Register R5 only
5152 operand iRegP_R5()
5153 %{
5154 constraint(ALLOC_IN_RC(r5_reg));
5155 match(RegP);
5156 // match(iRegP);
5157 match(iRegPNoSp);
5158 op_cost(0);
5159 format %{ %}
5160 interface(REG_INTER);
5161 %}
5162
5163 // Pointer 64 bit Register R10 only
5164 operand iRegP_R10()
5165 %{
5166 constraint(ALLOC_IN_RC(r10_reg));
5167 match(RegP);
5168 // match(iRegP);
5169 match(iRegPNoSp);
5170 op_cost(0);
5171 format %{ %}
5172 interface(REG_INTER);
5173 %}
5174
5175 // Long 64 bit Register R0 only
5176 operand iRegL_R0()
5177 %{
5178 constraint(ALLOC_IN_RC(r0_reg));
5179 match(RegL);
5180 match(iRegLNoSp);
5181 op_cost(0);
5182 format %{ %}
5183 interface(REG_INTER);
5184 %}
5185
5186 // Long 64 bit Register R2 only
5187 operand iRegL_R2()
5188 %{
5189 constraint(ALLOC_IN_RC(r2_reg));
5190 match(RegL);
5191 match(iRegLNoSp);
5192 op_cost(0);
5193 format %{ %}
5194 interface(REG_INTER);
5195 %}
5196
5197 // Long 64 bit Register R3 only
5198 operand iRegL_R3()
5199 %{
5200 constraint(ALLOC_IN_RC(r3_reg));
5201 match(RegL);
5202 match(iRegLNoSp);
5203 op_cost(0);
5204 format %{ %}
5205 interface(REG_INTER);
5206 %}
5207
5208 // Long 64 bit Register R11 only
5209 operand iRegL_R11()
5210 %{
5211 constraint(ALLOC_IN_RC(r11_reg));
5212 match(RegL);
5213 match(iRegLNoSp);
5214 op_cost(0);
5215 format %{ %}
5216 interface(REG_INTER);
5217 %}
5218
5219 // Pointer 64 bit Register FP only
5220 operand iRegP_FP()
5221 %{
5222 constraint(ALLOC_IN_RC(fp_reg));
5223 match(RegP);
5224 // match(iRegP);
5225 op_cost(0);
5226 format %{ %}
5227 interface(REG_INTER);
5228 %}
5229
5230 // Register R0 only
5231 operand iRegI_R0()
5232 %{
5233 constraint(ALLOC_IN_RC(int_r0_reg));
5234 match(RegI);
5235 match(iRegINoSp);
5236 op_cost(0);
5237 format %{ %}
5238 interface(REG_INTER);
5239 %}
5240
5241 // Register R2 only
5242 operand iRegI_R2()
5243 %{
5244 constraint(ALLOC_IN_RC(int_r2_reg));
5245 match(RegI);
5246 match(iRegINoSp);
5247 op_cost(0);
5248 format %{ %}
5249 interface(REG_INTER);
5250 %}
5251
5252 // Register R3 only
5253 operand iRegI_R3()
5254 %{
5255 constraint(ALLOC_IN_RC(int_r3_reg));
5256 match(RegI);
5257 match(iRegINoSp);
5258 op_cost(0);
5259 format %{ %}
5260 interface(REG_INTER);
5261 %}
5262
5263
5264 // Register R4 only
5265 operand iRegI_R4()
5266 %{
5267 constraint(ALLOC_IN_RC(int_r4_reg));
5268 match(RegI);
5269 match(iRegINoSp);
5270 op_cost(0);
5271 format %{ %}
5272 interface(REG_INTER);
5273 %}
5274
5275
5276 // Pointer Register Operands
5277 // Narrow Pointer Register
5278 operand iRegN()
5279 %{
5280 constraint(ALLOC_IN_RC(any_reg32));
5281 match(RegN);
5282 match(iRegNNoSp);
5283 op_cost(0);
5284 format %{ %}
5285 interface(REG_INTER);
5286 %}
5287
5288 operand iRegN_R0()
5289 %{
5290 constraint(ALLOC_IN_RC(r0_reg));
5291 match(iRegN);
5292 op_cost(0);
5293 format %{ %}
5294 interface(REG_INTER);
5295 %}
5296
5297 operand iRegN_R2()
5298 %{
5299 constraint(ALLOC_IN_RC(r2_reg));
5300 match(iRegN);
5301 op_cost(0);
5302 format %{ %}
5303 interface(REG_INTER);
5304 %}
5305
5306 operand iRegN_R3()
5307 %{
5308 constraint(ALLOC_IN_RC(r3_reg));
5309 match(iRegN);
5310 op_cost(0);
5311 format %{ %}
5312 interface(REG_INTER);
5313 %}
5314
5315 // Integer 64 bit Register not Special
5316 operand iRegNNoSp()
5317 %{
5318 constraint(ALLOC_IN_RC(no_special_reg32));
5319 match(RegN);
5320 op_cost(0);
5321 format %{ %}
5322 interface(REG_INTER);
5323 %}
5324
5325 // Float Register
5326 // Float register operands
5327 operand vRegF()
5328 %{
5329 constraint(ALLOC_IN_RC(float_reg));
5330 match(RegF);
5331
5332 op_cost(0);
5333 format %{ %}
5334 interface(REG_INTER);
5335 %}
5336
5337 // Double Register
5338 // Double register operands
5339 operand vRegD()
5340 %{
5341 constraint(ALLOC_IN_RC(double_reg));
5342 match(RegD);
5343
5344 op_cost(0);
5345 format %{ %}
5346 interface(REG_INTER);
5347 %}
5348
5349 // Generic vector class. This will be used for
5350 // all vector operands, including NEON and SVE.
5351 operand vReg()
5352 %{
5353 constraint(ALLOC_IN_RC(dynamic));
5354 match(VecA);
5355 match(VecD);
5356 match(VecX);
5357
5358 op_cost(0);
5359 format %{ %}
5360 interface(REG_INTER);
5361 %}
5362
5363 operand vecA()
5364 %{
5365 constraint(ALLOC_IN_RC(vectora_reg));
5366 match(VecA);
5367
5368 op_cost(0);
5369 format %{ %}
5370 interface(REG_INTER);
5371 %}
5372
5373 operand vecD()
5374 %{
5375 constraint(ALLOC_IN_RC(vectord_reg));
5376 match(VecD);
5377
5378 op_cost(0);
5379 format %{ %}
5380 interface(REG_INTER);
5381 %}
5382
5383 operand vecX()
5384 %{
5385 constraint(ALLOC_IN_RC(vectorx_reg));
5386 match(VecX);
5387
5388 op_cost(0);
5389 format %{ %}
5390 interface(REG_INTER);
5391 %}
5392
5393 operand vRegD_V0()
5394 %{
5395 constraint(ALLOC_IN_RC(v0_reg));
5396 match(RegD);
5397 op_cost(0);
5398 format %{ %}
5399 interface(REG_INTER);
5400 %}
5401
5402 operand vRegD_V1()
5403 %{
5404 constraint(ALLOC_IN_RC(v1_reg));
5405 match(RegD);
5406 op_cost(0);
5407 format %{ %}
5408 interface(REG_INTER);
5409 %}
5410
5411 operand vRegD_V2()
5412 %{
5413 constraint(ALLOC_IN_RC(v2_reg));
5414 match(RegD);
5415 op_cost(0);
5416 format %{ %}
5417 interface(REG_INTER);
5418 %}
5419
5420 operand vRegD_V3()
5421 %{
5422 constraint(ALLOC_IN_RC(v3_reg));
5423 match(RegD);
5424 op_cost(0);
5425 format %{ %}
5426 interface(REG_INTER);
5427 %}
5428
5429 operand vRegD_V4()
5430 %{
5431 constraint(ALLOC_IN_RC(v4_reg));
5432 match(RegD);
5433 op_cost(0);
5434 format %{ %}
5435 interface(REG_INTER);
5436 %}
5437
5438 operand vRegD_V5()
5439 %{
5440 constraint(ALLOC_IN_RC(v5_reg));
5441 match(RegD);
5442 op_cost(0);
5443 format %{ %}
5444 interface(REG_INTER);
5445 %}
5446
5447 operand vRegD_V6()
5448 %{
5449 constraint(ALLOC_IN_RC(v6_reg));
5450 match(RegD);
5451 op_cost(0);
5452 format %{ %}
5453 interface(REG_INTER);
5454 %}
5455
5456 operand vRegD_V7()
5457 %{
5458 constraint(ALLOC_IN_RC(v7_reg));
5459 match(RegD);
5460 op_cost(0);
5461 format %{ %}
5462 interface(REG_INTER);
5463 %}
5464
5465 operand vRegD_V8()
5466 %{
5467 constraint(ALLOC_IN_RC(v8_reg));
5468 match(RegD);
5469 op_cost(0);
5470 format %{ %}
5471 interface(REG_INTER);
5472 %}
5473
5474 operand vRegD_V9()
5475 %{
5476 constraint(ALLOC_IN_RC(v9_reg));
5477 match(RegD);
5478 op_cost(0);
5479 format %{ %}
5480 interface(REG_INTER);
5481 %}
5482
5483 operand vRegD_V10()
5484 %{
5485 constraint(ALLOC_IN_RC(v10_reg));
5486 match(RegD);
5487 op_cost(0);
5488 format %{ %}
5489 interface(REG_INTER);
5490 %}
5491
5492 operand vRegD_V11()
5493 %{
5494 constraint(ALLOC_IN_RC(v11_reg));
5495 match(RegD);
5496 op_cost(0);
5497 format %{ %}
5498 interface(REG_INTER);
5499 %}
5500
5501 operand vRegD_V12()
5502 %{
5503 constraint(ALLOC_IN_RC(v12_reg));
5504 match(RegD);
5505 op_cost(0);
5506 format %{ %}
5507 interface(REG_INTER);
5508 %}
5509
5510 operand vRegD_V13()
5511 %{
5512 constraint(ALLOC_IN_RC(v13_reg));
5513 match(RegD);
5514 op_cost(0);
5515 format %{ %}
5516 interface(REG_INTER);
5517 %}
5518
5519 operand vRegD_V14()
5520 %{
5521 constraint(ALLOC_IN_RC(v14_reg));
5522 match(RegD);
5523 op_cost(0);
5524 format %{ %}
5525 interface(REG_INTER);
5526 %}
5527
5528 operand vRegD_V15()
5529 %{
5530 constraint(ALLOC_IN_RC(v15_reg));
5531 match(RegD);
5532 op_cost(0);
5533 format %{ %}
5534 interface(REG_INTER);
5535 %}
5536
5537 operand vRegD_V16()
5538 %{
5539 constraint(ALLOC_IN_RC(v16_reg));
5540 match(RegD);
5541 op_cost(0);
5542 format %{ %}
5543 interface(REG_INTER);
5544 %}
5545
5546 operand vRegD_V17()
5547 %{
5548 constraint(ALLOC_IN_RC(v17_reg));
5549 match(RegD);
5550 op_cost(0);
5551 format %{ %}
5552 interface(REG_INTER);
5553 %}
5554
5555 operand vRegD_V18()
5556 %{
5557 constraint(ALLOC_IN_RC(v18_reg));
5558 match(RegD);
5559 op_cost(0);
5560 format %{ %}
5561 interface(REG_INTER);
5562 %}
5563
5564 operand vRegD_V19()
5565 %{
5566 constraint(ALLOC_IN_RC(v19_reg));
5567 match(RegD);
5568 op_cost(0);
5569 format %{ %}
5570 interface(REG_INTER);
5571 %}
5572
5573 operand vRegD_V20()
5574 %{
5575 constraint(ALLOC_IN_RC(v20_reg));
5576 match(RegD);
5577 op_cost(0);
5578 format %{ %}
5579 interface(REG_INTER);
5580 %}
5581
5582 operand vRegD_V21()
5583 %{
5584 constraint(ALLOC_IN_RC(v21_reg));
5585 match(RegD);
5586 op_cost(0);
5587 format %{ %}
5588 interface(REG_INTER);
5589 %}
5590
5591 operand vRegD_V22()
5592 %{
5593 constraint(ALLOC_IN_RC(v22_reg));
5594 match(RegD);
5595 op_cost(0);
5596 format %{ %}
5597 interface(REG_INTER);
5598 %}
5599
5600 operand vRegD_V23()
5601 %{
5602 constraint(ALLOC_IN_RC(v23_reg));
5603 match(RegD);
5604 op_cost(0);
5605 format %{ %}
5606 interface(REG_INTER);
5607 %}
5608
5609 operand vRegD_V24()
5610 %{
5611 constraint(ALLOC_IN_RC(v24_reg));
5612 match(RegD);
5613 op_cost(0);
5614 format %{ %}
5615 interface(REG_INTER);
5616 %}
5617
5618 operand vRegD_V25()
5619 %{
5620 constraint(ALLOC_IN_RC(v25_reg));
5621 match(RegD);
5622 op_cost(0);
5623 format %{ %}
5624 interface(REG_INTER);
5625 %}
5626
5627 operand vRegD_V26()
5628 %{
5629 constraint(ALLOC_IN_RC(v26_reg));
5630 match(RegD);
5631 op_cost(0);
5632 format %{ %}
5633 interface(REG_INTER);
5634 %}
5635
5636 operand vRegD_V27()
5637 %{
5638 constraint(ALLOC_IN_RC(v27_reg));
5639 match(RegD);
5640 op_cost(0);
5641 format %{ %}
5642 interface(REG_INTER);
5643 %}
5644
5645 operand vRegD_V28()
5646 %{
5647 constraint(ALLOC_IN_RC(v28_reg));
5648 match(RegD);
5649 op_cost(0);
5650 format %{ %}
5651 interface(REG_INTER);
5652 %}
5653
5654 operand vRegD_V29()
5655 %{
5656 constraint(ALLOC_IN_RC(v29_reg));
5657 match(RegD);
5658 op_cost(0);
5659 format %{ %}
5660 interface(REG_INTER);
5661 %}
5662
5663 operand vRegD_V30()
5664 %{
5665 constraint(ALLOC_IN_RC(v30_reg));
5666 match(RegD);
5667 op_cost(0);
5668 format %{ %}
5669 interface(REG_INTER);
5670 %}
5671
5672 operand vRegD_V31()
5673 %{
5674 constraint(ALLOC_IN_RC(v31_reg));
5675 match(RegD);
5676 op_cost(0);
5677 format %{ %}
5678 interface(REG_INTER);
5679 %}
5680
5681 operand pReg()
5682 %{
5683 constraint(ALLOC_IN_RC(pr_reg));
5684 match(RegVectMask);
5685 match(pRegGov);
5686 op_cost(0);
5687 format %{ %}
5688 interface(REG_INTER);
5689 %}
5690
5691 operand pRegGov()
5692 %{
5693 constraint(ALLOC_IN_RC(gov_pr));
5694 match(RegVectMask);
5695 match(pReg);
5696 op_cost(0);
5697 format %{ %}
5698 interface(REG_INTER);
5699 %}
5700
5701 operand pRegGov_P0()
5702 %{
5703 constraint(ALLOC_IN_RC(p0_reg));
5704 match(RegVectMask);
5705 op_cost(0);
5706 format %{ %}
5707 interface(REG_INTER);
5708 %}
5709
5710 operand pRegGov_P1()
5711 %{
5712 constraint(ALLOC_IN_RC(p1_reg));
5713 match(RegVectMask);
5714 op_cost(0);
5715 format %{ %}
5716 interface(REG_INTER);
5717 %}
5718
5719 // Flags register, used as output of signed compare instructions
5720
5721 // note that on AArch64 we also use this register as the output for
5722 // for floating point compare instructions (CmpF CmpD). this ensures
5723 // that ordered inequality tests use GT, GE, LT or LE none of which
5724 // pass through cases where the result is unordered i.e. one or both
5725 // inputs to the compare is a NaN. this means that the ideal code can
5726 // replace e.g. a GT with an LE and not end up capturing the NaN case
5727 // (where the comparison should always fail). EQ and NE tests are
5728 // always generated in ideal code so that unordered folds into the NE
5729 // case, matching the behaviour of AArch64 NE.
5730 //
5731 // This differs from x86 where the outputs of FP compares use a
5732 // special FP flags registers and where compares based on this
5733 // register are distinguished into ordered inequalities (cmpOpUCF) and
5734 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5735 // to explicitly handle the unordered case in branches. x86 also has
5736 // to include extra CMoveX rules to accept a cmpOpUCF input.
5737
5738 operand rFlagsReg()
5739 %{
5740 constraint(ALLOC_IN_RC(int_flags));
5741 match(RegFlags);
5742
5743 op_cost(0);
5744 format %{ "RFLAGS" %}
5745 interface(REG_INTER);
5746 %}
5747
5748 // Flags register, used as output of unsigned compare instructions
5749 operand rFlagsRegU()
5750 %{
5751 constraint(ALLOC_IN_RC(int_flags));
5752 match(RegFlags);
5753
5754 op_cost(0);
5755 format %{ "RFLAGSU" %}
5756 interface(REG_INTER);
5757 %}
5758
5759 // Special Registers
5760
5761 // Method Register
5762 operand inline_cache_RegP(iRegP reg)
5763 %{
5764 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5765 match(reg);
5766 match(iRegPNoSp);
5767 op_cost(0);
5768 format %{ %}
5769 interface(REG_INTER);
5770 %}
5771
5772 // Thread Register
5773 operand thread_RegP(iRegP reg)
5774 %{
5775 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5776 match(reg);
5777 op_cost(0);
5778 format %{ %}
5779 interface(REG_INTER);
5780 %}
5781
5782 operand lr_RegP(iRegP reg)
5783 %{
5784 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5785 match(reg);
5786 op_cost(0);
5787 format %{ %}
5788 interface(REG_INTER);
5789 %}
5790
5791 //----------Memory Operands----------------------------------------------------
5792
5793 operand indirect(iRegP reg)
5794 %{
5795 constraint(ALLOC_IN_RC(ptr_reg));
5796 match(reg);
5797 op_cost(0);
5798 format %{ "[$reg]" %}
5799 interface(MEMORY_INTER) %{
5800 base($reg);
5801 index(0xffffffff);
5802 scale(0x0);
5803 disp(0x0);
5804 %}
5805 %}
5806
5807 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5808 %{
5809 constraint(ALLOC_IN_RC(ptr_reg));
5810 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5811 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5812 op_cost(0);
5813 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5814 interface(MEMORY_INTER) %{
5815 base($reg);
5816 index($ireg);
5817 scale($scale);
5818 disp(0x0);
5819 %}
5820 %}
5821
5822 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5823 %{
5824 constraint(ALLOC_IN_RC(ptr_reg));
5825 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5826 match(AddP reg (LShiftL lreg scale));
5827 op_cost(0);
5828 format %{ "$reg, $lreg lsl($scale)" %}
5829 interface(MEMORY_INTER) %{
5830 base($reg);
5831 index($lreg);
5832 scale($scale);
5833 disp(0x0);
5834 %}
5835 %}
5836
5837 operand indIndexI2L(iRegP reg, iRegI ireg)
5838 %{
5839 constraint(ALLOC_IN_RC(ptr_reg));
5840 match(AddP reg (ConvI2L ireg));
5841 op_cost(0);
5842 format %{ "$reg, $ireg, 0, I2L" %}
5843 interface(MEMORY_INTER) %{
5844 base($reg);
5845 index($ireg);
5846 scale(0x0);
5847 disp(0x0);
5848 %}
5849 %}
5850
5851 operand indIndex(iRegP reg, iRegL lreg)
5852 %{
5853 constraint(ALLOC_IN_RC(ptr_reg));
5854 match(AddP reg lreg);
5855 op_cost(0);
5856 format %{ "$reg, $lreg" %}
5857 interface(MEMORY_INTER) %{
5858 base($reg);
5859 index($lreg);
5860 scale(0x0);
5861 disp(0x0);
5862 %}
5863 %}
5864
5865 operand indOffI(iRegP reg, immIOffset off)
5866 %{
5867 constraint(ALLOC_IN_RC(ptr_reg));
5868 match(AddP reg off);
5869 op_cost(0);
5870 format %{ "[$reg, $off]" %}
5871 interface(MEMORY_INTER) %{
5872 base($reg);
5873 index(0xffffffff);
5874 scale(0x0);
5875 disp($off);
5876 %}
5877 %}
5878
5879 operand indOffI1(iRegP reg, immIOffset1 off)
5880 %{
5881 constraint(ALLOC_IN_RC(ptr_reg));
5882 match(AddP reg off);
5883 op_cost(0);
5884 format %{ "[$reg, $off]" %}
5885 interface(MEMORY_INTER) %{
5886 base($reg);
5887 index(0xffffffff);
5888 scale(0x0);
5889 disp($off);
5890 %}
5891 %}
5892
5893 operand indOffI2(iRegP reg, immIOffset2 off)
5894 %{
5895 constraint(ALLOC_IN_RC(ptr_reg));
5896 match(AddP reg off);
5897 op_cost(0);
5898 format %{ "[$reg, $off]" %}
5899 interface(MEMORY_INTER) %{
5900 base($reg);
5901 index(0xffffffff);
5902 scale(0x0);
5903 disp($off);
5904 %}
5905 %}
5906
5907 operand indOffI4(iRegP reg, immIOffset4 off)
5908 %{
5909 constraint(ALLOC_IN_RC(ptr_reg));
5910 match(AddP reg off);
5911 op_cost(0);
5912 format %{ "[$reg, $off]" %}
5913 interface(MEMORY_INTER) %{
5914 base($reg);
5915 index(0xffffffff);
5916 scale(0x0);
5917 disp($off);
5918 %}
5919 %}
5920
5921 operand indOffI8(iRegP reg, immIOffset8 off)
5922 %{
5923 constraint(ALLOC_IN_RC(ptr_reg));
5924 match(AddP reg off);
5925 op_cost(0);
5926 format %{ "[$reg, $off]" %}
5927 interface(MEMORY_INTER) %{
5928 base($reg);
5929 index(0xffffffff);
5930 scale(0x0);
5931 disp($off);
5932 %}
5933 %}
5934
5935 operand indOffI16(iRegP reg, immIOffset16 off)
5936 %{
5937 constraint(ALLOC_IN_RC(ptr_reg));
5938 match(AddP reg off);
5939 op_cost(0);
5940 format %{ "[$reg, $off]" %}
5941 interface(MEMORY_INTER) %{
5942 base($reg);
5943 index(0xffffffff);
5944 scale(0x0);
5945 disp($off);
5946 %}
5947 %}
5948
5949 operand indOffL(iRegP reg, immLoffset off)
5950 %{
5951 constraint(ALLOC_IN_RC(ptr_reg));
5952 match(AddP reg off);
5953 op_cost(0);
5954 format %{ "[$reg, $off]" %}
5955 interface(MEMORY_INTER) %{
5956 base($reg);
5957 index(0xffffffff);
5958 scale(0x0);
5959 disp($off);
5960 %}
5961 %}
5962
5963 operand indOffL1(iRegP reg, immLoffset1 off)
5964 %{
5965 constraint(ALLOC_IN_RC(ptr_reg));
5966 match(AddP reg off);
5967 op_cost(0);
5968 format %{ "[$reg, $off]" %}
5969 interface(MEMORY_INTER) %{
5970 base($reg);
5971 index(0xffffffff);
5972 scale(0x0);
5973 disp($off);
5974 %}
5975 %}
5976
5977 operand indOffL2(iRegP reg, immLoffset2 off)
5978 %{
5979 constraint(ALLOC_IN_RC(ptr_reg));
5980 match(AddP reg off);
5981 op_cost(0);
5982 format %{ "[$reg, $off]" %}
5983 interface(MEMORY_INTER) %{
5984 base($reg);
5985 index(0xffffffff);
5986 scale(0x0);
5987 disp($off);
5988 %}
5989 %}
5990
5991 operand indOffL4(iRegP reg, immLoffset4 off)
5992 %{
5993 constraint(ALLOC_IN_RC(ptr_reg));
5994 match(AddP reg off);
5995 op_cost(0);
5996 format %{ "[$reg, $off]" %}
5997 interface(MEMORY_INTER) %{
5998 base($reg);
5999 index(0xffffffff);
6000 scale(0x0);
6001 disp($off);
6002 %}
6003 %}
6004
6005 operand indOffL8(iRegP reg, immLoffset8 off)
6006 %{
6007 constraint(ALLOC_IN_RC(ptr_reg));
6008 match(AddP reg off);
6009 op_cost(0);
6010 format %{ "[$reg, $off]" %}
6011 interface(MEMORY_INTER) %{
6012 base($reg);
6013 index(0xffffffff);
6014 scale(0x0);
6015 disp($off);
6016 %}
6017 %}
6018
6019 operand indOffL16(iRegP reg, immLoffset16 off)
6020 %{
6021 constraint(ALLOC_IN_RC(ptr_reg));
6022 match(AddP reg off);
6023 op_cost(0);
6024 format %{ "[$reg, $off]" %}
6025 interface(MEMORY_INTER) %{
6026 base($reg);
6027 index(0xffffffff);
6028 scale(0x0);
6029 disp($off);
6030 %}
6031 %}
6032
6033 operand indirectN(iRegN reg)
6034 %{
6035 predicate(CompressedOops::shift() == 0);
6036 constraint(ALLOC_IN_RC(ptr_reg));
6037 match(DecodeN reg);
6038 op_cost(0);
6039 format %{ "[$reg]\t# narrow" %}
6040 interface(MEMORY_INTER) %{
6041 base($reg);
6042 index(0xffffffff);
6043 scale(0x0);
6044 disp(0x0);
6045 %}
6046 %}
6047
6048 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
6049 %{
6050 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
6051 constraint(ALLOC_IN_RC(ptr_reg));
6052 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
6053 op_cost(0);
6054 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
6055 interface(MEMORY_INTER) %{
6056 base($reg);
6057 index($ireg);
6058 scale($scale);
6059 disp(0x0);
6060 %}
6061 %}
6062
6063 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
6064 %{
6065 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
6066 constraint(ALLOC_IN_RC(ptr_reg));
6067 match(AddP (DecodeN reg) (LShiftL lreg scale));
6068 op_cost(0);
6069 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
6070 interface(MEMORY_INTER) %{
6071 base($reg);
6072 index($lreg);
6073 scale($scale);
6074 disp(0x0);
6075 %}
6076 %}
6077
6078 operand indIndexI2LN(iRegN reg, iRegI ireg)
6079 %{
6080 predicate(CompressedOops::shift() == 0);
6081 constraint(ALLOC_IN_RC(ptr_reg));
6082 match(AddP (DecodeN reg) (ConvI2L ireg));
6083 op_cost(0);
6084 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
6085 interface(MEMORY_INTER) %{
6086 base($reg);
6087 index($ireg);
6088 scale(0x0);
6089 disp(0x0);
6090 %}
6091 %}
6092
6093 operand indIndexN(iRegN reg, iRegL lreg)
6094 %{
6095 predicate(CompressedOops::shift() == 0);
6096 constraint(ALLOC_IN_RC(ptr_reg));
6097 match(AddP (DecodeN reg) lreg);
6098 op_cost(0);
6099 format %{ "$reg, $lreg\t# narrow" %}
6100 interface(MEMORY_INTER) %{
6101 base($reg);
6102 index($lreg);
6103 scale(0x0);
6104 disp(0x0);
6105 %}
6106 %}
6107
6108 operand indOffIN(iRegN reg, immIOffset off)
6109 %{
6110 predicate(CompressedOops::shift() == 0);
6111 constraint(ALLOC_IN_RC(ptr_reg));
6112 match(AddP (DecodeN reg) off);
6113 op_cost(0);
6114 format %{ "[$reg, $off]\t# narrow" %}
6115 interface(MEMORY_INTER) %{
6116 base($reg);
6117 index(0xffffffff);
6118 scale(0x0);
6119 disp($off);
6120 %}
6121 %}
6122
6123 operand indOffLN(iRegN reg, immLoffset off)
6124 %{
6125 predicate(CompressedOops::shift() == 0);
6126 constraint(ALLOC_IN_RC(ptr_reg));
6127 match(AddP (DecodeN reg) off);
6128 op_cost(0);
6129 format %{ "[$reg, $off]\t# narrow" %}
6130 interface(MEMORY_INTER) %{
6131 base($reg);
6132 index(0xffffffff);
6133 scale(0x0);
6134 disp($off);
6135 %}
6136 %}
6137
6138
6139
6140 // AArch64 opto stubs need to write to the pc slot in the thread anchor
6141 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
6142 %{
6143 constraint(ALLOC_IN_RC(ptr_reg));
6144 match(AddP reg off);
6145 op_cost(0);
6146 format %{ "[$reg, $off]" %}
6147 interface(MEMORY_INTER) %{
6148 base($reg);
6149 index(0xffffffff);
6150 scale(0x0);
6151 disp($off);
6152 %}
6153 %}
6154
6155 //----------Special Memory Operands--------------------------------------------
6156 // Stack Slot Operand - This operand is used for loading and storing temporary
6157 // values on the stack where a match requires a value to
6158 // flow through memory.
6159 operand stackSlotP(sRegP reg)
6160 %{
6161 constraint(ALLOC_IN_RC(stack_slots));
6162 op_cost(100);
6163 // No match rule because this operand is only generated in matching
6164 // match(RegP);
6165 format %{ "[$reg]" %}
6166 interface(MEMORY_INTER) %{
6167 base(0x1e); // RSP
6168 index(0x0); // No Index
6169 scale(0x0); // No Scale
6170 disp($reg); // Stack Offset
6171 %}
6172 %}
6173
6174 operand stackSlotI(sRegI reg)
6175 %{
6176 constraint(ALLOC_IN_RC(stack_slots));
6177 // No match rule because this operand is only generated in matching
6178 // match(RegI);
6179 format %{ "[$reg]" %}
6180 interface(MEMORY_INTER) %{
6181 base(0x1e); // RSP
6182 index(0x0); // No Index
6183 scale(0x0); // No Scale
6184 disp($reg); // Stack Offset
6185 %}
6186 %}
6187
6188 operand stackSlotF(sRegF reg)
6189 %{
6190 constraint(ALLOC_IN_RC(stack_slots));
6191 // No match rule because this operand is only generated in matching
6192 // match(RegF);
6193 format %{ "[$reg]" %}
6194 interface(MEMORY_INTER) %{
6195 base(0x1e); // RSP
6196 index(0x0); // No Index
6197 scale(0x0); // No Scale
6198 disp($reg); // Stack Offset
6199 %}
6200 %}
6201
6202 operand stackSlotD(sRegD reg)
6203 %{
6204 constraint(ALLOC_IN_RC(stack_slots));
6205 // No match rule because this operand is only generated in matching
6206 // match(RegD);
6207 format %{ "[$reg]" %}
6208 interface(MEMORY_INTER) %{
6209 base(0x1e); // RSP
6210 index(0x0); // No Index
6211 scale(0x0); // No Scale
6212 disp($reg); // Stack Offset
6213 %}
6214 %}
6215
6216 operand stackSlotL(sRegL reg)
6217 %{
6218 constraint(ALLOC_IN_RC(stack_slots));
6219 // No match rule because this operand is only generated in matching
6220 // match(RegL);
6221 format %{ "[$reg]" %}
6222 interface(MEMORY_INTER) %{
6223 base(0x1e); // RSP
6224 index(0x0); // No Index
6225 scale(0x0); // No Scale
6226 disp($reg); // Stack Offset
6227 %}
6228 %}
6229
6230 // Operands for expressing Control Flow
6231 // NOTE: Label is a predefined operand which should not be redefined in
6232 // the AD file. It is generically handled within the ADLC.
6233
6234 //----------Conditional Branch Operands----------------------------------------
6235 // Comparison Op - This is the operation of the comparison, and is limited to
6236 // the following set of codes:
6237 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6238 //
6239 // Other attributes of the comparison, such as unsignedness, are specified
6240 // by the comparison instruction that sets a condition code flags register.
6241 // That result is represented by a flags operand whose subtype is appropriate
6242 // to the unsignedness (etc.) of the comparison.
6243 //
6244 // Later, the instruction which matches both the Comparison Op (a Bool) and
6245 // the flags (produced by the Cmp) specifies the coding of the comparison op
6246 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6247
6248 // used for signed integral comparisons and fp comparisons
6249
6250 operand cmpOp()
6251 %{
6252 match(Bool);
6253
6254 format %{ "" %}
6255 interface(COND_INTER) %{
6256 equal(0x0, "eq");
6257 not_equal(0x1, "ne");
6258 less(0xb, "lt");
6259 greater_equal(0xa, "ge");
6260 less_equal(0xd, "le");
6261 greater(0xc, "gt");
6262 overflow(0x6, "vs");
6263 no_overflow(0x7, "vc");
6264 %}
6265 %}
6266
6267 // used for unsigned integral comparisons
6268
6269 operand cmpOpU()
6270 %{
6271 match(Bool);
6272
6273 format %{ "" %}
6274 interface(COND_INTER) %{
6275 equal(0x0, "eq");
6276 not_equal(0x1, "ne");
6277 less(0x3, "lo");
6278 greater_equal(0x2, "hs");
6279 less_equal(0x9, "ls");
6280 greater(0x8, "hi");
6281 overflow(0x6, "vs");
6282 no_overflow(0x7, "vc");
6283 %}
6284 %}
6285
6286 // used for certain integral comparisons which can be
6287 // converted to cbxx or tbxx instructions
6288
6289 operand cmpOpEqNe()
6290 %{
6291 match(Bool);
6292 op_cost(0);
6293 predicate(n->as_Bool()->_test._test == BoolTest::ne
6294 || n->as_Bool()->_test._test == BoolTest::eq);
6295
6296 format %{ "" %}
6297 interface(COND_INTER) %{
6298 equal(0x0, "eq");
6299 not_equal(0x1, "ne");
6300 less(0xb, "lt");
6301 greater_equal(0xa, "ge");
6302 less_equal(0xd, "le");
6303 greater(0xc, "gt");
6304 overflow(0x6, "vs");
6305 no_overflow(0x7, "vc");
6306 %}
6307 %}
6308
6309 // used for certain integral comparisons which can be
6310 // converted to cbxx or tbxx instructions
6311
6312 operand cmpOpLtGe()
6313 %{
6314 match(Bool);
6315 op_cost(0);
6316
6317 predicate(n->as_Bool()->_test._test == BoolTest::lt
6318 || n->as_Bool()->_test._test == BoolTest::ge);
6319
6320 format %{ "" %}
6321 interface(COND_INTER) %{
6322 equal(0x0, "eq");
6323 not_equal(0x1, "ne");
6324 less(0xb, "lt");
6325 greater_equal(0xa, "ge");
6326 less_equal(0xd, "le");
6327 greater(0xc, "gt");
6328 overflow(0x6, "vs");
6329 no_overflow(0x7, "vc");
6330 %}
6331 %}
6332
6333 // used for certain unsigned integral comparisons which can be
6334 // converted to cbxx or tbxx instructions
6335
6336 operand cmpOpUEqNeLtGe()
6337 %{
6338 match(Bool);
6339 op_cost(0);
6340
6341 predicate(n->as_Bool()->_test._test == BoolTest::eq
6342 || n->as_Bool()->_test._test == BoolTest::ne
6343 || n->as_Bool()->_test._test == BoolTest::lt
6344 || n->as_Bool()->_test._test == BoolTest::ge);
6345
6346 format %{ "" %}
6347 interface(COND_INTER) %{
6348 equal(0x0, "eq");
6349 not_equal(0x1, "ne");
6350 less(0xb, "lt");
6351 greater_equal(0xa, "ge");
6352 less_equal(0xd, "le");
6353 greater(0xc, "gt");
6354 overflow(0x6, "vs");
6355 no_overflow(0x7, "vc");
6356 %}
6357 %}
6358
6359 // Special operand allowing long args to int ops to be truncated for free
6360
6361 operand iRegL2I(iRegL reg) %{
6362
6363 op_cost(0);
6364
6365 match(ConvL2I reg);
6366
6367 format %{ "l2i($reg)" %}
6368
6369 interface(REG_INTER)
6370 %}
6371
6372 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
6373 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
6374 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
6375 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
6376
6377 //----------OPERAND CLASSES----------------------------------------------------
6378 // Operand Classes are groups of operands that are used as to simplify
6379 // instruction definitions by not requiring the AD writer to specify
6380 // separate instructions for every form of operand when the
6381 // instruction accepts multiple operand types with the same basic
6382 // encoding and format. The classic case of this is memory operands.
6383
6384 // memory is used to define read/write location for load/store
6385 // instruction defs. we can turn a memory op into an Address
6386
6387 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
6388 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6389
6390 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
6391 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6392
6393 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
6394 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6395
6396 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
6397 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6398
6399 // All of the memory operands. For the pipeline description.
6400 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
6401 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
6402 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6403
6404
6405 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
6406 // operations. it allows the src to be either an iRegI or a (ConvL2I
6407 // iRegL). in the latter case the l2i normally planted for a ConvL2I
6408 // can be elided because the 32-bit instruction will just employ the
6409 // lower 32 bits anyway.
6410 //
6411 // n.b. this does not elide all L2I conversions. if the truncated
6412 // value is consumed by more than one operation then the ConvL2I
6413 // cannot be bundled into the consuming nodes so an l2i gets planted
6414 // (actually a movw $dst $src) and the downstream instructions consume
6415 // the result of the l2i as an iRegI input. That's a shame since the
6416 // movw is actually redundant but its not too costly.
6417
6418 opclass iRegIorL2I(iRegI, iRegL2I);
6419
6420 //----------PIPELINE-----------------------------------------------------------
6421 // Rules which define the behavior of the target architectures pipeline.
6422
6423 // For specific pipelines, eg A53, define the stages of that pipeline
6424 //pipe_desc(ISS, EX1, EX2, WR);
6425 #define ISS S0
6426 #define EX1 S1
6427 #define EX2 S2
6428 #define WR S3
6429
6430 // Integer ALU reg operation
6431 pipeline %{
6432
6433 attributes %{
6434 // ARM instructions are of fixed length
6435 fixed_size_instructions; // Fixed size instructions TODO does
6436 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
6437 // ARM instructions come in 32-bit word units
6438 instruction_unit_size = 4; // An instruction is 4 bytes long
6439 instruction_fetch_unit_size = 64; // The processor fetches one line
6440 instruction_fetch_units = 1; // of 64 bytes
6441
6442 // List of nop instructions
6443 nops( MachNop );
6444 %}
6445
6446 // We don't use an actual pipeline model so don't care about resources
6447 // or description. we do use pipeline classes to introduce fixed
6448 // latencies
6449
6450 //----------RESOURCES----------------------------------------------------------
6451 // Resources are the functional units available to the machine
6452
6453 resources( INS0, INS1, INS01 = INS0 | INS1,
6454 ALU0, ALU1, ALU = ALU0 | ALU1,
6455 MAC,
6456 DIV,
6457 BRANCH,
6458 LDST,
6459 NEON_FP);
6460
6461 //----------PIPELINE DESCRIPTION-----------------------------------------------
6462 // Pipeline Description specifies the stages in the machine's pipeline
6463
6464 // Define the pipeline as a generic 6 stage pipeline
6465 pipe_desc(S0, S1, S2, S3, S4, S5);
6466
6467 //----------PIPELINE CLASSES---------------------------------------------------
6468 // Pipeline Classes describe the stages in which input and output are
6469 // referenced by the hardware pipeline.
6470
6471 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
6472 %{
6473 single_instruction;
6474 src1 : S1(read);
6475 src2 : S2(read);
6476 dst : S5(write);
6477 INS01 : ISS;
6478 NEON_FP : S5;
6479 %}
6480
6481 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6482 %{
6483 single_instruction;
6484 src1 : S1(read);
6485 src2 : S2(read);
6486 dst : S5(write);
6487 INS01 : ISS;
6488 NEON_FP : S5;
6489 %}
6490
6491 pipe_class fp_uop_s(vRegF dst, vRegF src)
6492 %{
6493 single_instruction;
6494 src : S1(read);
6495 dst : S5(write);
6496 INS01 : ISS;
6497 NEON_FP : S5;
6498 %}
6499
6500 pipe_class fp_uop_d(vRegD dst, vRegD src)
6501 %{
6502 single_instruction;
6503 src : S1(read);
6504 dst : S5(write);
6505 INS01 : ISS;
6506 NEON_FP : S5;
6507 %}
6508
6509 pipe_class fp_d2f(vRegF dst, vRegD src)
6510 %{
6511 single_instruction;
6512 src : S1(read);
6513 dst : S5(write);
6514 INS01 : ISS;
6515 NEON_FP : S5;
6516 %}
6517
6518 pipe_class fp_f2d(vRegD dst, vRegF src)
6519 %{
6520 single_instruction;
6521 src : S1(read);
6522 dst : S5(write);
6523 INS01 : ISS;
6524 NEON_FP : S5;
6525 %}
6526
6527 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6528 %{
6529 single_instruction;
6530 src : S1(read);
6531 dst : S5(write);
6532 INS01 : ISS;
6533 NEON_FP : S5;
6534 %}
6535
6536 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6537 %{
6538 single_instruction;
6539 src : S1(read);
6540 dst : S5(write);
6541 INS01 : ISS;
6542 NEON_FP : S5;
6543 %}
6544
6545 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6546 %{
6547 single_instruction;
6548 src : S1(read);
6549 dst : S5(write);
6550 INS01 : ISS;
6551 NEON_FP : S5;
6552 %}
6553
6554 pipe_class fp_l2f(vRegF dst, iRegL src)
6555 %{
6556 single_instruction;
6557 src : S1(read);
6558 dst : S5(write);
6559 INS01 : ISS;
6560 NEON_FP : S5;
6561 %}
6562
6563 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6564 %{
6565 single_instruction;
6566 src : S1(read);
6567 dst : S5(write);
6568 INS01 : ISS;
6569 NEON_FP : S5;
6570 %}
6571
6572 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6573 %{
6574 single_instruction;
6575 src : S1(read);
6576 dst : S5(write);
6577 INS01 : ISS;
6578 NEON_FP : S5;
6579 %}
6580
6581 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6582 %{
6583 single_instruction;
6584 src : S1(read);
6585 dst : S5(write);
6586 INS01 : ISS;
6587 NEON_FP : S5;
6588 %}
6589
6590 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6591 %{
6592 single_instruction;
6593 src : S1(read);
6594 dst : S5(write);
6595 INS01 : ISS;
6596 NEON_FP : S5;
6597 %}
6598
6599 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6600 %{
6601 single_instruction;
6602 src1 : S1(read);
6603 src2 : S2(read);
6604 dst : S5(write);
6605 INS0 : ISS;
6606 NEON_FP : S5;
6607 %}
6608
6609 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6610 %{
6611 single_instruction;
6612 src1 : S1(read);
6613 src2 : S2(read);
6614 dst : S5(write);
6615 INS0 : ISS;
6616 NEON_FP : S5;
6617 %}
6618
6619 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6620 %{
6621 single_instruction;
6622 cr : S1(read);
6623 src1 : S1(read);
6624 src2 : S1(read);
6625 dst : S3(write);
6626 INS01 : ISS;
6627 NEON_FP : S3;
6628 %}
6629
6630 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6631 %{
6632 single_instruction;
6633 cr : S1(read);
6634 src1 : S1(read);
6635 src2 : S1(read);
6636 dst : S3(write);
6637 INS01 : ISS;
6638 NEON_FP : S3;
6639 %}
6640
6641 pipe_class fp_imm_s(vRegF dst)
6642 %{
6643 single_instruction;
6644 dst : S3(write);
6645 INS01 : ISS;
6646 NEON_FP : S3;
6647 %}
6648
6649 pipe_class fp_imm_d(vRegD dst)
6650 %{
6651 single_instruction;
6652 dst : S3(write);
6653 INS01 : ISS;
6654 NEON_FP : S3;
6655 %}
6656
6657 pipe_class fp_load_constant_s(vRegF dst)
6658 %{
6659 single_instruction;
6660 dst : S4(write);
6661 INS01 : ISS;
6662 NEON_FP : S4;
6663 %}
6664
6665 pipe_class fp_load_constant_d(vRegD dst)
6666 %{
6667 single_instruction;
6668 dst : S4(write);
6669 INS01 : ISS;
6670 NEON_FP : S4;
6671 %}
6672
6673 //------- Integer ALU operations --------------------------
6674
6675 // Integer ALU reg-reg operation
6676 // Operands needed in EX1, result generated in EX2
6677 // Eg. ADD x0, x1, x2
6678 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6679 %{
6680 single_instruction;
6681 dst : EX2(write);
6682 src1 : EX1(read);
6683 src2 : EX1(read);
6684 INS01 : ISS; // Dual issue as instruction 0 or 1
6685 ALU : EX2;
6686 %}
6687
6688 // Integer ALU reg-reg operation with constant shift
6689 // Shifted register must be available in LATE_ISS instead of EX1
6690 // Eg. ADD x0, x1, x2, LSL #2
6691 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6692 %{
6693 single_instruction;
6694 dst : EX2(write);
6695 src1 : EX1(read);
6696 src2 : ISS(read);
6697 INS01 : ISS;
6698 ALU : EX2;
6699 %}
6700
6701 // Integer ALU reg operation with constant shift
6702 // Eg. LSL x0, x1, #shift
6703 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6704 %{
6705 single_instruction;
6706 dst : EX2(write);
6707 src1 : ISS(read);
6708 INS01 : ISS;
6709 ALU : EX2;
6710 %}
6711
6712 // Integer ALU reg-reg operation with variable shift
6713 // Both operands must be available in LATE_ISS instead of EX1
6714 // Result is available in EX1 instead of EX2
6715 // Eg. LSLV x0, x1, x2
6716 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6717 %{
6718 single_instruction;
6719 dst : EX1(write);
6720 src1 : ISS(read);
6721 src2 : ISS(read);
6722 INS01 : ISS;
6723 ALU : EX1;
6724 %}
6725
6726 // Integer ALU reg-reg operation with extract
6727 // As for _vshift above, but result generated in EX2
6728 // Eg. EXTR x0, x1, x2, #N
6729 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6730 %{
6731 single_instruction;
6732 dst : EX2(write);
6733 src1 : ISS(read);
6734 src2 : ISS(read);
6735 INS1 : ISS; // Can only dual issue as Instruction 1
6736 ALU : EX1;
6737 %}
6738
6739 // Integer ALU reg operation
6740 // Eg. NEG x0, x1
6741 pipe_class ialu_reg(iRegI dst, iRegI src)
6742 %{
6743 single_instruction;
6744 dst : EX2(write);
6745 src : EX1(read);
6746 INS01 : ISS;
6747 ALU : EX2;
6748 %}
6749
6750 // Integer ALU reg mmediate operation
6751 // Eg. ADD x0, x1, #N
6752 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6753 %{
6754 single_instruction;
6755 dst : EX2(write);
6756 src1 : EX1(read);
6757 INS01 : ISS;
6758 ALU : EX2;
6759 %}
6760
6761 // Integer ALU immediate operation (no source operands)
6762 // Eg. MOV x0, #N
6763 pipe_class ialu_imm(iRegI dst)
6764 %{
6765 single_instruction;
6766 dst : EX1(write);
6767 INS01 : ISS;
6768 ALU : EX1;
6769 %}
6770
6771 //------- Compare operation -------------------------------
6772
6773 // Compare reg-reg
6774 // Eg. CMP x0, x1
6775 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6776 %{
6777 single_instruction;
6778 // fixed_latency(16);
6779 cr : EX2(write);
6780 op1 : EX1(read);
6781 op2 : EX1(read);
6782 INS01 : ISS;
6783 ALU : EX2;
6784 %}
6785
6786 // Compare reg-reg
6787 // Eg. CMP x0, #N
6788 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6789 %{
6790 single_instruction;
6791 // fixed_latency(16);
6792 cr : EX2(write);
6793 op1 : EX1(read);
6794 INS01 : ISS;
6795 ALU : EX2;
6796 %}
6797
6798 //------- Conditional instructions ------------------------
6799
6800 // Conditional no operands
6801 // Eg. CSINC x0, zr, zr, <cond>
6802 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6803 %{
6804 single_instruction;
6805 cr : EX1(read);
6806 dst : EX2(write);
6807 INS01 : ISS;
6808 ALU : EX2;
6809 %}
6810
6811 // Conditional 2 operand
6812 // EG. CSEL X0, X1, X2, <cond>
6813 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6814 %{
6815 single_instruction;
6816 cr : EX1(read);
6817 src1 : EX1(read);
6818 src2 : EX1(read);
6819 dst : EX2(write);
6820 INS01 : ISS;
6821 ALU : EX2;
6822 %}
6823
6824 // Conditional 2 operand
6825 // EG. CSEL X0, X1, X2, <cond>
6826 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6827 %{
6828 single_instruction;
6829 cr : EX1(read);
6830 src : EX1(read);
6831 dst : EX2(write);
6832 INS01 : ISS;
6833 ALU : EX2;
6834 %}
6835
6836 //------- Multiply pipeline operations --------------------
6837
6838 // Multiply reg-reg
6839 // Eg. MUL w0, w1, w2
6840 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6841 %{
6842 single_instruction;
6843 dst : WR(write);
6844 src1 : ISS(read);
6845 src2 : ISS(read);
6846 INS01 : ISS;
6847 MAC : WR;
6848 %}
6849
6850 // Multiply accumulate
6851 // Eg. MADD w0, w1, w2, w3
6852 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6853 %{
6854 single_instruction;
6855 dst : WR(write);
6856 src1 : ISS(read);
6857 src2 : ISS(read);
6858 src3 : ISS(read);
6859 INS01 : ISS;
6860 MAC : WR;
6861 %}
6862
6863 // Eg. MUL w0, w1, w2
6864 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6865 %{
6866 single_instruction;
6867 fixed_latency(3); // Maximum latency for 64 bit mul
6868 dst : WR(write);
6869 src1 : ISS(read);
6870 src2 : ISS(read);
6871 INS01 : ISS;
6872 MAC : WR;
6873 %}
6874
6875 // Multiply accumulate
6876 // Eg. MADD w0, w1, w2, w3
6877 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6878 %{
6879 single_instruction;
6880 fixed_latency(3); // Maximum latency for 64 bit mul
6881 dst : WR(write);
6882 src1 : ISS(read);
6883 src2 : ISS(read);
6884 src3 : ISS(read);
6885 INS01 : ISS;
6886 MAC : WR;
6887 %}
6888
6889 //------- Divide pipeline operations --------------------
6890
6891 // Eg. SDIV w0, w1, w2
6892 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6893 %{
6894 single_instruction;
6895 fixed_latency(8); // Maximum latency for 32 bit divide
6896 dst : WR(write);
6897 src1 : ISS(read);
6898 src2 : ISS(read);
6899 INS0 : ISS; // Can only dual issue as instruction 0
6900 DIV : WR;
6901 %}
6902
6903 // Eg. SDIV x0, x1, x2
6904 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6905 %{
6906 single_instruction;
6907 fixed_latency(16); // Maximum latency for 64 bit divide
6908 dst : WR(write);
6909 src1 : ISS(read);
6910 src2 : ISS(read);
6911 INS0 : ISS; // Can only dual issue as instruction 0
6912 DIV : WR;
6913 %}
6914
6915 //------- Load pipeline operations ------------------------
6916
6917 // Load - prefetch
6918 // Eg. PFRM <mem>
6919 pipe_class iload_prefetch(memory mem)
6920 %{
6921 single_instruction;
6922 mem : ISS(read);
6923 INS01 : ISS;
6924 LDST : WR;
6925 %}
6926
6927 // Load - reg, mem
6928 // Eg. LDR x0, <mem>
6929 pipe_class iload_reg_mem(iRegI dst, memory mem)
6930 %{
6931 single_instruction;
6932 dst : WR(write);
6933 mem : ISS(read);
6934 INS01 : ISS;
6935 LDST : WR;
6936 %}
6937
6938 // Load - reg, reg
6939 // Eg. LDR x0, [sp, x1]
6940 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6941 %{
6942 single_instruction;
6943 dst : WR(write);
6944 src : ISS(read);
6945 INS01 : ISS;
6946 LDST : WR;
6947 %}
6948
6949 //------- Store pipeline operations -----------------------
6950
6951 // Store - zr, mem
6952 // Eg. STR zr, <mem>
6953 pipe_class istore_mem(memory mem)
6954 %{
6955 single_instruction;
6956 mem : ISS(read);
6957 INS01 : ISS;
6958 LDST : WR;
6959 %}
6960
6961 // Store - reg, mem
6962 // Eg. STR x0, <mem>
6963 pipe_class istore_reg_mem(iRegI src, memory mem)
6964 %{
6965 single_instruction;
6966 mem : ISS(read);
6967 src : EX2(read);
6968 INS01 : ISS;
6969 LDST : WR;
6970 %}
6971
6972 // Store - reg, reg
6973 // Eg. STR x0, [sp, x1]
6974 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6975 %{
6976 single_instruction;
6977 dst : ISS(read);
6978 src : EX2(read);
6979 INS01 : ISS;
6980 LDST : WR;
6981 %}
6982
6983 //------- Store pipeline operations -----------------------
6984
6985 // Branch
6986 pipe_class pipe_branch()
6987 %{
6988 single_instruction;
6989 INS01 : ISS;
6990 BRANCH : EX1;
6991 %}
6992
6993 // Conditional branch
6994 pipe_class pipe_branch_cond(rFlagsReg cr)
6995 %{
6996 single_instruction;
6997 cr : EX1(read);
6998 INS01 : ISS;
6999 BRANCH : EX1;
7000 %}
7001
7002 // Compare & Branch
7003 // EG. CBZ/CBNZ
7004 pipe_class pipe_cmp_branch(iRegI op1)
7005 %{
7006 single_instruction;
7007 op1 : EX1(read);
7008 INS01 : ISS;
7009 BRANCH : EX1;
7010 %}
7011
7012 //------- Synchronisation operations ----------------------
7013
7014 // Any operation requiring serialization.
7015 // EG. DMB/Atomic Ops/Load Acquire/Str Release
7016 pipe_class pipe_serial()
7017 %{
7018 single_instruction;
7019 force_serialization;
7020 fixed_latency(16);
7021 INS01 : ISS(2); // Cannot dual issue with any other instruction
7022 LDST : WR;
7023 %}
7024
7025 // Generic big/slow expanded idiom - also serialized
7026 pipe_class pipe_slow()
7027 %{
7028 instruction_count(10);
7029 multiple_bundles;
7030 force_serialization;
7031 fixed_latency(16);
7032 INS01 : ISS(2); // Cannot dual issue with any other instruction
7033 LDST : WR;
7034 %}
7035
7036 // Empty pipeline class
7037 pipe_class pipe_class_empty()
7038 %{
7039 single_instruction;
7040 fixed_latency(0);
7041 %}
7042
7043 // Default pipeline class.
7044 pipe_class pipe_class_default()
7045 %{
7046 single_instruction;
7047 fixed_latency(2);
7048 %}
7049
7050 // Pipeline class for compares.
7051 pipe_class pipe_class_compare()
7052 %{
7053 single_instruction;
7054 fixed_latency(16);
7055 %}
7056
7057 // Pipeline class for memory operations.
7058 pipe_class pipe_class_memory()
7059 %{
7060 single_instruction;
7061 fixed_latency(16);
7062 %}
7063
7064 // Pipeline class for call.
7065 pipe_class pipe_class_call()
7066 %{
7067 single_instruction;
7068 fixed_latency(100);
7069 %}
7070
7071 // Define the class for the Nop node.
7072 define %{
7073 MachNop = pipe_class_empty;
7074 %}
7075
7076 %}
7077 //----------INSTRUCTIONS-------------------------------------------------------
7078 //
7079 // match -- States which machine-independent subtree may be replaced
7080 // by this instruction.
7081 // ins_cost -- The estimated cost of this instruction is used by instruction
7082 // selection to identify a minimum cost tree of machine
7083 // instructions that matches a tree of machine-independent
7084 // instructions.
7085 // format -- A string providing the disassembly for this instruction.
7086 // The value of an instruction's operand may be inserted
7087 // by referring to it with a '$' prefix.
7088 // opcode -- Three instruction opcodes may be provided. These are referred
7089 // to within an encode class as $primary, $secondary, and $tertiary
7090 // rrspectively. The primary opcode is commonly used to
7091 // indicate the type of machine instruction, while secondary
7092 // and tertiary are often used for prefix options or addressing
7093 // modes.
7094 // ins_encode -- A list of encode classes with parameters. The encode class
7095 // name must have been defined in an 'enc_class' specification
7096 // in the encode section of the architecture description.
7097
7098 // ============================================================================
7099 // Memory (Load/Store) Instructions
7100
7101 // Load Instructions
7102
7103 // Load Byte (8 bit signed)
7104 instruct loadB(iRegINoSp dst, memory1 mem)
7105 %{
7106 match(Set dst (LoadB mem));
7107 predicate(!needs_acquiring_load(n));
7108
7109 ins_cost(4 * INSN_COST);
7110 format %{ "ldrsbw $dst, $mem\t# byte" %}
7111
7112 ins_encode(aarch64_enc_ldrsbw(dst, mem));
7113
7114 ins_pipe(iload_reg_mem);
7115 %}
7116
7117 // Load Byte (8 bit signed) into long
7118 instruct loadB2L(iRegLNoSp dst, memory1 mem)
7119 %{
7120 match(Set dst (ConvI2L (LoadB mem)));
7121 predicate(!needs_acquiring_load(n->in(1)));
7122
7123 ins_cost(4 * INSN_COST);
7124 format %{ "ldrsb $dst, $mem\t# byte" %}
7125
7126 ins_encode(aarch64_enc_ldrsb(dst, mem));
7127
7128 ins_pipe(iload_reg_mem);
7129 %}
7130
7131 // Load Byte (8 bit unsigned)
7132 instruct loadUB(iRegINoSp dst, memory1 mem)
7133 %{
7134 match(Set dst (LoadUB mem));
7135 predicate(!needs_acquiring_load(n));
7136
7137 ins_cost(4 * INSN_COST);
7138 format %{ "ldrbw $dst, $mem\t# byte" %}
7139
7140 ins_encode(aarch64_enc_ldrb(dst, mem));
7141
7142 ins_pipe(iload_reg_mem);
7143 %}
7144
7145 // Load Byte (8 bit unsigned) into long
7146 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
7147 %{
7148 match(Set dst (ConvI2L (LoadUB mem)));
7149 predicate(!needs_acquiring_load(n->in(1)));
7150
7151 ins_cost(4 * INSN_COST);
7152 format %{ "ldrb $dst, $mem\t# byte" %}
7153
7154 ins_encode(aarch64_enc_ldrb(dst, mem));
7155
7156 ins_pipe(iload_reg_mem);
7157 %}
7158
7159 // Load Short (16 bit signed)
7160 instruct loadS(iRegINoSp dst, memory2 mem)
7161 %{
7162 match(Set dst (LoadS mem));
7163 predicate(!needs_acquiring_load(n));
7164
7165 ins_cost(4 * INSN_COST);
7166 format %{ "ldrshw $dst, $mem\t# short" %}
7167
7168 ins_encode(aarch64_enc_ldrshw(dst, mem));
7169
7170 ins_pipe(iload_reg_mem);
7171 %}
7172
7173 // Load Short (16 bit signed) into long
7174 instruct loadS2L(iRegLNoSp dst, memory2 mem)
7175 %{
7176 match(Set dst (ConvI2L (LoadS mem)));
7177 predicate(!needs_acquiring_load(n->in(1)));
7178
7179 ins_cost(4 * INSN_COST);
7180 format %{ "ldrsh $dst, $mem\t# short" %}
7181
7182 ins_encode(aarch64_enc_ldrsh(dst, mem));
7183
7184 ins_pipe(iload_reg_mem);
7185 %}
7186
7187 // Load Char (16 bit unsigned)
7188 instruct loadUS(iRegINoSp dst, memory2 mem)
7189 %{
7190 match(Set dst (LoadUS mem));
7191 predicate(!needs_acquiring_load(n));
7192
7193 ins_cost(4 * INSN_COST);
7194 format %{ "ldrh $dst, $mem\t# short" %}
7195
7196 ins_encode(aarch64_enc_ldrh(dst, mem));
7197
7198 ins_pipe(iload_reg_mem);
7199 %}
7200
7201 // Load Short/Char (16 bit unsigned) into long
7202 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
7203 %{
7204 match(Set dst (ConvI2L (LoadUS mem)));
7205 predicate(!needs_acquiring_load(n->in(1)));
7206
7207 ins_cost(4 * INSN_COST);
7208 format %{ "ldrh $dst, $mem\t# short" %}
7209
7210 ins_encode(aarch64_enc_ldrh(dst, mem));
7211
7212 ins_pipe(iload_reg_mem);
7213 %}
7214
7215 // Load Integer (32 bit signed)
7216 instruct loadI(iRegINoSp dst, memory4 mem)
7217 %{
7218 match(Set dst (LoadI mem));
7219 predicate(!needs_acquiring_load(n));
7220
7221 ins_cost(4 * INSN_COST);
7222 format %{ "ldrw $dst, $mem\t# int" %}
7223
7224 ins_encode(aarch64_enc_ldrw(dst, mem));
7225
7226 ins_pipe(iload_reg_mem);
7227 %}
7228
7229 // Load Integer (32 bit signed) into long
7230 instruct loadI2L(iRegLNoSp dst, memory4 mem)
7231 %{
7232 match(Set dst (ConvI2L (LoadI mem)));
7233 predicate(!needs_acquiring_load(n->in(1)));
7234
7235 ins_cost(4 * INSN_COST);
7236 format %{ "ldrsw $dst, $mem\t# int" %}
7237
7238 ins_encode(aarch64_enc_ldrsw(dst, mem));
7239
7240 ins_pipe(iload_reg_mem);
7241 %}
7242
7243 // Load Integer (32 bit unsigned) into long
7244 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
7245 %{
7246 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7247 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
7248
7249 ins_cost(4 * INSN_COST);
7250 format %{ "ldrw $dst, $mem\t# int" %}
7251
7252 ins_encode(aarch64_enc_ldrw(dst, mem));
7253
7254 ins_pipe(iload_reg_mem);
7255 %}
7256
7257 // Load Long (64 bit signed)
7258 instruct loadL(iRegLNoSp dst, memory8 mem)
7259 %{
7260 match(Set dst (LoadL mem));
7261 predicate(!needs_acquiring_load(n));
7262
7263 ins_cost(4 * INSN_COST);
7264 format %{ "ldr $dst, $mem\t# int" %}
7265
7266 ins_encode(aarch64_enc_ldr(dst, mem));
7267
7268 ins_pipe(iload_reg_mem);
7269 %}
7270
7271 // Load Range
7272 instruct loadRange(iRegINoSp dst, memory4 mem)
7273 %{
7274 match(Set dst (LoadRange mem));
7275
7276 ins_cost(4 * INSN_COST);
7277 format %{ "ldrw $dst, $mem\t# range" %}
7278
7279 ins_encode(aarch64_enc_ldrw(dst, mem));
7280
7281 ins_pipe(iload_reg_mem);
7282 %}
7283
7284 // Load Pointer
7285 instruct loadP(iRegPNoSp dst, memory8 mem)
7286 %{
7287 match(Set dst (LoadP mem));
7288 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
7289
7290 ins_cost(4 * INSN_COST);
7291 format %{ "ldr $dst, $mem\t# ptr" %}
7292
7293 ins_encode(aarch64_enc_ldr(dst, mem));
7294
7295 ins_pipe(iload_reg_mem);
7296 %}
7297
7298 // Load Compressed Pointer
7299 instruct loadN(iRegNNoSp dst, memory4 mem)
7300 %{
7301 match(Set dst (LoadN mem));
7302 predicate(!needs_acquiring_load(n));
7303
7304 ins_cost(4 * INSN_COST);
7305 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
7306
7307 ins_encode(aarch64_enc_ldrw(dst, mem));
7308
7309 ins_pipe(iload_reg_mem);
7310 %}
7311
7312 // Load Klass Pointer
7313 instruct loadKlass(iRegPNoSp dst, memory8 mem)
7314 %{
7315 match(Set dst (LoadKlass mem));
7316 predicate(!needs_acquiring_load(n));
7317
7318 ins_cost(4 * INSN_COST);
7319 format %{ "ldr $dst, $mem\t# class" %}
7320
7321 ins_encode(aarch64_enc_ldr(dst, mem));
7322
7323 ins_pipe(iload_reg_mem);
7324 %}
7325
7326 // Load Narrow Klass Pointer
7327 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
7328 %{
7329 match(Set dst (LoadNKlass mem));
7330 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
7331
7332 ins_cost(4 * INSN_COST);
7333 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7334
7335 ins_encode(aarch64_enc_ldrw(dst, mem));
7336
7337 ins_pipe(iload_reg_mem);
7338 %}
7339
7340 instruct loadNKlassLilliput(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
7341 %{
7342 match(Set dst (LoadNKlass mem));
7343 effect(KILL cr);
7344 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
7345
7346 ins_cost(4 * INSN_COST);
7347 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7348 ins_encode %{
7349 assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset");
7350 assert($mem$$index$$Register == noreg, "expect no index");
7351 Register dst = $dst$$Register;
7352 Register obj = $mem$$base$$Register;
7353 C2LoadNKlassStub* stub = new (Compile::current()->comp_arena()) C2LoadNKlassStub(dst);
7354 Compile::current()->output()->add_stub(stub);
7355 __ ldr(dst, Address(obj, oopDesc::mark_offset_in_bytes()));
7356 // NOTE: We can't use tbnz here, because the target is sometimes too far away
7357 // and cannot be encoded.
7358 __ tst(dst, markWord::monitor_value);
7359 __ br(Assembler::NE, stub->entry());
7360 __ bind(stub->continuation());
7361 __ lsr(dst, dst, markWord::klass_shift);
7362 %}
7363 ins_pipe(pipe_slow);
7364 %}
7365
7366 // Load Float
7367 instruct loadF(vRegF dst, memory4 mem)
7368 %{
7369 match(Set dst (LoadF mem));
7370 predicate(!needs_acquiring_load(n));
7371
7372 ins_cost(4 * INSN_COST);
7373 format %{ "ldrs $dst, $mem\t# float" %}
7374
7375 ins_encode( aarch64_enc_ldrs(dst, mem) );
7376
7377 ins_pipe(pipe_class_memory);
7378 %}
7379
7380 // Load Double
7381 instruct loadD(vRegD dst, memory8 mem)
7382 %{
7383 match(Set dst (LoadD mem));
7384 predicate(!needs_acquiring_load(n));
7385
7386 ins_cost(4 * INSN_COST);
7387 format %{ "ldrd $dst, $mem\t# double" %}
7388
7389 ins_encode( aarch64_enc_ldrd(dst, mem) );
7390
7391 ins_pipe(pipe_class_memory);
7392 %}
7393
7394
7395 // Load Int Constant
7396 instruct loadConI(iRegINoSp dst, immI src)
7397 %{
7398 match(Set dst src);
7399
7400 ins_cost(INSN_COST);
7401 format %{ "mov $dst, $src\t# int" %}
7402
7403 ins_encode( aarch64_enc_movw_imm(dst, src) );
7404
7405 ins_pipe(ialu_imm);
7406 %}
7407
7408 // Load Long Constant
7409 instruct loadConL(iRegLNoSp dst, immL src)
7410 %{
7411 match(Set dst src);
7412
7413 ins_cost(INSN_COST);
7414 format %{ "mov $dst, $src\t# long" %}
7415
7416 ins_encode( aarch64_enc_mov_imm(dst, src) );
7417
7418 ins_pipe(ialu_imm);
7419 %}
7420
7421 // Load Pointer Constant
7422
7423 instruct loadConP(iRegPNoSp dst, immP con)
7424 %{
7425 match(Set dst con);
7426
7427 ins_cost(INSN_COST * 4);
7428 format %{
7429 "mov $dst, $con\t# ptr\n\t"
7430 %}
7431
7432 ins_encode(aarch64_enc_mov_p(dst, con));
7433
7434 ins_pipe(ialu_imm);
7435 %}
7436
7437 // Load Null Pointer Constant
7438
7439 instruct loadConP0(iRegPNoSp dst, immP0 con)
7440 %{
7441 match(Set dst con);
7442
7443 ins_cost(INSN_COST);
7444 format %{ "mov $dst, $con\t# NULL ptr" %}
7445
7446 ins_encode(aarch64_enc_mov_p0(dst, con));
7447
7448 ins_pipe(ialu_imm);
7449 %}
7450
7451 // Load Pointer Constant One
7452
7453 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7454 %{
7455 match(Set dst con);
7456
7457 ins_cost(INSN_COST);
7458 format %{ "mov $dst, $con\t# NULL ptr" %}
7459
7460 ins_encode(aarch64_enc_mov_p1(dst, con));
7461
7462 ins_pipe(ialu_imm);
7463 %}
7464
7465 // Load Byte Map Base Constant
7466
7467 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7468 %{
7469 match(Set dst con);
7470
7471 ins_cost(INSN_COST);
7472 format %{ "adr $dst, $con\t# Byte Map Base" %}
7473
7474 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7475
7476 ins_pipe(ialu_imm);
7477 %}
7478
7479 // Load Narrow Pointer Constant
7480
7481 instruct loadConN(iRegNNoSp dst, immN con)
7482 %{
7483 match(Set dst con);
7484
7485 ins_cost(INSN_COST * 4);
7486 format %{ "mov $dst, $con\t# compressed ptr" %}
7487
7488 ins_encode(aarch64_enc_mov_n(dst, con));
7489
7490 ins_pipe(ialu_imm);
7491 %}
7492
7493 // Load Narrow Null Pointer Constant
7494
7495 instruct loadConN0(iRegNNoSp dst, immN0 con)
7496 %{
7497 match(Set dst con);
7498
7499 ins_cost(INSN_COST);
7500 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
7501
7502 ins_encode(aarch64_enc_mov_n0(dst, con));
7503
7504 ins_pipe(ialu_imm);
7505 %}
7506
7507 // Load Narrow Klass Constant
7508
7509 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7510 %{
7511 match(Set dst con);
7512
7513 ins_cost(INSN_COST);
7514 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7515
7516 ins_encode(aarch64_enc_mov_nk(dst, con));
7517
7518 ins_pipe(ialu_imm);
7519 %}
7520
7521 // Load Packed Float Constant
7522
7523 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7524 match(Set dst con);
7525 ins_cost(INSN_COST * 4);
7526 format %{ "fmovs $dst, $con"%}
7527 ins_encode %{
7528 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7529 %}
7530
7531 ins_pipe(fp_imm_s);
7532 %}
7533
7534 // Load Float Constant
7535
7536 instruct loadConF(vRegF dst, immF con) %{
7537 match(Set dst con);
7538
7539 ins_cost(INSN_COST * 4);
7540
7541 format %{
7542 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7543 %}
7544
7545 ins_encode %{
7546 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7547 %}
7548
7549 ins_pipe(fp_load_constant_s);
7550 %}
7551
7552 // Load Packed Double Constant
7553
7554 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7555 match(Set dst con);
7556 ins_cost(INSN_COST);
7557 format %{ "fmovd $dst, $con"%}
7558 ins_encode %{
7559 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7560 %}
7561
7562 ins_pipe(fp_imm_d);
7563 %}
7564
7565 // Load Double Constant
7566
7567 instruct loadConD(vRegD dst, immD con) %{
7568 match(Set dst con);
7569
7570 ins_cost(INSN_COST * 5);
7571 format %{
7572 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7573 %}
7574
7575 ins_encode %{
7576 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7577 %}
7578
7579 ins_pipe(fp_load_constant_d);
7580 %}
7581
7582 // Store Instructions
7583
7584 // Store CMS card-mark Immediate
7585 instruct storeimmCM0(immI0 zero, memory1 mem)
7586 %{
7587 match(Set mem (StoreCM mem zero));
7588
7589 ins_cost(INSN_COST);
7590 format %{ "storestore (elided)\n\t"
7591 "strb zr, $mem\t# byte" %}
7592
7593 ins_encode(aarch64_enc_strb0(mem));
7594
7595 ins_pipe(istore_mem);
7596 %}
7597
7598 // Store CMS card-mark Immediate with intervening StoreStore
7599 // needed when using CMS with no conditional card marking
7600 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
7601 %{
7602 match(Set mem (StoreCM mem zero));
7603
7604 ins_cost(INSN_COST * 2);
7605 format %{ "storestore\n\t"
7606 "dmb ishst"
7607 "\n\tstrb zr, $mem\t# byte" %}
7608
7609 ins_encode(aarch64_enc_strb0_ordered(mem));
7610
7611 ins_pipe(istore_mem);
7612 %}
7613
7614 // Store Byte
7615 instruct storeB(iRegIorL2I src, memory1 mem)
7616 %{
7617 match(Set mem (StoreB mem src));
7618 predicate(!needs_releasing_store(n));
7619
7620 ins_cost(INSN_COST);
7621 format %{ "strb $src, $mem\t# byte" %}
7622
7623 ins_encode(aarch64_enc_strb(src, mem));
7624
7625 ins_pipe(istore_reg_mem);
7626 %}
7627
7628
7629 instruct storeimmB0(immI0 zero, memory1 mem)
7630 %{
7631 match(Set mem (StoreB mem zero));
7632 predicate(!needs_releasing_store(n));
7633
7634 ins_cost(INSN_COST);
7635 format %{ "strb rscractch2, $mem\t# byte" %}
7636
7637 ins_encode(aarch64_enc_strb0(mem));
7638
7639 ins_pipe(istore_mem);
7640 %}
7641
7642 // Store Char/Short
7643 instruct storeC(iRegIorL2I src, memory2 mem)
7644 %{
7645 match(Set mem (StoreC mem src));
7646 predicate(!needs_releasing_store(n));
7647
7648 ins_cost(INSN_COST);
7649 format %{ "strh $src, $mem\t# short" %}
7650
7651 ins_encode(aarch64_enc_strh(src, mem));
7652
7653 ins_pipe(istore_reg_mem);
7654 %}
7655
7656 instruct storeimmC0(immI0 zero, memory2 mem)
7657 %{
7658 match(Set mem (StoreC mem zero));
7659 predicate(!needs_releasing_store(n));
7660
7661 ins_cost(INSN_COST);
7662 format %{ "strh zr, $mem\t# short" %}
7663
7664 ins_encode(aarch64_enc_strh0(mem));
7665
7666 ins_pipe(istore_mem);
7667 %}
7668
7669 // Store Integer
7670
7671 instruct storeI(iRegIorL2I src, memory4 mem)
7672 %{
7673 match(Set mem(StoreI mem src));
7674 predicate(!needs_releasing_store(n));
7675
7676 ins_cost(INSN_COST);
7677 format %{ "strw $src, $mem\t# int" %}
7678
7679 ins_encode(aarch64_enc_strw(src, mem));
7680
7681 ins_pipe(istore_reg_mem);
7682 %}
7683
7684 instruct storeimmI0(immI0 zero, memory4 mem)
7685 %{
7686 match(Set mem(StoreI mem zero));
7687 predicate(!needs_releasing_store(n));
7688
7689 ins_cost(INSN_COST);
7690 format %{ "strw zr, $mem\t# int" %}
7691
7692 ins_encode(aarch64_enc_strw0(mem));
7693
7694 ins_pipe(istore_mem);
7695 %}
7696
7697 // Store Long (64 bit signed)
7698 instruct storeL(iRegL src, memory8 mem)
7699 %{
7700 match(Set mem (StoreL mem src));
7701 predicate(!needs_releasing_store(n));
7702
7703 ins_cost(INSN_COST);
7704 format %{ "str $src, $mem\t# int" %}
7705
7706 ins_encode(aarch64_enc_str(src, mem));
7707
7708 ins_pipe(istore_reg_mem);
7709 %}
7710
7711 // Store Long (64 bit signed)
7712 instruct storeimmL0(immL0 zero, memory8 mem)
7713 %{
7714 match(Set mem (StoreL mem zero));
7715 predicate(!needs_releasing_store(n));
7716
7717 ins_cost(INSN_COST);
7718 format %{ "str zr, $mem\t# int" %}
7719
7720 ins_encode(aarch64_enc_str0(mem));
7721
7722 ins_pipe(istore_mem);
7723 %}
7724
7725 // Store Pointer
7726 instruct storeP(iRegP src, memory8 mem)
7727 %{
7728 match(Set mem (StoreP mem src));
7729 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7730
7731 ins_cost(INSN_COST);
7732 format %{ "str $src, $mem\t# ptr" %}
7733
7734 ins_encode(aarch64_enc_str(src, mem));
7735
7736 ins_pipe(istore_reg_mem);
7737 %}
7738
7739 // Store Pointer
7740 instruct storeimmP0(immP0 zero, memory8 mem)
7741 %{
7742 match(Set mem (StoreP mem zero));
7743 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7744
7745 ins_cost(INSN_COST);
7746 format %{ "str zr, $mem\t# ptr" %}
7747
7748 ins_encode(aarch64_enc_str0(mem));
7749
7750 ins_pipe(istore_mem);
7751 %}
7752
7753 // Store Compressed Pointer
7754 instruct storeN(iRegN src, memory4 mem)
7755 %{
7756 match(Set mem (StoreN mem src));
7757 predicate(!needs_releasing_store(n));
7758
7759 ins_cost(INSN_COST);
7760 format %{ "strw $src, $mem\t# compressed ptr" %}
7761
7762 ins_encode(aarch64_enc_strw(src, mem));
7763
7764 ins_pipe(istore_reg_mem);
7765 %}
7766
7767 instruct storeImmN0(immN0 zero, memory4 mem)
7768 %{
7769 match(Set mem (StoreN mem zero));
7770 predicate(!needs_releasing_store(n));
7771
7772 ins_cost(INSN_COST);
7773 format %{ "strw zr, $mem\t# compressed ptr" %}
7774
7775 ins_encode(aarch64_enc_strw0(mem));
7776
7777 ins_pipe(istore_mem);
7778 %}
7779
7780 // Store Float
7781 instruct storeF(vRegF src, memory4 mem)
7782 %{
7783 match(Set mem (StoreF mem src));
7784 predicate(!needs_releasing_store(n));
7785
7786 ins_cost(INSN_COST);
7787 format %{ "strs $src, $mem\t# float" %}
7788
7789 ins_encode( aarch64_enc_strs(src, mem) );
7790
7791 ins_pipe(pipe_class_memory);
7792 %}
7793
7794 // TODO
7795 // implement storeImmF0 and storeFImmPacked
7796
7797 // Store Double
7798 instruct storeD(vRegD src, memory8 mem)
7799 %{
7800 match(Set mem (StoreD mem src));
7801 predicate(!needs_releasing_store(n));
7802
7803 ins_cost(INSN_COST);
7804 format %{ "strd $src, $mem\t# double" %}
7805
7806 ins_encode( aarch64_enc_strd(src, mem) );
7807
7808 ins_pipe(pipe_class_memory);
7809 %}
7810
7811 // Store Compressed Klass Pointer
7812 instruct storeNKlass(iRegN src, memory4 mem)
7813 %{
7814 predicate(!needs_releasing_store(n));
7815 match(Set mem (StoreNKlass mem src));
7816
7817 ins_cost(INSN_COST);
7818 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7819
7820 ins_encode(aarch64_enc_strw(src, mem));
7821
7822 ins_pipe(istore_reg_mem);
7823 %}
7824
7825 // TODO
7826 // implement storeImmD0 and storeDImmPacked
7827
7828 // prefetch instructions
7829 // Must be safe to execute with invalid address (cannot fault).
7830
7831 instruct prefetchalloc( memory8 mem ) %{
7832 match(PrefetchAllocation mem);
7833
7834 ins_cost(INSN_COST);
7835 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7836
7837 ins_encode( aarch64_enc_prefetchw(mem) );
7838
7839 ins_pipe(iload_prefetch);
7840 %}
7841
7842 // ---------------- volatile loads and stores ----------------
7843
7844 // Load Byte (8 bit signed)
7845 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7846 %{
7847 match(Set dst (LoadB mem));
7848
7849 ins_cost(VOLATILE_REF_COST);
7850 format %{ "ldarsb $dst, $mem\t# byte" %}
7851
7852 ins_encode(aarch64_enc_ldarsb(dst, mem));
7853
7854 ins_pipe(pipe_serial);
7855 %}
7856
7857 // Load Byte (8 bit signed) into long
7858 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7859 %{
7860 match(Set dst (ConvI2L (LoadB mem)));
7861
7862 ins_cost(VOLATILE_REF_COST);
7863 format %{ "ldarsb $dst, $mem\t# byte" %}
7864
7865 ins_encode(aarch64_enc_ldarsb(dst, mem));
7866
7867 ins_pipe(pipe_serial);
7868 %}
7869
7870 // Load Byte (8 bit unsigned)
7871 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7872 %{
7873 match(Set dst (LoadUB mem));
7874
7875 ins_cost(VOLATILE_REF_COST);
7876 format %{ "ldarb $dst, $mem\t# byte" %}
7877
7878 ins_encode(aarch64_enc_ldarb(dst, mem));
7879
7880 ins_pipe(pipe_serial);
7881 %}
7882
7883 // Load Byte (8 bit unsigned) into long
7884 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7885 %{
7886 match(Set dst (ConvI2L (LoadUB mem)));
7887
7888 ins_cost(VOLATILE_REF_COST);
7889 format %{ "ldarb $dst, $mem\t# byte" %}
7890
7891 ins_encode(aarch64_enc_ldarb(dst, mem));
7892
7893 ins_pipe(pipe_serial);
7894 %}
7895
7896 // Load Short (16 bit signed)
7897 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7898 %{
7899 match(Set dst (LoadS mem));
7900
7901 ins_cost(VOLATILE_REF_COST);
7902 format %{ "ldarshw $dst, $mem\t# short" %}
7903
7904 ins_encode(aarch64_enc_ldarshw(dst, mem));
7905
7906 ins_pipe(pipe_serial);
7907 %}
7908
7909 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7910 %{
7911 match(Set dst (LoadUS mem));
7912
7913 ins_cost(VOLATILE_REF_COST);
7914 format %{ "ldarhw $dst, $mem\t# short" %}
7915
7916 ins_encode(aarch64_enc_ldarhw(dst, mem));
7917
7918 ins_pipe(pipe_serial);
7919 %}
7920
7921 // Load Short/Char (16 bit unsigned) into long
7922 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7923 %{
7924 match(Set dst (ConvI2L (LoadUS mem)));
7925
7926 ins_cost(VOLATILE_REF_COST);
7927 format %{ "ldarh $dst, $mem\t# short" %}
7928
7929 ins_encode(aarch64_enc_ldarh(dst, mem));
7930
7931 ins_pipe(pipe_serial);
7932 %}
7933
7934 // Load Short/Char (16 bit signed) into long
7935 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7936 %{
7937 match(Set dst (ConvI2L (LoadS mem)));
7938
7939 ins_cost(VOLATILE_REF_COST);
7940 format %{ "ldarh $dst, $mem\t# short" %}
7941
7942 ins_encode(aarch64_enc_ldarsh(dst, mem));
7943
7944 ins_pipe(pipe_serial);
7945 %}
7946
7947 // Load Integer (32 bit signed)
7948 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7949 %{
7950 match(Set dst (LoadI mem));
7951
7952 ins_cost(VOLATILE_REF_COST);
7953 format %{ "ldarw $dst, $mem\t# int" %}
7954
7955 ins_encode(aarch64_enc_ldarw(dst, mem));
7956
7957 ins_pipe(pipe_serial);
7958 %}
7959
7960 // Load Integer (32 bit unsigned) into long
7961 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7962 %{
7963 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7964
7965 ins_cost(VOLATILE_REF_COST);
7966 format %{ "ldarw $dst, $mem\t# int" %}
7967
7968 ins_encode(aarch64_enc_ldarw(dst, mem));
7969
7970 ins_pipe(pipe_serial);
7971 %}
7972
7973 // Load Long (64 bit signed)
7974 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7975 %{
7976 match(Set dst (LoadL mem));
7977
7978 ins_cost(VOLATILE_REF_COST);
7979 format %{ "ldar $dst, $mem\t# int" %}
7980
7981 ins_encode(aarch64_enc_ldar(dst, mem));
7982
7983 ins_pipe(pipe_serial);
7984 %}
7985
7986 // Load Pointer
7987 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7988 %{
7989 match(Set dst (LoadP mem));
7990 predicate(n->as_Load()->barrier_data() == 0);
7991
7992 ins_cost(VOLATILE_REF_COST);
7993 format %{ "ldar $dst, $mem\t# ptr" %}
7994
7995 ins_encode(aarch64_enc_ldar(dst, mem));
7996
7997 ins_pipe(pipe_serial);
7998 %}
7999
8000 // Load Compressed Pointer
8001 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
8002 %{
8003 match(Set dst (LoadN mem));
8004
8005 ins_cost(VOLATILE_REF_COST);
8006 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
8007
8008 ins_encode(aarch64_enc_ldarw(dst, mem));
8009
8010 ins_pipe(pipe_serial);
8011 %}
8012
8013 // Load Float
8014 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
8015 %{
8016 match(Set dst (LoadF mem));
8017
8018 ins_cost(VOLATILE_REF_COST);
8019 format %{ "ldars $dst, $mem\t# float" %}
8020
8021 ins_encode( aarch64_enc_fldars(dst, mem) );
8022
8023 ins_pipe(pipe_serial);
8024 %}
8025
8026 // Load Double
8027 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
8028 %{
8029 match(Set dst (LoadD mem));
8030
8031 ins_cost(VOLATILE_REF_COST);
8032 format %{ "ldard $dst, $mem\t# double" %}
8033
8034 ins_encode( aarch64_enc_fldard(dst, mem) );
8035
8036 ins_pipe(pipe_serial);
8037 %}
8038
8039 // Store Byte
8040 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8041 %{
8042 match(Set mem (StoreB mem src));
8043
8044 ins_cost(VOLATILE_REF_COST);
8045 format %{ "stlrb $src, $mem\t# byte" %}
8046
8047 ins_encode(aarch64_enc_stlrb(src, mem));
8048
8049 ins_pipe(pipe_class_memory);
8050 %}
8051
8052 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8053 %{
8054 match(Set mem (StoreB mem zero));
8055
8056 ins_cost(VOLATILE_REF_COST);
8057 format %{ "stlrb zr, $mem\t# byte" %}
8058
8059 ins_encode(aarch64_enc_stlrb0(mem));
8060
8061 ins_pipe(pipe_class_memory);
8062 %}
8063
8064 // Store Char/Short
8065 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8066 %{
8067 match(Set mem (StoreC mem src));
8068
8069 ins_cost(VOLATILE_REF_COST);
8070 format %{ "stlrh $src, $mem\t# short" %}
8071
8072 ins_encode(aarch64_enc_stlrh(src, mem));
8073
8074 ins_pipe(pipe_class_memory);
8075 %}
8076
8077 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8078 %{
8079 match(Set mem (StoreC mem zero));
8080
8081 ins_cost(VOLATILE_REF_COST);
8082 format %{ "stlrh zr, $mem\t# short" %}
8083
8084 ins_encode(aarch64_enc_stlrh0(mem));
8085
8086 ins_pipe(pipe_class_memory);
8087 %}
8088
8089 // Store Integer
8090
8091 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8092 %{
8093 match(Set mem(StoreI mem src));
8094
8095 ins_cost(VOLATILE_REF_COST);
8096 format %{ "stlrw $src, $mem\t# int" %}
8097
8098 ins_encode(aarch64_enc_stlrw(src, mem));
8099
8100 ins_pipe(pipe_class_memory);
8101 %}
8102
8103 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8104 %{
8105 match(Set mem(StoreI mem zero));
8106
8107 ins_cost(VOLATILE_REF_COST);
8108 format %{ "stlrw zr, $mem\t# int" %}
8109
8110 ins_encode(aarch64_enc_stlrw0(mem));
8111
8112 ins_pipe(pipe_class_memory);
8113 %}
8114
8115 // Store Long (64 bit signed)
8116 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
8117 %{
8118 match(Set mem (StoreL mem src));
8119
8120 ins_cost(VOLATILE_REF_COST);
8121 format %{ "stlr $src, $mem\t# int" %}
8122
8123 ins_encode(aarch64_enc_stlr(src, mem));
8124
8125 ins_pipe(pipe_class_memory);
8126 %}
8127
8128 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
8129 %{
8130 match(Set mem (StoreL mem zero));
8131
8132 ins_cost(VOLATILE_REF_COST);
8133 format %{ "stlr zr, $mem\t# int" %}
8134
8135 ins_encode(aarch64_enc_stlr0(mem));
8136
8137 ins_pipe(pipe_class_memory);
8138 %}
8139
8140 // Store Pointer
8141 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
8142 %{
8143 match(Set mem (StoreP mem src));
8144 predicate(n->as_Store()->barrier_data() == 0);
8145
8146 ins_cost(VOLATILE_REF_COST);
8147 format %{ "stlr $src, $mem\t# ptr" %}
8148
8149 ins_encode(aarch64_enc_stlr(src, mem));
8150
8151 ins_pipe(pipe_class_memory);
8152 %}
8153
8154 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
8155 %{
8156 match(Set mem (StoreP mem zero));
8157 predicate(n->as_Store()->barrier_data() == 0);
8158
8159 ins_cost(VOLATILE_REF_COST);
8160 format %{ "stlr zr, $mem\t# ptr" %}
8161
8162 ins_encode(aarch64_enc_stlr0(mem));
8163
8164 ins_pipe(pipe_class_memory);
8165 %}
8166
8167 // Store Compressed Pointer
8168 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
8169 %{
8170 match(Set mem (StoreN mem src));
8171
8172 ins_cost(VOLATILE_REF_COST);
8173 format %{ "stlrw $src, $mem\t# compressed ptr" %}
8174
8175 ins_encode(aarch64_enc_stlrw(src, mem));
8176
8177 ins_pipe(pipe_class_memory);
8178 %}
8179
8180 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
8181 %{
8182 match(Set mem (StoreN mem zero));
8183
8184 ins_cost(VOLATILE_REF_COST);
8185 format %{ "stlrw zr, $mem\t# compressed ptr" %}
8186
8187 ins_encode(aarch64_enc_stlrw0(mem));
8188
8189 ins_pipe(pipe_class_memory);
8190 %}
8191
8192 // Store Float
8193 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
8194 %{
8195 match(Set mem (StoreF mem src));
8196
8197 ins_cost(VOLATILE_REF_COST);
8198 format %{ "stlrs $src, $mem\t# float" %}
8199
8200 ins_encode( aarch64_enc_fstlrs(src, mem) );
8201
8202 ins_pipe(pipe_class_memory);
8203 %}
8204
8205 // TODO
8206 // implement storeImmF0 and storeFImmPacked
8207
8208 // Store Double
8209 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
8210 %{
8211 match(Set mem (StoreD mem src));
8212
8213 ins_cost(VOLATILE_REF_COST);
8214 format %{ "stlrd $src, $mem\t# double" %}
8215
8216 ins_encode( aarch64_enc_fstlrd(src, mem) );
8217
8218 ins_pipe(pipe_class_memory);
8219 %}
8220
8221 // ---------------- end of volatile loads and stores ----------------
8222
8223 instruct cacheWB(indirect addr)
8224 %{
8225 predicate(VM_Version::supports_data_cache_line_flush());
8226 match(CacheWB addr);
8227
8228 ins_cost(100);
8229 format %{"cache wb $addr" %}
8230 ins_encode %{
8231 assert($addr->index_position() < 0, "should be");
8232 assert($addr$$disp == 0, "should be");
8233 __ cache_wb(Address($addr$$base$$Register, 0));
8234 %}
8235 ins_pipe(pipe_slow); // XXX
8236 %}
8237
8238 instruct cacheWBPreSync()
8239 %{
8240 predicate(VM_Version::supports_data_cache_line_flush());
8241 match(CacheWBPreSync);
8242
8243 ins_cost(100);
8244 format %{"cache wb presync" %}
8245 ins_encode %{
8246 __ cache_wbsync(true);
8247 %}
8248 ins_pipe(pipe_slow); // XXX
8249 %}
8250
8251 instruct cacheWBPostSync()
8252 %{
8253 predicate(VM_Version::supports_data_cache_line_flush());
8254 match(CacheWBPostSync);
8255
8256 ins_cost(100);
8257 format %{"cache wb postsync" %}
8258 ins_encode %{
8259 __ cache_wbsync(false);
8260 %}
8261 ins_pipe(pipe_slow); // XXX
8262 %}
8263
8264 // ============================================================================
8265 // BSWAP Instructions
8266
8267 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
8268 match(Set dst (ReverseBytesI src));
8269
8270 ins_cost(INSN_COST);
8271 format %{ "revw $dst, $src" %}
8272
8273 ins_encode %{
8274 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
8275 %}
8276
8277 ins_pipe(ialu_reg);
8278 %}
8279
8280 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
8281 match(Set dst (ReverseBytesL src));
8282
8283 ins_cost(INSN_COST);
8284 format %{ "rev $dst, $src" %}
8285
8286 ins_encode %{
8287 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
8288 %}
8289
8290 ins_pipe(ialu_reg);
8291 %}
8292
8293 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
8294 match(Set dst (ReverseBytesUS src));
8295
8296 ins_cost(INSN_COST);
8297 format %{ "rev16w $dst, $src" %}
8298
8299 ins_encode %{
8300 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8301 %}
8302
8303 ins_pipe(ialu_reg);
8304 %}
8305
8306 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
8307 match(Set dst (ReverseBytesS src));
8308
8309 ins_cost(INSN_COST);
8310 format %{ "rev16w $dst, $src\n\t"
8311 "sbfmw $dst, $dst, #0, #15" %}
8312
8313 ins_encode %{
8314 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8315 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
8316 %}
8317
8318 ins_pipe(ialu_reg);
8319 %}
8320
8321 // ============================================================================
8322 // Zero Count Instructions
8323
8324 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8325 match(Set dst (CountLeadingZerosI src));
8326
8327 ins_cost(INSN_COST);
8328 format %{ "clzw $dst, $src" %}
8329 ins_encode %{
8330 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
8331 %}
8332
8333 ins_pipe(ialu_reg);
8334 %}
8335
8336 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
8337 match(Set dst (CountLeadingZerosL src));
8338
8339 ins_cost(INSN_COST);
8340 format %{ "clz $dst, $src" %}
8341 ins_encode %{
8342 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
8343 %}
8344
8345 ins_pipe(ialu_reg);
8346 %}
8347
8348 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8349 match(Set dst (CountTrailingZerosI src));
8350
8351 ins_cost(INSN_COST * 2);
8352 format %{ "rbitw $dst, $src\n\t"
8353 "clzw $dst, $dst" %}
8354 ins_encode %{
8355 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
8356 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
8357 %}
8358
8359 ins_pipe(ialu_reg);
8360 %}
8361
8362 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
8363 match(Set dst (CountTrailingZerosL src));
8364
8365 ins_cost(INSN_COST * 2);
8366 format %{ "rbit $dst, $src\n\t"
8367 "clz $dst, $dst" %}
8368 ins_encode %{
8369 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
8370 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
8371 %}
8372
8373 ins_pipe(ialu_reg);
8374 %}
8375
8376 //---------- Population Count Instructions -------------------------------------
8377 //
8378
8379 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
8380 match(Set dst (PopCountI src));
8381 effect(TEMP tmp);
8382 ins_cost(INSN_COST * 13);
8383
8384 format %{ "movw $src, $src\n\t"
8385 "mov $tmp, $src\t# vector (1D)\n\t"
8386 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8387 "addv $tmp, $tmp\t# vector (8B)\n\t"
8388 "mov $dst, $tmp\t# vector (1D)" %}
8389 ins_encode %{
8390 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
8391 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8392 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8393 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8394 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8395 %}
8396
8397 ins_pipe(pipe_class_default);
8398 %}
8399
8400 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
8401 match(Set dst (PopCountI (LoadI mem)));
8402 effect(TEMP tmp);
8403 ins_cost(INSN_COST * 13);
8404
8405 format %{ "ldrs $tmp, $mem\n\t"
8406 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8407 "addv $tmp, $tmp\t# vector (8B)\n\t"
8408 "mov $dst, $tmp\t# vector (1D)" %}
8409 ins_encode %{
8410 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8411 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
8412 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
8413 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8414 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8415 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8416 %}
8417
8418 ins_pipe(pipe_class_default);
8419 %}
8420
8421 // Note: Long.bitCount(long) returns an int.
8422 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
8423 match(Set dst (PopCountL src));
8424 effect(TEMP tmp);
8425 ins_cost(INSN_COST * 13);
8426
8427 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
8428 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8429 "addv $tmp, $tmp\t# vector (8B)\n\t"
8430 "mov $dst, $tmp\t# vector (1D)" %}
8431 ins_encode %{
8432 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8433 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8434 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8435 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8436 %}
8437
8438 ins_pipe(pipe_class_default);
8439 %}
8440
8441 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
8442 match(Set dst (PopCountL (LoadL mem)));
8443 effect(TEMP tmp);
8444 ins_cost(INSN_COST * 13);
8445
8446 format %{ "ldrd $tmp, $mem\n\t"
8447 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8448 "addv $tmp, $tmp\t# vector (8B)\n\t"
8449 "mov $dst, $tmp\t# vector (1D)" %}
8450 ins_encode %{
8451 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8452 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8453 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
8454 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8455 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8456 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8457 %}
8458
8459 ins_pipe(pipe_class_default);
8460 %}
8461
8462 // ============================================================================
8463 // MemBar Instruction
8464
8465 instruct load_fence() %{
8466 match(LoadFence);
8467 ins_cost(VOLATILE_REF_COST);
8468
8469 format %{ "load_fence" %}
8470
8471 ins_encode %{
8472 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8473 %}
8474 ins_pipe(pipe_serial);
8475 %}
8476
8477 instruct unnecessary_membar_acquire() %{
8478 predicate(unnecessary_acquire(n));
8479 match(MemBarAcquire);
8480 ins_cost(0);
8481
8482 format %{ "membar_acquire (elided)" %}
8483
8484 ins_encode %{
8485 __ block_comment("membar_acquire (elided)");
8486 %}
8487
8488 ins_pipe(pipe_class_empty);
8489 %}
8490
8491 instruct membar_acquire() %{
8492 match(MemBarAcquire);
8493 ins_cost(VOLATILE_REF_COST);
8494
8495 format %{ "membar_acquire\n\t"
8496 "dmb ish" %}
8497
8498 ins_encode %{
8499 __ block_comment("membar_acquire");
8500 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8501 %}
8502
8503 ins_pipe(pipe_serial);
8504 %}
8505
8506
8507 instruct membar_acquire_lock() %{
8508 match(MemBarAcquireLock);
8509 ins_cost(VOLATILE_REF_COST);
8510
8511 format %{ "membar_acquire_lock (elided)" %}
8512
8513 ins_encode %{
8514 __ block_comment("membar_acquire_lock (elided)");
8515 %}
8516
8517 ins_pipe(pipe_serial);
8518 %}
8519
8520 instruct store_fence() %{
8521 match(StoreFence);
8522 ins_cost(VOLATILE_REF_COST);
8523
8524 format %{ "store_fence" %}
8525
8526 ins_encode %{
8527 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8528 %}
8529 ins_pipe(pipe_serial);
8530 %}
8531
8532 instruct unnecessary_membar_release() %{
8533 predicate(unnecessary_release(n));
8534 match(MemBarRelease);
8535 ins_cost(0);
8536
8537 format %{ "membar_release (elided)" %}
8538
8539 ins_encode %{
8540 __ block_comment("membar_release (elided)");
8541 %}
8542 ins_pipe(pipe_serial);
8543 %}
8544
8545 instruct membar_release() %{
8546 match(MemBarRelease);
8547 ins_cost(VOLATILE_REF_COST);
8548
8549 format %{ "membar_release\n\t"
8550 "dmb ish" %}
8551
8552 ins_encode %{
8553 __ block_comment("membar_release");
8554 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8555 %}
8556 ins_pipe(pipe_serial);
8557 %}
8558
8559 instruct membar_storestore() %{
8560 match(MemBarStoreStore);
8561 match(StoreStoreFence);
8562 ins_cost(VOLATILE_REF_COST);
8563
8564 format %{ "MEMBAR-store-store" %}
8565
8566 ins_encode %{
8567 __ membar(Assembler::StoreStore);
8568 %}
8569 ins_pipe(pipe_serial);
8570 %}
8571
8572 instruct membar_release_lock() %{
8573 match(MemBarReleaseLock);
8574 ins_cost(VOLATILE_REF_COST);
8575
8576 format %{ "membar_release_lock (elided)" %}
8577
8578 ins_encode %{
8579 __ block_comment("membar_release_lock (elided)");
8580 %}
8581
8582 ins_pipe(pipe_serial);
8583 %}
8584
8585 instruct unnecessary_membar_volatile() %{
8586 predicate(unnecessary_volatile(n));
8587 match(MemBarVolatile);
8588 ins_cost(0);
8589
8590 format %{ "membar_volatile (elided)" %}
8591
8592 ins_encode %{
8593 __ block_comment("membar_volatile (elided)");
8594 %}
8595
8596 ins_pipe(pipe_serial);
8597 %}
8598
8599 instruct membar_volatile() %{
8600 match(MemBarVolatile);
8601 ins_cost(VOLATILE_REF_COST*100);
8602
8603 format %{ "membar_volatile\n\t"
8604 "dmb ish"%}
8605
8606 ins_encode %{
8607 __ block_comment("membar_volatile");
8608 __ membar(Assembler::StoreLoad);
8609 %}
8610
8611 ins_pipe(pipe_serial);
8612 %}
8613
8614 // ============================================================================
8615 // Cast/Convert Instructions
8616
8617 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8618 match(Set dst (CastX2P src));
8619
8620 ins_cost(INSN_COST);
8621 format %{ "mov $dst, $src\t# long -> ptr" %}
8622
8623 ins_encode %{
8624 if ($dst$$reg != $src$$reg) {
8625 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8626 }
8627 %}
8628
8629 ins_pipe(ialu_reg);
8630 %}
8631
8632 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8633 match(Set dst (CastP2X src));
8634
8635 ins_cost(INSN_COST);
8636 format %{ "mov $dst, $src\t# ptr -> long" %}
8637
8638 ins_encode %{
8639 if ($dst$$reg != $src$$reg) {
8640 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8641 }
8642 %}
8643
8644 ins_pipe(ialu_reg);
8645 %}
8646
8647 // Convert oop into int for vectors alignment masking
8648 instruct convP2I(iRegINoSp dst, iRegP src) %{
8649 match(Set dst (ConvL2I (CastP2X src)));
8650
8651 ins_cost(INSN_COST);
8652 format %{ "movw $dst, $src\t# ptr -> int" %}
8653 ins_encode %{
8654 __ movw($dst$$Register, $src$$Register);
8655 %}
8656
8657 ins_pipe(ialu_reg);
8658 %}
8659
8660 // Convert compressed oop into int for vectors alignment masking
8661 // in case of 32bit oops (heap < 4Gb).
8662 instruct convN2I(iRegINoSp dst, iRegN src)
8663 %{
8664 predicate(CompressedOops::shift() == 0);
8665 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8666
8667 ins_cost(INSN_COST);
8668 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8669 ins_encode %{
8670 __ movw($dst$$Register, $src$$Register);
8671 %}
8672
8673 ins_pipe(ialu_reg);
8674 %}
8675
8676
8677 // Convert oop pointer into compressed form
8678 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8679 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8680 match(Set dst (EncodeP src));
8681 effect(KILL cr);
8682 ins_cost(INSN_COST * 3);
8683 format %{ "encode_heap_oop $dst, $src" %}
8684 ins_encode %{
8685 Register s = $src$$Register;
8686 Register d = $dst$$Register;
8687 __ encode_heap_oop(d, s);
8688 %}
8689 ins_pipe(ialu_reg);
8690 %}
8691
8692 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8693 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8694 match(Set dst (EncodeP src));
8695 ins_cost(INSN_COST * 3);
8696 format %{ "encode_heap_oop_not_null $dst, $src" %}
8697 ins_encode %{
8698 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8699 %}
8700 ins_pipe(ialu_reg);
8701 %}
8702
8703 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8704 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8705 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8706 match(Set dst (DecodeN src));
8707 ins_cost(INSN_COST * 3);
8708 format %{ "decode_heap_oop $dst, $src" %}
8709 ins_encode %{
8710 Register s = $src$$Register;
8711 Register d = $dst$$Register;
8712 __ decode_heap_oop(d, s);
8713 %}
8714 ins_pipe(ialu_reg);
8715 %}
8716
8717 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8718 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8719 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8720 match(Set dst (DecodeN src));
8721 ins_cost(INSN_COST * 3);
8722 format %{ "decode_heap_oop_not_null $dst, $src" %}
8723 ins_encode %{
8724 Register s = $src$$Register;
8725 Register d = $dst$$Register;
8726 __ decode_heap_oop_not_null(d, s);
8727 %}
8728 ins_pipe(ialu_reg);
8729 %}
8730
8731 // n.b. AArch64 implementations of encode_klass_not_null and
8732 // decode_klass_not_null do not modify the flags register so, unlike
8733 // Intel, we don't kill CR as a side effect here
8734
8735 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8736 match(Set dst (EncodePKlass src));
8737
8738 ins_cost(INSN_COST * 3);
8739 format %{ "encode_klass_not_null $dst,$src" %}
8740
8741 ins_encode %{
8742 Register src_reg = as_Register($src$$reg);
8743 Register dst_reg = as_Register($dst$$reg);
8744 __ encode_klass_not_null(dst_reg, src_reg);
8745 %}
8746
8747 ins_pipe(ialu_reg);
8748 %}
8749
8750 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8751 match(Set dst (DecodeNKlass src));
8752
8753 ins_cost(INSN_COST * 3);
8754 format %{ "decode_klass_not_null $dst,$src" %}
8755
8756 ins_encode %{
8757 Register src_reg = as_Register($src$$reg);
8758 Register dst_reg = as_Register($dst$$reg);
8759 if (dst_reg != src_reg) {
8760 __ decode_klass_not_null(dst_reg, src_reg);
8761 } else {
8762 __ decode_klass_not_null(dst_reg);
8763 }
8764 %}
8765
8766 ins_pipe(ialu_reg);
8767 %}
8768
8769 instruct checkCastPP(iRegPNoSp dst)
8770 %{
8771 match(Set dst (CheckCastPP dst));
8772
8773 size(0);
8774 format %{ "# checkcastPP of $dst" %}
8775 ins_encode(/* empty encoding */);
8776 ins_pipe(pipe_class_empty);
8777 %}
8778
8779 instruct castPP(iRegPNoSp dst)
8780 %{
8781 match(Set dst (CastPP dst));
8782
8783 size(0);
8784 format %{ "# castPP of $dst" %}
8785 ins_encode(/* empty encoding */);
8786 ins_pipe(pipe_class_empty);
8787 %}
8788
8789 instruct castII(iRegI dst)
8790 %{
8791 match(Set dst (CastII dst));
8792
8793 size(0);
8794 format %{ "# castII of $dst" %}
8795 ins_encode(/* empty encoding */);
8796 ins_cost(0);
8797 ins_pipe(pipe_class_empty);
8798 %}
8799
8800 instruct castLL(iRegL dst)
8801 %{
8802 match(Set dst (CastLL dst));
8803
8804 size(0);
8805 format %{ "# castLL of $dst" %}
8806 ins_encode(/* empty encoding */);
8807 ins_cost(0);
8808 ins_pipe(pipe_class_empty);
8809 %}
8810
8811 instruct castFF(vRegF dst)
8812 %{
8813 match(Set dst (CastFF dst));
8814
8815 size(0);
8816 format %{ "# castFF of $dst" %}
8817 ins_encode(/* empty encoding */);
8818 ins_cost(0);
8819 ins_pipe(pipe_class_empty);
8820 %}
8821
8822 instruct castDD(vRegD dst)
8823 %{
8824 match(Set dst (CastDD dst));
8825
8826 size(0);
8827 format %{ "# castDD of $dst" %}
8828 ins_encode(/* empty encoding */);
8829 ins_cost(0);
8830 ins_pipe(pipe_class_empty);
8831 %}
8832
8833 instruct castVV(vReg dst)
8834 %{
8835 match(Set dst (CastVV dst));
8836
8837 size(0);
8838 format %{ "# castVV of $dst" %}
8839 ins_encode(/* empty encoding */);
8840 ins_cost(0);
8841 ins_pipe(pipe_class_empty);
8842 %}
8843
8844 instruct castVVMask(pRegGov dst)
8845 %{
8846 match(Set dst (CastVV dst));
8847
8848 size(0);
8849 format %{ "# castVV of $dst" %}
8850 ins_encode(/* empty encoding */);
8851 ins_cost(0);
8852 ins_pipe(pipe_class_empty);
8853 %}
8854
8855 // ============================================================================
8856 // Atomic operation instructions
8857 //
8858
8859 // standard CompareAndSwapX when we are using barriers
8860 // these have higher priority than the rules selected by a predicate
8861
8862 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8863 // can't match them
8864
8865 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8866
8867 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8868 ins_cost(2 * VOLATILE_REF_COST);
8869
8870 effect(KILL cr);
8871
8872 format %{
8873 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8874 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8875 %}
8876
8877 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8878 aarch64_enc_cset_eq(res));
8879
8880 ins_pipe(pipe_slow);
8881 %}
8882
8883 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8884
8885 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8886 ins_cost(2 * VOLATILE_REF_COST);
8887
8888 effect(KILL cr);
8889
8890 format %{
8891 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8892 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8893 %}
8894
8895 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8896 aarch64_enc_cset_eq(res));
8897
8898 ins_pipe(pipe_slow);
8899 %}
8900
8901 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8902
8903 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8904 ins_cost(2 * VOLATILE_REF_COST);
8905
8906 effect(KILL cr);
8907
8908 format %{
8909 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8910 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8911 %}
8912
8913 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8914 aarch64_enc_cset_eq(res));
8915
8916 ins_pipe(pipe_slow);
8917 %}
8918
8919 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8920
8921 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8922 ins_cost(2 * VOLATILE_REF_COST);
8923
8924 effect(KILL cr);
8925
8926 format %{
8927 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8928 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8929 %}
8930
8931 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8932 aarch64_enc_cset_eq(res));
8933
8934 ins_pipe(pipe_slow);
8935 %}
8936
8937 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8938
8939 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8940 predicate(n->as_LoadStore()->barrier_data() == 0);
8941 ins_cost(2 * VOLATILE_REF_COST);
8942
8943 effect(KILL cr);
8944
8945 format %{
8946 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8947 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8948 %}
8949
8950 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8951 aarch64_enc_cset_eq(res));
8952
8953 ins_pipe(pipe_slow);
8954 %}
8955
8956 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8957
8958 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8959 ins_cost(2 * VOLATILE_REF_COST);
8960
8961 effect(KILL cr);
8962
8963 format %{
8964 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8965 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8966 %}
8967
8968 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8969 aarch64_enc_cset_eq(res));
8970
8971 ins_pipe(pipe_slow);
8972 %}
8973
8974 // alternative CompareAndSwapX when we are eliding barriers
8975
8976 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8977
8978 predicate(needs_acquiring_load_exclusive(n));
8979 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8980 ins_cost(VOLATILE_REF_COST);
8981
8982 effect(KILL cr);
8983
8984 format %{
8985 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8986 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8987 %}
8988
8989 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8990 aarch64_enc_cset_eq(res));
8991
8992 ins_pipe(pipe_slow);
8993 %}
8994
8995 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8996
8997 predicate(needs_acquiring_load_exclusive(n));
8998 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8999 ins_cost(VOLATILE_REF_COST);
9000
9001 effect(KILL cr);
9002
9003 format %{
9004 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9005 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9006 %}
9007
9008 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
9009 aarch64_enc_cset_eq(res));
9010
9011 ins_pipe(pipe_slow);
9012 %}
9013
9014 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9015
9016 predicate(needs_acquiring_load_exclusive(n));
9017 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9018 ins_cost(VOLATILE_REF_COST);
9019
9020 effect(KILL cr);
9021
9022 format %{
9023 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9024 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9025 %}
9026
9027 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9028 aarch64_enc_cset_eq(res));
9029
9030 ins_pipe(pipe_slow);
9031 %}
9032
9033 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9034
9035 predicate(needs_acquiring_load_exclusive(n));
9036 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9037 ins_cost(VOLATILE_REF_COST);
9038
9039 effect(KILL cr);
9040
9041 format %{
9042 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9043 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9044 %}
9045
9046 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9047 aarch64_enc_cset_eq(res));
9048
9049 ins_pipe(pipe_slow);
9050 %}
9051
9052 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9053
9054 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9055 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9056 ins_cost(VOLATILE_REF_COST);
9057
9058 effect(KILL cr);
9059
9060 format %{
9061 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9062 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9063 %}
9064
9065 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9066 aarch64_enc_cset_eq(res));
9067
9068 ins_pipe(pipe_slow);
9069 %}
9070
9071 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9072
9073 predicate(needs_acquiring_load_exclusive(n));
9074 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9075 ins_cost(VOLATILE_REF_COST);
9076
9077 effect(KILL cr);
9078
9079 format %{
9080 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9081 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9082 %}
9083
9084 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9085 aarch64_enc_cset_eq(res));
9086
9087 ins_pipe(pipe_slow);
9088 %}
9089
9090
9091 // ---------------------------------------------------------------------
9092
9093 // BEGIN This section of the file is automatically generated. Do not edit --------------
9094
9095 // Sundry CAS operations. Note that release is always true,
9096 // regardless of the memory ordering of the CAS. This is because we
9097 // need the volatile case to be sequentially consistent but there is
9098 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
9099 // can't check the type of memory ordering here, so we always emit a
9100 // STLXR.
9101
9102 // This section is generated from cas.m4
9103
9104
9105 // This pattern is generated automatically from cas.m4.
9106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9107 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9108 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9109 ins_cost(2 * VOLATILE_REF_COST);
9110 effect(TEMP_DEF res, KILL cr);
9111 format %{
9112 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9113 %}
9114 ins_encode %{
9115 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9116 Assembler::byte, /*acquire*/ false, /*release*/ true,
9117 /*weak*/ false, $res$$Register);
9118 __ sxtbw($res$$Register, $res$$Register);
9119 %}
9120 ins_pipe(pipe_slow);
9121 %}
9122
9123 // This pattern is generated automatically from cas.m4.
9124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9125 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9126 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9127 ins_cost(2 * VOLATILE_REF_COST);
9128 effect(TEMP_DEF res, KILL cr);
9129 format %{
9130 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9131 %}
9132 ins_encode %{
9133 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9134 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9135 /*weak*/ false, $res$$Register);
9136 __ sxthw($res$$Register, $res$$Register);
9137 %}
9138 ins_pipe(pipe_slow);
9139 %}
9140
9141 // This pattern is generated automatically from cas.m4.
9142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9143 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9144 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9145 ins_cost(2 * VOLATILE_REF_COST);
9146 effect(TEMP_DEF res, KILL cr);
9147 format %{
9148 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9149 %}
9150 ins_encode %{
9151 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9152 Assembler::word, /*acquire*/ false, /*release*/ true,
9153 /*weak*/ false, $res$$Register);
9154 %}
9155 ins_pipe(pipe_slow);
9156 %}
9157
9158 // This pattern is generated automatically from cas.m4.
9159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9160 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9161 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9162 ins_cost(2 * VOLATILE_REF_COST);
9163 effect(TEMP_DEF res, KILL cr);
9164 format %{
9165 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9166 %}
9167 ins_encode %{
9168 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9169 Assembler::xword, /*acquire*/ false, /*release*/ true,
9170 /*weak*/ false, $res$$Register);
9171 %}
9172 ins_pipe(pipe_slow);
9173 %}
9174
9175 // This pattern is generated automatically from cas.m4.
9176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9177 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9178 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9179 ins_cost(2 * VOLATILE_REF_COST);
9180 effect(TEMP_DEF res, KILL cr);
9181 format %{
9182 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9183 %}
9184 ins_encode %{
9185 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9186 Assembler::word, /*acquire*/ false, /*release*/ true,
9187 /*weak*/ false, $res$$Register);
9188 %}
9189 ins_pipe(pipe_slow);
9190 %}
9191
9192 // This pattern is generated automatically from cas.m4.
9193 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9194 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9195 predicate(n->as_LoadStore()->barrier_data() == 0);
9196 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9197 ins_cost(2 * VOLATILE_REF_COST);
9198 effect(TEMP_DEF res, KILL cr);
9199 format %{
9200 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9201 %}
9202 ins_encode %{
9203 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9204 Assembler::xword, /*acquire*/ false, /*release*/ true,
9205 /*weak*/ false, $res$$Register);
9206 %}
9207 ins_pipe(pipe_slow);
9208 %}
9209
9210 // This pattern is generated automatically from cas.m4.
9211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9212 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9213 predicate(needs_acquiring_load_exclusive(n));
9214 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9215 ins_cost(VOLATILE_REF_COST);
9216 effect(TEMP_DEF res, KILL cr);
9217 format %{
9218 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9219 %}
9220 ins_encode %{
9221 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9222 Assembler::byte, /*acquire*/ true, /*release*/ true,
9223 /*weak*/ false, $res$$Register);
9224 __ sxtbw($res$$Register, $res$$Register);
9225 %}
9226 ins_pipe(pipe_slow);
9227 %}
9228
9229 // This pattern is generated automatically from cas.m4.
9230 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9231 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9232 predicate(needs_acquiring_load_exclusive(n));
9233 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9234 ins_cost(VOLATILE_REF_COST);
9235 effect(TEMP_DEF res, KILL cr);
9236 format %{
9237 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9238 %}
9239 ins_encode %{
9240 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9241 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9242 /*weak*/ false, $res$$Register);
9243 __ sxthw($res$$Register, $res$$Register);
9244 %}
9245 ins_pipe(pipe_slow);
9246 %}
9247
9248 // This pattern is generated automatically from cas.m4.
9249 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9250 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9251 predicate(needs_acquiring_load_exclusive(n));
9252 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9253 ins_cost(VOLATILE_REF_COST);
9254 effect(TEMP_DEF res, KILL cr);
9255 format %{
9256 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9257 %}
9258 ins_encode %{
9259 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9260 Assembler::word, /*acquire*/ true, /*release*/ true,
9261 /*weak*/ false, $res$$Register);
9262 %}
9263 ins_pipe(pipe_slow);
9264 %}
9265
9266 // This pattern is generated automatically from cas.m4.
9267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9268 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9269 predicate(needs_acquiring_load_exclusive(n));
9270 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9271 ins_cost(VOLATILE_REF_COST);
9272 effect(TEMP_DEF res, KILL cr);
9273 format %{
9274 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9275 %}
9276 ins_encode %{
9277 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9278 Assembler::xword, /*acquire*/ true, /*release*/ true,
9279 /*weak*/ false, $res$$Register);
9280 %}
9281 ins_pipe(pipe_slow);
9282 %}
9283
9284 // This pattern is generated automatically from cas.m4.
9285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9286 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9287 predicate(needs_acquiring_load_exclusive(n));
9288 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9289 ins_cost(VOLATILE_REF_COST);
9290 effect(TEMP_DEF res, KILL cr);
9291 format %{
9292 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9293 %}
9294 ins_encode %{
9295 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9296 Assembler::word, /*acquire*/ true, /*release*/ true,
9297 /*weak*/ false, $res$$Register);
9298 %}
9299 ins_pipe(pipe_slow);
9300 %}
9301
9302 // This pattern is generated automatically from cas.m4.
9303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9304 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9305 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9306 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9307 ins_cost(VOLATILE_REF_COST);
9308 effect(TEMP_DEF res, KILL cr);
9309 format %{
9310 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9311 %}
9312 ins_encode %{
9313 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9314 Assembler::xword, /*acquire*/ true, /*release*/ true,
9315 /*weak*/ false, $res$$Register);
9316 %}
9317 ins_pipe(pipe_slow);
9318 %}
9319
9320 // This pattern is generated automatically from cas.m4.
9321 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9322 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9323 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9324 ins_cost(2 * VOLATILE_REF_COST);
9325 effect(KILL cr);
9326 format %{
9327 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9328 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9329 %}
9330 ins_encode %{
9331 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9332 Assembler::byte, /*acquire*/ false, /*release*/ true,
9333 /*weak*/ true, noreg);
9334 __ csetw($res$$Register, Assembler::EQ);
9335 %}
9336 ins_pipe(pipe_slow);
9337 %}
9338
9339 // This pattern is generated automatically from cas.m4.
9340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9341 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9342 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9343 ins_cost(2 * VOLATILE_REF_COST);
9344 effect(KILL cr);
9345 format %{
9346 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9347 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9348 %}
9349 ins_encode %{
9350 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9351 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9352 /*weak*/ true, noreg);
9353 __ csetw($res$$Register, Assembler::EQ);
9354 %}
9355 ins_pipe(pipe_slow);
9356 %}
9357
9358 // This pattern is generated automatically from cas.m4.
9359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9360 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9361 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9362 ins_cost(2 * VOLATILE_REF_COST);
9363 effect(KILL cr);
9364 format %{
9365 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9366 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9367 %}
9368 ins_encode %{
9369 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9370 Assembler::word, /*acquire*/ false, /*release*/ true,
9371 /*weak*/ true, noreg);
9372 __ csetw($res$$Register, Assembler::EQ);
9373 %}
9374 ins_pipe(pipe_slow);
9375 %}
9376
9377 // This pattern is generated automatically from cas.m4.
9378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9379 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9380 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9381 ins_cost(2 * VOLATILE_REF_COST);
9382 effect(KILL cr);
9383 format %{
9384 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9385 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9386 %}
9387 ins_encode %{
9388 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9389 Assembler::xword, /*acquire*/ false, /*release*/ true,
9390 /*weak*/ true, noreg);
9391 __ csetw($res$$Register, Assembler::EQ);
9392 %}
9393 ins_pipe(pipe_slow);
9394 %}
9395
9396 // This pattern is generated automatically from cas.m4.
9397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9398 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9399 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9400 ins_cost(2 * VOLATILE_REF_COST);
9401 effect(KILL cr);
9402 format %{
9403 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9404 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9405 %}
9406 ins_encode %{
9407 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9408 Assembler::word, /*acquire*/ false, /*release*/ true,
9409 /*weak*/ true, noreg);
9410 __ csetw($res$$Register, Assembler::EQ);
9411 %}
9412 ins_pipe(pipe_slow);
9413 %}
9414
9415 // This pattern is generated automatically from cas.m4.
9416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9417 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9418 predicate(n->as_LoadStore()->barrier_data() == 0);
9419 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9420 ins_cost(2 * VOLATILE_REF_COST);
9421 effect(KILL cr);
9422 format %{
9423 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9424 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9425 %}
9426 ins_encode %{
9427 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9428 Assembler::xword, /*acquire*/ false, /*release*/ true,
9429 /*weak*/ true, noreg);
9430 __ csetw($res$$Register, Assembler::EQ);
9431 %}
9432 ins_pipe(pipe_slow);
9433 %}
9434
9435 // This pattern is generated automatically from cas.m4.
9436 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9437 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9438 predicate(needs_acquiring_load_exclusive(n));
9439 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9440 ins_cost(VOLATILE_REF_COST);
9441 effect(KILL cr);
9442 format %{
9443 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9444 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9445 %}
9446 ins_encode %{
9447 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9448 Assembler::byte, /*acquire*/ true, /*release*/ true,
9449 /*weak*/ true, noreg);
9450 __ csetw($res$$Register, Assembler::EQ);
9451 %}
9452 ins_pipe(pipe_slow);
9453 %}
9454
9455 // This pattern is generated automatically from cas.m4.
9456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9457 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9458 predicate(needs_acquiring_load_exclusive(n));
9459 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9460 ins_cost(VOLATILE_REF_COST);
9461 effect(KILL cr);
9462 format %{
9463 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9464 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9465 %}
9466 ins_encode %{
9467 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9468 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9469 /*weak*/ true, noreg);
9470 __ csetw($res$$Register, Assembler::EQ);
9471 %}
9472 ins_pipe(pipe_slow);
9473 %}
9474
9475 // This pattern is generated automatically from cas.m4.
9476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9477 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9478 predicate(needs_acquiring_load_exclusive(n));
9479 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9480 ins_cost(VOLATILE_REF_COST);
9481 effect(KILL cr);
9482 format %{
9483 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9484 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9485 %}
9486 ins_encode %{
9487 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9488 Assembler::word, /*acquire*/ true, /*release*/ true,
9489 /*weak*/ true, noreg);
9490 __ csetw($res$$Register, Assembler::EQ);
9491 %}
9492 ins_pipe(pipe_slow);
9493 %}
9494
9495 // This pattern is generated automatically from cas.m4.
9496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9497 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9498 predicate(needs_acquiring_load_exclusive(n));
9499 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9500 ins_cost(VOLATILE_REF_COST);
9501 effect(KILL cr);
9502 format %{
9503 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9504 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9505 %}
9506 ins_encode %{
9507 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9508 Assembler::xword, /*acquire*/ true, /*release*/ true,
9509 /*weak*/ true, noreg);
9510 __ csetw($res$$Register, Assembler::EQ);
9511 %}
9512 ins_pipe(pipe_slow);
9513 %}
9514
9515 // This pattern is generated automatically from cas.m4.
9516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9517 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9518 predicate(needs_acquiring_load_exclusive(n));
9519 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9520 ins_cost(VOLATILE_REF_COST);
9521 effect(KILL cr);
9522 format %{
9523 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9524 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9525 %}
9526 ins_encode %{
9527 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9528 Assembler::word, /*acquire*/ true, /*release*/ true,
9529 /*weak*/ true, noreg);
9530 __ csetw($res$$Register, Assembler::EQ);
9531 %}
9532 ins_pipe(pipe_slow);
9533 %}
9534
9535 // This pattern is generated automatically from cas.m4.
9536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9537 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9538 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9539 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9540 ins_cost(VOLATILE_REF_COST);
9541 effect(KILL cr);
9542 format %{
9543 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9544 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9545 %}
9546 ins_encode %{
9547 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9548 Assembler::xword, /*acquire*/ true, /*release*/ true,
9549 /*weak*/ true, noreg);
9550 __ csetw($res$$Register, Assembler::EQ);
9551 %}
9552 ins_pipe(pipe_slow);
9553 %}
9554
9555 // END This section of the file is automatically generated. Do not edit --------------
9556 // ---------------------------------------------------------------------
9557
9558 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9559 match(Set prev (GetAndSetI mem newv));
9560 ins_cost(2 * VOLATILE_REF_COST);
9561 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9562 ins_encode %{
9563 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9564 %}
9565 ins_pipe(pipe_serial);
9566 %}
9567
9568 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9569 match(Set prev (GetAndSetL mem newv));
9570 ins_cost(2 * VOLATILE_REF_COST);
9571 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9572 ins_encode %{
9573 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9574 %}
9575 ins_pipe(pipe_serial);
9576 %}
9577
9578 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9579 match(Set prev (GetAndSetN mem newv));
9580 ins_cost(2 * VOLATILE_REF_COST);
9581 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9582 ins_encode %{
9583 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9584 %}
9585 ins_pipe(pipe_serial);
9586 %}
9587
9588 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9589 predicate(n->as_LoadStore()->barrier_data() == 0);
9590 match(Set prev (GetAndSetP mem newv));
9591 ins_cost(2 * VOLATILE_REF_COST);
9592 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9593 ins_encode %{
9594 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9595 %}
9596 ins_pipe(pipe_serial);
9597 %}
9598
9599 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9600 predicate(needs_acquiring_load_exclusive(n));
9601 match(Set prev (GetAndSetI mem newv));
9602 ins_cost(VOLATILE_REF_COST);
9603 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9604 ins_encode %{
9605 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9606 %}
9607 ins_pipe(pipe_serial);
9608 %}
9609
9610 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9611 predicate(needs_acquiring_load_exclusive(n));
9612 match(Set prev (GetAndSetL mem newv));
9613 ins_cost(VOLATILE_REF_COST);
9614 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9615 ins_encode %{
9616 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9617 %}
9618 ins_pipe(pipe_serial);
9619 %}
9620
9621 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9622 predicate(needs_acquiring_load_exclusive(n));
9623 match(Set prev (GetAndSetN mem newv));
9624 ins_cost(VOLATILE_REF_COST);
9625 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9626 ins_encode %{
9627 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9628 %}
9629 ins_pipe(pipe_serial);
9630 %}
9631
9632 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9633 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9634 match(Set prev (GetAndSetP mem newv));
9635 ins_cost(VOLATILE_REF_COST);
9636 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9637 ins_encode %{
9638 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9639 %}
9640 ins_pipe(pipe_serial);
9641 %}
9642
9643
9644 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9645 match(Set newval (GetAndAddL mem incr));
9646 ins_cost(2 * VOLATILE_REF_COST + 1);
9647 format %{ "get_and_addL $newval, [$mem], $incr" %}
9648 ins_encode %{
9649 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9650 %}
9651 ins_pipe(pipe_serial);
9652 %}
9653
9654 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9655 predicate(n->as_LoadStore()->result_not_used());
9656 match(Set dummy (GetAndAddL mem incr));
9657 ins_cost(2 * VOLATILE_REF_COST);
9658 format %{ "get_and_addL [$mem], $incr" %}
9659 ins_encode %{
9660 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9661 %}
9662 ins_pipe(pipe_serial);
9663 %}
9664
9665 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9666 match(Set newval (GetAndAddL mem incr));
9667 ins_cost(2 * VOLATILE_REF_COST + 1);
9668 format %{ "get_and_addL $newval, [$mem], $incr" %}
9669 ins_encode %{
9670 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9671 %}
9672 ins_pipe(pipe_serial);
9673 %}
9674
9675 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9676 predicate(n->as_LoadStore()->result_not_used());
9677 match(Set dummy (GetAndAddL mem incr));
9678 ins_cost(2 * VOLATILE_REF_COST);
9679 format %{ "get_and_addL [$mem], $incr" %}
9680 ins_encode %{
9681 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9682 %}
9683 ins_pipe(pipe_serial);
9684 %}
9685
9686 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9687 match(Set newval (GetAndAddI mem incr));
9688 ins_cost(2 * VOLATILE_REF_COST + 1);
9689 format %{ "get_and_addI $newval, [$mem], $incr" %}
9690 ins_encode %{
9691 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9692 %}
9693 ins_pipe(pipe_serial);
9694 %}
9695
9696 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9697 predicate(n->as_LoadStore()->result_not_used());
9698 match(Set dummy (GetAndAddI mem incr));
9699 ins_cost(2 * VOLATILE_REF_COST);
9700 format %{ "get_and_addI [$mem], $incr" %}
9701 ins_encode %{
9702 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9703 %}
9704 ins_pipe(pipe_serial);
9705 %}
9706
9707 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9708 match(Set newval (GetAndAddI mem incr));
9709 ins_cost(2 * VOLATILE_REF_COST + 1);
9710 format %{ "get_and_addI $newval, [$mem], $incr" %}
9711 ins_encode %{
9712 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9713 %}
9714 ins_pipe(pipe_serial);
9715 %}
9716
9717 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9718 predicate(n->as_LoadStore()->result_not_used());
9719 match(Set dummy (GetAndAddI mem incr));
9720 ins_cost(2 * VOLATILE_REF_COST);
9721 format %{ "get_and_addI [$mem], $incr" %}
9722 ins_encode %{
9723 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9724 %}
9725 ins_pipe(pipe_serial);
9726 %}
9727
9728 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9729 predicate(needs_acquiring_load_exclusive(n));
9730 match(Set newval (GetAndAddL mem incr));
9731 ins_cost(VOLATILE_REF_COST + 1);
9732 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9733 ins_encode %{
9734 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9735 %}
9736 ins_pipe(pipe_serial);
9737 %}
9738
9739 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9740 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9741 match(Set dummy (GetAndAddL mem incr));
9742 ins_cost(VOLATILE_REF_COST);
9743 format %{ "get_and_addL_acq [$mem], $incr" %}
9744 ins_encode %{
9745 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9746 %}
9747 ins_pipe(pipe_serial);
9748 %}
9749
9750 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9751 predicate(needs_acquiring_load_exclusive(n));
9752 match(Set newval (GetAndAddL mem incr));
9753 ins_cost(VOLATILE_REF_COST + 1);
9754 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9755 ins_encode %{
9756 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9757 %}
9758 ins_pipe(pipe_serial);
9759 %}
9760
9761 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9762 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9763 match(Set dummy (GetAndAddL mem incr));
9764 ins_cost(VOLATILE_REF_COST);
9765 format %{ "get_and_addL_acq [$mem], $incr" %}
9766 ins_encode %{
9767 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9768 %}
9769 ins_pipe(pipe_serial);
9770 %}
9771
9772 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9773 predicate(needs_acquiring_load_exclusive(n));
9774 match(Set newval (GetAndAddI mem incr));
9775 ins_cost(VOLATILE_REF_COST + 1);
9776 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9777 ins_encode %{
9778 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9779 %}
9780 ins_pipe(pipe_serial);
9781 %}
9782
9783 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9784 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9785 match(Set dummy (GetAndAddI mem incr));
9786 ins_cost(VOLATILE_REF_COST);
9787 format %{ "get_and_addI_acq [$mem], $incr" %}
9788 ins_encode %{
9789 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9790 %}
9791 ins_pipe(pipe_serial);
9792 %}
9793
9794 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9795 predicate(needs_acquiring_load_exclusive(n));
9796 match(Set newval (GetAndAddI mem incr));
9797 ins_cost(VOLATILE_REF_COST + 1);
9798 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9799 ins_encode %{
9800 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9801 %}
9802 ins_pipe(pipe_serial);
9803 %}
9804
9805 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9806 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9807 match(Set dummy (GetAndAddI mem incr));
9808 ins_cost(VOLATILE_REF_COST);
9809 format %{ "get_and_addI_acq [$mem], $incr" %}
9810 ins_encode %{
9811 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9812 %}
9813 ins_pipe(pipe_serial);
9814 %}
9815
9816 // Manifest a CmpU result in an integer register.
9817 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9818 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9819 %{
9820 match(Set dst (CmpU3 src1 src2));
9821 effect(KILL flags);
9822
9823 ins_cost(INSN_COST * 3);
9824 format %{
9825 "cmpw $src1, $src2\n\t"
9826 "csetw $dst, ne\n\t"
9827 "cnegw $dst, lo\t# CmpU3(reg)"
9828 %}
9829 ins_encode %{
9830 __ cmpw($src1$$Register, $src2$$Register);
9831 __ csetw($dst$$Register, Assembler::NE);
9832 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9833 %}
9834
9835 ins_pipe(pipe_class_default);
9836 %}
9837
9838 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9839 %{
9840 match(Set dst (CmpU3 src1 src2));
9841 effect(KILL flags);
9842
9843 ins_cost(INSN_COST * 3);
9844 format %{
9845 "subsw zr, $src1, $src2\n\t"
9846 "csetw $dst, ne\n\t"
9847 "cnegw $dst, lo\t# CmpU3(imm)"
9848 %}
9849 ins_encode %{
9850 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9851 __ csetw($dst$$Register, Assembler::NE);
9852 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9853 %}
9854
9855 ins_pipe(pipe_class_default);
9856 %}
9857
9858 // Manifest a CmpUL result in an integer register.
9859 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9860 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9861 %{
9862 match(Set dst (CmpUL3 src1 src2));
9863 effect(KILL flags);
9864
9865 ins_cost(INSN_COST * 3);
9866 format %{
9867 "cmp $src1, $src2\n\t"
9868 "csetw $dst, ne\n\t"
9869 "cnegw $dst, lo\t# CmpUL3(reg)"
9870 %}
9871 ins_encode %{
9872 __ cmp($src1$$Register, $src2$$Register);
9873 __ csetw($dst$$Register, Assembler::NE);
9874 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9875 %}
9876
9877 ins_pipe(pipe_class_default);
9878 %}
9879
9880 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9881 %{
9882 match(Set dst (CmpUL3 src1 src2));
9883 effect(KILL flags);
9884
9885 ins_cost(INSN_COST * 3);
9886 format %{
9887 "subs zr, $src1, $src2\n\t"
9888 "csetw $dst, ne\n\t"
9889 "cnegw $dst, lo\t# CmpUL3(imm)"
9890 %}
9891 ins_encode %{
9892 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9893 __ csetw($dst$$Register, Assembler::NE);
9894 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9895 %}
9896
9897 ins_pipe(pipe_class_default);
9898 %}
9899
9900 // Manifest a CmpL result in an integer register.
9901 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9902 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9903 %{
9904 match(Set dst (CmpL3 src1 src2));
9905 effect(KILL flags);
9906
9907 ins_cost(INSN_COST * 3);
9908 format %{
9909 "cmp $src1, $src2\n\t"
9910 "csetw $dst, ne\n\t"
9911 "cnegw $dst, lt\t# CmpL3(reg)"
9912 %}
9913 ins_encode %{
9914 __ cmp($src1$$Register, $src2$$Register);
9915 __ csetw($dst$$Register, Assembler::NE);
9916 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9917 %}
9918
9919 ins_pipe(pipe_class_default);
9920 %}
9921
9922 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9923 %{
9924 match(Set dst (CmpL3 src1 src2));
9925 effect(KILL flags);
9926
9927 ins_cost(INSN_COST * 3);
9928 format %{
9929 "subs zr, $src1, $src2\n\t"
9930 "csetw $dst, ne\n\t"
9931 "cnegw $dst, lt\t# CmpL3(imm)"
9932 %}
9933 ins_encode %{
9934 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9935 __ csetw($dst$$Register, Assembler::NE);
9936 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9937 %}
9938
9939 ins_pipe(pipe_class_default);
9940 %}
9941
9942 // ============================================================================
9943 // Conditional Move Instructions
9944
9945 // n.b. we have identical rules for both a signed compare op (cmpOp)
9946 // and an unsigned compare op (cmpOpU). it would be nice if we could
9947 // define an op class which merged both inputs and use it to type the
9948 // argument to a single rule. unfortunatelyt his fails because the
9949 // opclass does not live up to the COND_INTER interface of its
9950 // component operands. When the generic code tries to negate the
9951 // operand it ends up running the generci Machoper::negate method
9952 // which throws a ShouldNotHappen. So, we have to provide two flavours
9953 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9954
9955 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9956 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9957
9958 ins_cost(INSN_COST * 2);
9959 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9960
9961 ins_encode %{
9962 __ cselw(as_Register($dst$$reg),
9963 as_Register($src2$$reg),
9964 as_Register($src1$$reg),
9965 (Assembler::Condition)$cmp$$cmpcode);
9966 %}
9967
9968 ins_pipe(icond_reg_reg);
9969 %}
9970
9971 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9972 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9973
9974 ins_cost(INSN_COST * 2);
9975 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9976
9977 ins_encode %{
9978 __ cselw(as_Register($dst$$reg),
9979 as_Register($src2$$reg),
9980 as_Register($src1$$reg),
9981 (Assembler::Condition)$cmp$$cmpcode);
9982 %}
9983
9984 ins_pipe(icond_reg_reg);
9985 %}
9986
9987 // special cases where one arg is zero
9988
9989 // n.b. this is selected in preference to the rule above because it
9990 // avoids loading constant 0 into a source register
9991
9992 // TODO
9993 // we ought only to be able to cull one of these variants as the ideal
9994 // transforms ought always to order the zero consistently (to left/right?)
9995
9996 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9997 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9998
9999 ins_cost(INSN_COST * 2);
10000 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
10001
10002 ins_encode %{
10003 __ cselw(as_Register($dst$$reg),
10004 as_Register($src$$reg),
10005 zr,
10006 (Assembler::Condition)$cmp$$cmpcode);
10007 %}
10008
10009 ins_pipe(icond_reg);
10010 %}
10011
10012 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10013 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10014
10015 ins_cost(INSN_COST * 2);
10016 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
10017
10018 ins_encode %{
10019 __ cselw(as_Register($dst$$reg),
10020 as_Register($src$$reg),
10021 zr,
10022 (Assembler::Condition)$cmp$$cmpcode);
10023 %}
10024
10025 ins_pipe(icond_reg);
10026 %}
10027
10028 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10029 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10030
10031 ins_cost(INSN_COST * 2);
10032 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
10033
10034 ins_encode %{
10035 __ cselw(as_Register($dst$$reg),
10036 zr,
10037 as_Register($src$$reg),
10038 (Assembler::Condition)$cmp$$cmpcode);
10039 %}
10040
10041 ins_pipe(icond_reg);
10042 %}
10043
10044 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10045 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10046
10047 ins_cost(INSN_COST * 2);
10048 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
10049
10050 ins_encode %{
10051 __ cselw(as_Register($dst$$reg),
10052 zr,
10053 as_Register($src$$reg),
10054 (Assembler::Condition)$cmp$$cmpcode);
10055 %}
10056
10057 ins_pipe(icond_reg);
10058 %}
10059
10060 // special case for creating a boolean 0 or 1
10061
10062 // n.b. this is selected in preference to the rule above because it
10063 // avoids loading constants 0 and 1 into a source register
10064
10065 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10066 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10067
10068 ins_cost(INSN_COST * 2);
10069 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
10070
10071 ins_encode %{
10072 // equivalently
10073 // cset(as_Register($dst$$reg),
10074 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10075 __ csincw(as_Register($dst$$reg),
10076 zr,
10077 zr,
10078 (Assembler::Condition)$cmp$$cmpcode);
10079 %}
10080
10081 ins_pipe(icond_none);
10082 %}
10083
10084 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10085 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10086
10087 ins_cost(INSN_COST * 2);
10088 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
10089
10090 ins_encode %{
10091 // equivalently
10092 // cset(as_Register($dst$$reg),
10093 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10094 __ csincw(as_Register($dst$$reg),
10095 zr,
10096 zr,
10097 (Assembler::Condition)$cmp$$cmpcode);
10098 %}
10099
10100 ins_pipe(icond_none);
10101 %}
10102
10103 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10104 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10105
10106 ins_cost(INSN_COST * 2);
10107 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
10108
10109 ins_encode %{
10110 __ csel(as_Register($dst$$reg),
10111 as_Register($src2$$reg),
10112 as_Register($src1$$reg),
10113 (Assembler::Condition)$cmp$$cmpcode);
10114 %}
10115
10116 ins_pipe(icond_reg_reg);
10117 %}
10118
10119 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10120 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10121
10122 ins_cost(INSN_COST * 2);
10123 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
10124
10125 ins_encode %{
10126 __ csel(as_Register($dst$$reg),
10127 as_Register($src2$$reg),
10128 as_Register($src1$$reg),
10129 (Assembler::Condition)$cmp$$cmpcode);
10130 %}
10131
10132 ins_pipe(icond_reg_reg);
10133 %}
10134
10135 // special cases where one arg is zero
10136
10137 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10138 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10139
10140 ins_cost(INSN_COST * 2);
10141 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
10142
10143 ins_encode %{
10144 __ csel(as_Register($dst$$reg),
10145 zr,
10146 as_Register($src$$reg),
10147 (Assembler::Condition)$cmp$$cmpcode);
10148 %}
10149
10150 ins_pipe(icond_reg);
10151 %}
10152
10153 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10154 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10155
10156 ins_cost(INSN_COST * 2);
10157 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
10158
10159 ins_encode %{
10160 __ csel(as_Register($dst$$reg),
10161 zr,
10162 as_Register($src$$reg),
10163 (Assembler::Condition)$cmp$$cmpcode);
10164 %}
10165
10166 ins_pipe(icond_reg);
10167 %}
10168
10169 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10170 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10171
10172 ins_cost(INSN_COST * 2);
10173 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
10174
10175 ins_encode %{
10176 __ csel(as_Register($dst$$reg),
10177 as_Register($src$$reg),
10178 zr,
10179 (Assembler::Condition)$cmp$$cmpcode);
10180 %}
10181
10182 ins_pipe(icond_reg);
10183 %}
10184
10185 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10186 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10187
10188 ins_cost(INSN_COST * 2);
10189 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
10190
10191 ins_encode %{
10192 __ csel(as_Register($dst$$reg),
10193 as_Register($src$$reg),
10194 zr,
10195 (Assembler::Condition)$cmp$$cmpcode);
10196 %}
10197
10198 ins_pipe(icond_reg);
10199 %}
10200
10201 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10202 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10203
10204 ins_cost(INSN_COST * 2);
10205 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
10206
10207 ins_encode %{
10208 __ csel(as_Register($dst$$reg),
10209 as_Register($src2$$reg),
10210 as_Register($src1$$reg),
10211 (Assembler::Condition)$cmp$$cmpcode);
10212 %}
10213
10214 ins_pipe(icond_reg_reg);
10215 %}
10216
10217 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10218 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10219
10220 ins_cost(INSN_COST * 2);
10221 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
10222
10223 ins_encode %{
10224 __ csel(as_Register($dst$$reg),
10225 as_Register($src2$$reg),
10226 as_Register($src1$$reg),
10227 (Assembler::Condition)$cmp$$cmpcode);
10228 %}
10229
10230 ins_pipe(icond_reg_reg);
10231 %}
10232
10233 // special cases where one arg is zero
10234
10235 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10236 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10237
10238 ins_cost(INSN_COST * 2);
10239 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
10240
10241 ins_encode %{
10242 __ csel(as_Register($dst$$reg),
10243 zr,
10244 as_Register($src$$reg),
10245 (Assembler::Condition)$cmp$$cmpcode);
10246 %}
10247
10248 ins_pipe(icond_reg);
10249 %}
10250
10251 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10252 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10253
10254 ins_cost(INSN_COST * 2);
10255 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
10256
10257 ins_encode %{
10258 __ csel(as_Register($dst$$reg),
10259 zr,
10260 as_Register($src$$reg),
10261 (Assembler::Condition)$cmp$$cmpcode);
10262 %}
10263
10264 ins_pipe(icond_reg);
10265 %}
10266
10267 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10268 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10269
10270 ins_cost(INSN_COST * 2);
10271 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
10272
10273 ins_encode %{
10274 __ csel(as_Register($dst$$reg),
10275 as_Register($src$$reg),
10276 zr,
10277 (Assembler::Condition)$cmp$$cmpcode);
10278 %}
10279
10280 ins_pipe(icond_reg);
10281 %}
10282
10283 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10284 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10285
10286 ins_cost(INSN_COST * 2);
10287 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
10288
10289 ins_encode %{
10290 __ csel(as_Register($dst$$reg),
10291 as_Register($src$$reg),
10292 zr,
10293 (Assembler::Condition)$cmp$$cmpcode);
10294 %}
10295
10296 ins_pipe(icond_reg);
10297 %}
10298
10299 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10300 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10301
10302 ins_cost(INSN_COST * 2);
10303 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10304
10305 ins_encode %{
10306 __ cselw(as_Register($dst$$reg),
10307 as_Register($src2$$reg),
10308 as_Register($src1$$reg),
10309 (Assembler::Condition)$cmp$$cmpcode);
10310 %}
10311
10312 ins_pipe(icond_reg_reg);
10313 %}
10314
10315 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10316 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10317
10318 ins_cost(INSN_COST * 2);
10319 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10320
10321 ins_encode %{
10322 __ cselw(as_Register($dst$$reg),
10323 as_Register($src2$$reg),
10324 as_Register($src1$$reg),
10325 (Assembler::Condition)$cmp$$cmpcode);
10326 %}
10327
10328 ins_pipe(icond_reg_reg);
10329 %}
10330
10331 // special cases where one arg is zero
10332
10333 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10334 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10335
10336 ins_cost(INSN_COST * 2);
10337 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
10338
10339 ins_encode %{
10340 __ cselw(as_Register($dst$$reg),
10341 zr,
10342 as_Register($src$$reg),
10343 (Assembler::Condition)$cmp$$cmpcode);
10344 %}
10345
10346 ins_pipe(icond_reg);
10347 %}
10348
10349 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10350 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10351
10352 ins_cost(INSN_COST * 2);
10353 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
10354
10355 ins_encode %{
10356 __ cselw(as_Register($dst$$reg),
10357 zr,
10358 as_Register($src$$reg),
10359 (Assembler::Condition)$cmp$$cmpcode);
10360 %}
10361
10362 ins_pipe(icond_reg);
10363 %}
10364
10365 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10366 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10367
10368 ins_cost(INSN_COST * 2);
10369 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
10370
10371 ins_encode %{
10372 __ cselw(as_Register($dst$$reg),
10373 as_Register($src$$reg),
10374 zr,
10375 (Assembler::Condition)$cmp$$cmpcode);
10376 %}
10377
10378 ins_pipe(icond_reg);
10379 %}
10380
10381 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10382 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10383
10384 ins_cost(INSN_COST * 2);
10385 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
10386
10387 ins_encode %{
10388 __ cselw(as_Register($dst$$reg),
10389 as_Register($src$$reg),
10390 zr,
10391 (Assembler::Condition)$cmp$$cmpcode);
10392 %}
10393
10394 ins_pipe(icond_reg);
10395 %}
10396
10397 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
10398 %{
10399 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10400
10401 ins_cost(INSN_COST * 3);
10402
10403 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10404 ins_encode %{
10405 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10406 __ fcsels(as_FloatRegister($dst$$reg),
10407 as_FloatRegister($src2$$reg),
10408 as_FloatRegister($src1$$reg),
10409 cond);
10410 %}
10411
10412 ins_pipe(fp_cond_reg_reg_s);
10413 %}
10414
10415 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
10416 %{
10417 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10418
10419 ins_cost(INSN_COST * 3);
10420
10421 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10422 ins_encode %{
10423 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10424 __ fcsels(as_FloatRegister($dst$$reg),
10425 as_FloatRegister($src2$$reg),
10426 as_FloatRegister($src1$$reg),
10427 cond);
10428 %}
10429
10430 ins_pipe(fp_cond_reg_reg_s);
10431 %}
10432
10433 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10434 %{
10435 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10436
10437 ins_cost(INSN_COST * 3);
10438
10439 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10440 ins_encode %{
10441 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10442 __ fcseld(as_FloatRegister($dst$$reg),
10443 as_FloatRegister($src2$$reg),
10444 as_FloatRegister($src1$$reg),
10445 cond);
10446 %}
10447
10448 ins_pipe(fp_cond_reg_reg_d);
10449 %}
10450
10451 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10452 %{
10453 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10454
10455 ins_cost(INSN_COST * 3);
10456
10457 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10458 ins_encode %{
10459 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10460 __ fcseld(as_FloatRegister($dst$$reg),
10461 as_FloatRegister($src2$$reg),
10462 as_FloatRegister($src1$$reg),
10463 cond);
10464 %}
10465
10466 ins_pipe(fp_cond_reg_reg_d);
10467 %}
10468
10469 // ============================================================================
10470 // Arithmetic Instructions
10471 //
10472
10473 // Integer Addition
10474
10475 // TODO
10476 // these currently employ operations which do not set CR and hence are
10477 // not flagged as killing CR but we would like to isolate the cases
10478 // where we want to set flags from those where we don't. need to work
10479 // out how to do that.
10480
10481 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10482 match(Set dst (AddI src1 src2));
10483
10484 ins_cost(INSN_COST);
10485 format %{ "addw $dst, $src1, $src2" %}
10486
10487 ins_encode %{
10488 __ addw(as_Register($dst$$reg),
10489 as_Register($src1$$reg),
10490 as_Register($src2$$reg));
10491 %}
10492
10493 ins_pipe(ialu_reg_reg);
10494 %}
10495
10496 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10497 match(Set dst (AddI src1 src2));
10498
10499 ins_cost(INSN_COST);
10500 format %{ "addw $dst, $src1, $src2" %}
10501
10502 // use opcode to indicate that this is an add not a sub
10503 opcode(0x0);
10504
10505 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10506
10507 ins_pipe(ialu_reg_imm);
10508 %}
10509
10510 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10511 match(Set dst (AddI (ConvL2I src1) src2));
10512
10513 ins_cost(INSN_COST);
10514 format %{ "addw $dst, $src1, $src2" %}
10515
10516 // use opcode to indicate that this is an add not a sub
10517 opcode(0x0);
10518
10519 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10520
10521 ins_pipe(ialu_reg_imm);
10522 %}
10523
10524 // Pointer Addition
10525 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10526 match(Set dst (AddP src1 src2));
10527
10528 ins_cost(INSN_COST);
10529 format %{ "add $dst, $src1, $src2\t# ptr" %}
10530
10531 ins_encode %{
10532 __ add(as_Register($dst$$reg),
10533 as_Register($src1$$reg),
10534 as_Register($src2$$reg));
10535 %}
10536
10537 ins_pipe(ialu_reg_reg);
10538 %}
10539
10540 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10541 match(Set dst (AddP src1 (ConvI2L src2)));
10542
10543 ins_cost(1.9 * INSN_COST);
10544 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10545
10546 ins_encode %{
10547 __ add(as_Register($dst$$reg),
10548 as_Register($src1$$reg),
10549 as_Register($src2$$reg), ext::sxtw);
10550 %}
10551
10552 ins_pipe(ialu_reg_reg);
10553 %}
10554
10555 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10556 match(Set dst (AddP src1 (LShiftL src2 scale)));
10557
10558 ins_cost(1.9 * INSN_COST);
10559 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10560
10561 ins_encode %{
10562 __ lea(as_Register($dst$$reg),
10563 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10564 Address::lsl($scale$$constant)));
10565 %}
10566
10567 ins_pipe(ialu_reg_reg_shift);
10568 %}
10569
10570 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10571 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10572
10573 ins_cost(1.9 * INSN_COST);
10574 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10575
10576 ins_encode %{
10577 __ lea(as_Register($dst$$reg),
10578 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10579 Address::sxtw($scale$$constant)));
10580 %}
10581
10582 ins_pipe(ialu_reg_reg_shift);
10583 %}
10584
10585 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10586 match(Set dst (LShiftL (ConvI2L src) scale));
10587
10588 ins_cost(INSN_COST);
10589 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10590
10591 ins_encode %{
10592 __ sbfiz(as_Register($dst$$reg),
10593 as_Register($src$$reg),
10594 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10595 %}
10596
10597 ins_pipe(ialu_reg_shift);
10598 %}
10599
10600 // Pointer Immediate Addition
10601 // n.b. this needs to be more expensive than using an indirect memory
10602 // operand
10603 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10604 match(Set dst (AddP src1 src2));
10605
10606 ins_cost(INSN_COST);
10607 format %{ "add $dst, $src1, $src2\t# ptr" %}
10608
10609 // use opcode to indicate that this is an add not a sub
10610 opcode(0x0);
10611
10612 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10613
10614 ins_pipe(ialu_reg_imm);
10615 %}
10616
10617 // Long Addition
10618 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10619
10620 match(Set dst (AddL src1 src2));
10621
10622 ins_cost(INSN_COST);
10623 format %{ "add $dst, $src1, $src2" %}
10624
10625 ins_encode %{
10626 __ add(as_Register($dst$$reg),
10627 as_Register($src1$$reg),
10628 as_Register($src2$$reg));
10629 %}
10630
10631 ins_pipe(ialu_reg_reg);
10632 %}
10633
10634 // No constant pool entries requiredLong Immediate Addition.
10635 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10636 match(Set dst (AddL src1 src2));
10637
10638 ins_cost(INSN_COST);
10639 format %{ "add $dst, $src1, $src2" %}
10640
10641 // use opcode to indicate that this is an add not a sub
10642 opcode(0x0);
10643
10644 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10645
10646 ins_pipe(ialu_reg_imm);
10647 %}
10648
10649 // Integer Subtraction
10650 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10651 match(Set dst (SubI src1 src2));
10652
10653 ins_cost(INSN_COST);
10654 format %{ "subw $dst, $src1, $src2" %}
10655
10656 ins_encode %{
10657 __ subw(as_Register($dst$$reg),
10658 as_Register($src1$$reg),
10659 as_Register($src2$$reg));
10660 %}
10661
10662 ins_pipe(ialu_reg_reg);
10663 %}
10664
10665 // Immediate Subtraction
10666 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10667 match(Set dst (SubI src1 src2));
10668
10669 ins_cost(INSN_COST);
10670 format %{ "subw $dst, $src1, $src2" %}
10671
10672 // use opcode to indicate that this is a sub not an add
10673 opcode(0x1);
10674
10675 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10676
10677 ins_pipe(ialu_reg_imm);
10678 %}
10679
10680 // Long Subtraction
10681 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10682
10683 match(Set dst (SubL src1 src2));
10684
10685 ins_cost(INSN_COST);
10686 format %{ "sub $dst, $src1, $src2" %}
10687
10688 ins_encode %{
10689 __ sub(as_Register($dst$$reg),
10690 as_Register($src1$$reg),
10691 as_Register($src2$$reg));
10692 %}
10693
10694 ins_pipe(ialu_reg_reg);
10695 %}
10696
10697 // No constant pool entries requiredLong Immediate Subtraction.
10698 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10699 match(Set dst (SubL src1 src2));
10700
10701 ins_cost(INSN_COST);
10702 format %{ "sub$dst, $src1, $src2" %}
10703
10704 // use opcode to indicate that this is a sub not an add
10705 opcode(0x1);
10706
10707 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10708
10709 ins_pipe(ialu_reg_imm);
10710 %}
10711
10712 // Integer Negation (special case for sub)
10713
10714 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10715 match(Set dst (SubI zero src));
10716
10717 ins_cost(INSN_COST);
10718 format %{ "negw $dst, $src\t# int" %}
10719
10720 ins_encode %{
10721 __ negw(as_Register($dst$$reg),
10722 as_Register($src$$reg));
10723 %}
10724
10725 ins_pipe(ialu_reg);
10726 %}
10727
10728 // Long Negation
10729
10730 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10731 match(Set dst (SubL zero src));
10732
10733 ins_cost(INSN_COST);
10734 format %{ "neg $dst, $src\t# long" %}
10735
10736 ins_encode %{
10737 __ neg(as_Register($dst$$reg),
10738 as_Register($src$$reg));
10739 %}
10740
10741 ins_pipe(ialu_reg);
10742 %}
10743
10744 // Integer Multiply
10745
10746 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10747 match(Set dst (MulI src1 src2));
10748
10749 ins_cost(INSN_COST * 3);
10750 format %{ "mulw $dst, $src1, $src2" %}
10751
10752 ins_encode %{
10753 __ mulw(as_Register($dst$$reg),
10754 as_Register($src1$$reg),
10755 as_Register($src2$$reg));
10756 %}
10757
10758 ins_pipe(imul_reg_reg);
10759 %}
10760
10761 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10762 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10763
10764 ins_cost(INSN_COST * 3);
10765 format %{ "smull $dst, $src1, $src2" %}
10766
10767 ins_encode %{
10768 __ smull(as_Register($dst$$reg),
10769 as_Register($src1$$reg),
10770 as_Register($src2$$reg));
10771 %}
10772
10773 ins_pipe(imul_reg_reg);
10774 %}
10775
10776 // Long Multiply
10777
10778 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10779 match(Set dst (MulL src1 src2));
10780
10781 ins_cost(INSN_COST * 5);
10782 format %{ "mul $dst, $src1, $src2" %}
10783
10784 ins_encode %{
10785 __ mul(as_Register($dst$$reg),
10786 as_Register($src1$$reg),
10787 as_Register($src2$$reg));
10788 %}
10789
10790 ins_pipe(lmul_reg_reg);
10791 %}
10792
10793 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10794 %{
10795 match(Set dst (MulHiL src1 src2));
10796
10797 ins_cost(INSN_COST * 7);
10798 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10799
10800 ins_encode %{
10801 __ smulh(as_Register($dst$$reg),
10802 as_Register($src1$$reg),
10803 as_Register($src2$$reg));
10804 %}
10805
10806 ins_pipe(lmul_reg_reg);
10807 %}
10808
10809 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10810 %{
10811 match(Set dst (UMulHiL src1 src2));
10812
10813 ins_cost(INSN_COST * 7);
10814 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10815
10816 ins_encode %{
10817 __ umulh(as_Register($dst$$reg),
10818 as_Register($src1$$reg),
10819 as_Register($src2$$reg));
10820 %}
10821
10822 ins_pipe(lmul_reg_reg);
10823 %}
10824
10825 // Combined Integer Multiply & Add/Sub
10826
10827 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10828 match(Set dst (AddI src3 (MulI src1 src2)));
10829
10830 ins_cost(INSN_COST * 3);
10831 format %{ "madd $dst, $src1, $src2, $src3" %}
10832
10833 ins_encode %{
10834 __ maddw(as_Register($dst$$reg),
10835 as_Register($src1$$reg),
10836 as_Register($src2$$reg),
10837 as_Register($src3$$reg));
10838 %}
10839
10840 ins_pipe(imac_reg_reg);
10841 %}
10842
10843 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10844 match(Set dst (SubI src3 (MulI src1 src2)));
10845
10846 ins_cost(INSN_COST * 3);
10847 format %{ "msub $dst, $src1, $src2, $src3" %}
10848
10849 ins_encode %{
10850 __ msubw(as_Register($dst$$reg),
10851 as_Register($src1$$reg),
10852 as_Register($src2$$reg),
10853 as_Register($src3$$reg));
10854 %}
10855
10856 ins_pipe(imac_reg_reg);
10857 %}
10858
10859 // Combined Integer Multiply & Neg
10860
10861 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10862 match(Set dst (MulI (SubI zero src1) src2));
10863
10864 ins_cost(INSN_COST * 3);
10865 format %{ "mneg $dst, $src1, $src2" %}
10866
10867 ins_encode %{
10868 __ mnegw(as_Register($dst$$reg),
10869 as_Register($src1$$reg),
10870 as_Register($src2$$reg));
10871 %}
10872
10873 ins_pipe(imac_reg_reg);
10874 %}
10875
10876 // Combined Long Multiply & Add/Sub
10877
10878 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10879 match(Set dst (AddL src3 (MulL src1 src2)));
10880
10881 ins_cost(INSN_COST * 5);
10882 format %{ "madd $dst, $src1, $src2, $src3" %}
10883
10884 ins_encode %{
10885 __ madd(as_Register($dst$$reg),
10886 as_Register($src1$$reg),
10887 as_Register($src2$$reg),
10888 as_Register($src3$$reg));
10889 %}
10890
10891 ins_pipe(lmac_reg_reg);
10892 %}
10893
10894 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10895 match(Set dst (SubL src3 (MulL src1 src2)));
10896
10897 ins_cost(INSN_COST * 5);
10898 format %{ "msub $dst, $src1, $src2, $src3" %}
10899
10900 ins_encode %{
10901 __ msub(as_Register($dst$$reg),
10902 as_Register($src1$$reg),
10903 as_Register($src2$$reg),
10904 as_Register($src3$$reg));
10905 %}
10906
10907 ins_pipe(lmac_reg_reg);
10908 %}
10909
10910 // Combined Long Multiply & Neg
10911
10912 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10913 match(Set dst (MulL (SubL zero src1) src2));
10914
10915 ins_cost(INSN_COST * 5);
10916 format %{ "mneg $dst, $src1, $src2" %}
10917
10918 ins_encode %{
10919 __ mneg(as_Register($dst$$reg),
10920 as_Register($src1$$reg),
10921 as_Register($src2$$reg));
10922 %}
10923
10924 ins_pipe(lmac_reg_reg);
10925 %}
10926
10927 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10928
10929 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10930 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10931
10932 ins_cost(INSN_COST * 3);
10933 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10934
10935 ins_encode %{
10936 __ smaddl(as_Register($dst$$reg),
10937 as_Register($src1$$reg),
10938 as_Register($src2$$reg),
10939 as_Register($src3$$reg));
10940 %}
10941
10942 ins_pipe(imac_reg_reg);
10943 %}
10944
10945 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10946 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10947
10948 ins_cost(INSN_COST * 3);
10949 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10950
10951 ins_encode %{
10952 __ smsubl(as_Register($dst$$reg),
10953 as_Register($src1$$reg),
10954 as_Register($src2$$reg),
10955 as_Register($src3$$reg));
10956 %}
10957
10958 ins_pipe(imac_reg_reg);
10959 %}
10960
10961 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10962 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10963
10964 ins_cost(INSN_COST * 3);
10965 format %{ "smnegl $dst, $src1, $src2" %}
10966
10967 ins_encode %{
10968 __ smnegl(as_Register($dst$$reg),
10969 as_Register($src1$$reg),
10970 as_Register($src2$$reg));
10971 %}
10972
10973 ins_pipe(imac_reg_reg);
10974 %}
10975
10976 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10977
10978 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10979 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10980
10981 ins_cost(INSN_COST * 5);
10982 format %{ "mulw rscratch1, $src1, $src2\n\t"
10983 "maddw $dst, $src3, $src4, rscratch1" %}
10984
10985 ins_encode %{
10986 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10987 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10988
10989 ins_pipe(imac_reg_reg);
10990 %}
10991
10992 // Integer Divide
10993
10994 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10995 match(Set dst (DivI src1 src2));
10996
10997 ins_cost(INSN_COST * 19);
10998 format %{ "sdivw $dst, $src1, $src2" %}
10999
11000 ins_encode(aarch64_enc_divw(dst, src1, src2));
11001 ins_pipe(idiv_reg_reg);
11002 %}
11003
11004 // Long Divide
11005
11006 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11007 match(Set dst (DivL src1 src2));
11008
11009 ins_cost(INSN_COST * 35);
11010 format %{ "sdiv $dst, $src1, $src2" %}
11011
11012 ins_encode(aarch64_enc_div(dst, src1, src2));
11013 ins_pipe(ldiv_reg_reg);
11014 %}
11015
11016 // Integer Remainder
11017
11018 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11019 match(Set dst (ModI src1 src2));
11020
11021 ins_cost(INSN_COST * 22);
11022 format %{ "sdivw rscratch1, $src1, $src2\n\t"
11023 "msubw $dst, rscratch1, $src2, $src1" %}
11024
11025 ins_encode(aarch64_enc_modw(dst, src1, src2));
11026 ins_pipe(idiv_reg_reg);
11027 %}
11028
11029 // Long Remainder
11030
11031 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11032 match(Set dst (ModL src1 src2));
11033
11034 ins_cost(INSN_COST * 38);
11035 format %{ "sdiv rscratch1, $src1, $src2\n"
11036 "msub $dst, rscratch1, $src2, $src1" %}
11037
11038 ins_encode(aarch64_enc_mod(dst, src1, src2));
11039 ins_pipe(ldiv_reg_reg);
11040 %}
11041
11042 // Unsigned Integer Divide
11043
11044 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11045 match(Set dst (UDivI src1 src2));
11046
11047 ins_cost(INSN_COST * 19);
11048 format %{ "udivw $dst, $src1, $src2" %}
11049
11050 ins_encode %{
11051 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
11052 %}
11053
11054 ins_pipe(idiv_reg_reg);
11055 %}
11056
11057 // Unsigned Long Divide
11058
11059 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11060 match(Set dst (UDivL src1 src2));
11061
11062 ins_cost(INSN_COST * 35);
11063 format %{ "udiv $dst, $src1, $src2" %}
11064
11065 ins_encode %{
11066 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
11067 %}
11068
11069 ins_pipe(ldiv_reg_reg);
11070 %}
11071
11072 // Unsigned Integer Remainder
11073
11074 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11075 match(Set dst (UModI src1 src2));
11076
11077 ins_cost(INSN_COST * 22);
11078 format %{ "udivw rscratch1, $src1, $src2\n\t"
11079 "msubw $dst, rscratch1, $src2, $src1" %}
11080
11081 ins_encode %{
11082 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
11083 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
11084 %}
11085
11086 ins_pipe(idiv_reg_reg);
11087 %}
11088
11089 // Unsigned Long Remainder
11090
11091 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11092 match(Set dst (UModL src1 src2));
11093
11094 ins_cost(INSN_COST * 38);
11095 format %{ "udiv rscratch1, $src1, $src2\n"
11096 "msub $dst, rscratch1, $src2, $src1" %}
11097
11098 ins_encode %{
11099 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
11100 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
11101 %}
11102
11103 ins_pipe(ldiv_reg_reg);
11104 %}
11105
11106 // Integer Shifts
11107
11108 // Shift Left Register
11109 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11110 match(Set dst (LShiftI src1 src2));
11111
11112 ins_cost(INSN_COST * 2);
11113 format %{ "lslvw $dst, $src1, $src2" %}
11114
11115 ins_encode %{
11116 __ lslvw(as_Register($dst$$reg),
11117 as_Register($src1$$reg),
11118 as_Register($src2$$reg));
11119 %}
11120
11121 ins_pipe(ialu_reg_reg_vshift);
11122 %}
11123
11124 // Shift Left Immediate
11125 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11126 match(Set dst (LShiftI src1 src2));
11127
11128 ins_cost(INSN_COST);
11129 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11130
11131 ins_encode %{
11132 __ lslw(as_Register($dst$$reg),
11133 as_Register($src1$$reg),
11134 $src2$$constant & 0x1f);
11135 %}
11136
11137 ins_pipe(ialu_reg_shift);
11138 %}
11139
11140 // Shift Right Logical Register
11141 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11142 match(Set dst (URShiftI src1 src2));
11143
11144 ins_cost(INSN_COST * 2);
11145 format %{ "lsrvw $dst, $src1, $src2" %}
11146
11147 ins_encode %{
11148 __ lsrvw(as_Register($dst$$reg),
11149 as_Register($src1$$reg),
11150 as_Register($src2$$reg));
11151 %}
11152
11153 ins_pipe(ialu_reg_reg_vshift);
11154 %}
11155
11156 // Shift Right Logical Immediate
11157 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11158 match(Set dst (URShiftI src1 src2));
11159
11160 ins_cost(INSN_COST);
11161 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11162
11163 ins_encode %{
11164 __ lsrw(as_Register($dst$$reg),
11165 as_Register($src1$$reg),
11166 $src2$$constant & 0x1f);
11167 %}
11168
11169 ins_pipe(ialu_reg_shift);
11170 %}
11171
11172 // Shift Right Arithmetic Register
11173 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11174 match(Set dst (RShiftI src1 src2));
11175
11176 ins_cost(INSN_COST * 2);
11177 format %{ "asrvw $dst, $src1, $src2" %}
11178
11179 ins_encode %{
11180 __ asrvw(as_Register($dst$$reg),
11181 as_Register($src1$$reg),
11182 as_Register($src2$$reg));
11183 %}
11184
11185 ins_pipe(ialu_reg_reg_vshift);
11186 %}
11187
11188 // Shift Right Arithmetic Immediate
11189 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11190 match(Set dst (RShiftI src1 src2));
11191
11192 ins_cost(INSN_COST);
11193 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11194
11195 ins_encode %{
11196 __ asrw(as_Register($dst$$reg),
11197 as_Register($src1$$reg),
11198 $src2$$constant & 0x1f);
11199 %}
11200
11201 ins_pipe(ialu_reg_shift);
11202 %}
11203
11204 // Combined Int Mask and Right Shift (using UBFM)
11205 // TODO
11206
11207 // Long Shifts
11208
11209 // Shift Left Register
11210 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11211 match(Set dst (LShiftL src1 src2));
11212
11213 ins_cost(INSN_COST * 2);
11214 format %{ "lslv $dst, $src1, $src2" %}
11215
11216 ins_encode %{
11217 __ lslv(as_Register($dst$$reg),
11218 as_Register($src1$$reg),
11219 as_Register($src2$$reg));
11220 %}
11221
11222 ins_pipe(ialu_reg_reg_vshift);
11223 %}
11224
11225 // Shift Left Immediate
11226 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11227 match(Set dst (LShiftL src1 src2));
11228
11229 ins_cost(INSN_COST);
11230 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11231
11232 ins_encode %{
11233 __ lsl(as_Register($dst$$reg),
11234 as_Register($src1$$reg),
11235 $src2$$constant & 0x3f);
11236 %}
11237
11238 ins_pipe(ialu_reg_shift);
11239 %}
11240
11241 // Shift Right Logical Register
11242 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11243 match(Set dst (URShiftL src1 src2));
11244
11245 ins_cost(INSN_COST * 2);
11246 format %{ "lsrv $dst, $src1, $src2" %}
11247
11248 ins_encode %{
11249 __ lsrv(as_Register($dst$$reg),
11250 as_Register($src1$$reg),
11251 as_Register($src2$$reg));
11252 %}
11253
11254 ins_pipe(ialu_reg_reg_vshift);
11255 %}
11256
11257 // Shift Right Logical Immediate
11258 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11259 match(Set dst (URShiftL src1 src2));
11260
11261 ins_cost(INSN_COST);
11262 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11263
11264 ins_encode %{
11265 __ lsr(as_Register($dst$$reg),
11266 as_Register($src1$$reg),
11267 $src2$$constant & 0x3f);
11268 %}
11269
11270 ins_pipe(ialu_reg_shift);
11271 %}
11272
11273 // A special-case pattern for card table stores.
11274 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11275 match(Set dst (URShiftL (CastP2X src1) src2));
11276
11277 ins_cost(INSN_COST);
11278 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11279
11280 ins_encode %{
11281 __ lsr(as_Register($dst$$reg),
11282 as_Register($src1$$reg),
11283 $src2$$constant & 0x3f);
11284 %}
11285
11286 ins_pipe(ialu_reg_shift);
11287 %}
11288
11289 // Shift Right Arithmetic Register
11290 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11291 match(Set dst (RShiftL src1 src2));
11292
11293 ins_cost(INSN_COST * 2);
11294 format %{ "asrv $dst, $src1, $src2" %}
11295
11296 ins_encode %{
11297 __ asrv(as_Register($dst$$reg),
11298 as_Register($src1$$reg),
11299 as_Register($src2$$reg));
11300 %}
11301
11302 ins_pipe(ialu_reg_reg_vshift);
11303 %}
11304
11305 // Shift Right Arithmetic Immediate
11306 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11307 match(Set dst (RShiftL src1 src2));
11308
11309 ins_cost(INSN_COST);
11310 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11311
11312 ins_encode %{
11313 __ asr(as_Register($dst$$reg),
11314 as_Register($src1$$reg),
11315 $src2$$constant & 0x3f);
11316 %}
11317
11318 ins_pipe(ialu_reg_shift);
11319 %}
11320
11321 // BEGIN This section of the file is automatically generated. Do not edit --------------
11322 // This section is generated from aarch64_ad.m4
11323
11324 // This pattern is automatically generated from aarch64_ad.m4
11325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11326 instruct regL_not_reg(iRegLNoSp dst,
11327 iRegL src1, immL_M1 m1,
11328 rFlagsReg cr) %{
11329 match(Set dst (XorL src1 m1));
11330 ins_cost(INSN_COST);
11331 format %{ "eon $dst, $src1, zr" %}
11332
11333 ins_encode %{
11334 __ eon(as_Register($dst$$reg),
11335 as_Register($src1$$reg),
11336 zr,
11337 Assembler::LSL, 0);
11338 %}
11339
11340 ins_pipe(ialu_reg);
11341 %}
11342
11343 // This pattern is automatically generated from aarch64_ad.m4
11344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11345 instruct regI_not_reg(iRegINoSp dst,
11346 iRegIorL2I src1, immI_M1 m1,
11347 rFlagsReg cr) %{
11348 match(Set dst (XorI src1 m1));
11349 ins_cost(INSN_COST);
11350 format %{ "eonw $dst, $src1, zr" %}
11351
11352 ins_encode %{
11353 __ eonw(as_Register($dst$$reg),
11354 as_Register($src1$$reg),
11355 zr,
11356 Assembler::LSL, 0);
11357 %}
11358
11359 ins_pipe(ialu_reg);
11360 %}
11361
11362 // This pattern is automatically generated from aarch64_ad.m4
11363 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11364 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11365 immI0 zero, iRegIorL2I src1, immI src2) %{
11366 match(Set dst (SubI zero (URShiftI src1 src2)));
11367
11368 ins_cost(1.9 * INSN_COST);
11369 format %{ "negw $dst, $src1, LSR $src2" %}
11370
11371 ins_encode %{
11372 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11373 Assembler::LSR, $src2$$constant & 0x1f);
11374 %}
11375
11376 ins_pipe(ialu_reg_shift);
11377 %}
11378
11379 // This pattern is automatically generated from aarch64_ad.m4
11380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11381 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11382 immI0 zero, iRegIorL2I src1, immI src2) %{
11383 match(Set dst (SubI zero (RShiftI src1 src2)));
11384
11385 ins_cost(1.9 * INSN_COST);
11386 format %{ "negw $dst, $src1, ASR $src2" %}
11387
11388 ins_encode %{
11389 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11390 Assembler::ASR, $src2$$constant & 0x1f);
11391 %}
11392
11393 ins_pipe(ialu_reg_shift);
11394 %}
11395
11396 // This pattern is automatically generated from aarch64_ad.m4
11397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11398 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11399 immI0 zero, iRegIorL2I src1, immI src2) %{
11400 match(Set dst (SubI zero (LShiftI src1 src2)));
11401
11402 ins_cost(1.9 * INSN_COST);
11403 format %{ "negw $dst, $src1, LSL $src2" %}
11404
11405 ins_encode %{
11406 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11407 Assembler::LSL, $src2$$constant & 0x1f);
11408 %}
11409
11410 ins_pipe(ialu_reg_shift);
11411 %}
11412
11413 // This pattern is automatically generated from aarch64_ad.m4
11414 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11415 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11416 immL0 zero, iRegL src1, immI src2) %{
11417 match(Set dst (SubL zero (URShiftL src1 src2)));
11418
11419 ins_cost(1.9 * INSN_COST);
11420 format %{ "neg $dst, $src1, LSR $src2" %}
11421
11422 ins_encode %{
11423 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11424 Assembler::LSR, $src2$$constant & 0x3f);
11425 %}
11426
11427 ins_pipe(ialu_reg_shift);
11428 %}
11429
11430 // This pattern is automatically generated from aarch64_ad.m4
11431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11432 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11433 immL0 zero, iRegL src1, immI src2) %{
11434 match(Set dst (SubL zero (RShiftL src1 src2)));
11435
11436 ins_cost(1.9 * INSN_COST);
11437 format %{ "neg $dst, $src1, ASR $src2" %}
11438
11439 ins_encode %{
11440 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11441 Assembler::ASR, $src2$$constant & 0x3f);
11442 %}
11443
11444 ins_pipe(ialu_reg_shift);
11445 %}
11446
11447 // This pattern is automatically generated from aarch64_ad.m4
11448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11449 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11450 immL0 zero, iRegL src1, immI src2) %{
11451 match(Set dst (SubL zero (LShiftL src1 src2)));
11452
11453 ins_cost(1.9 * INSN_COST);
11454 format %{ "neg $dst, $src1, LSL $src2" %}
11455
11456 ins_encode %{
11457 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11458 Assembler::LSL, $src2$$constant & 0x3f);
11459 %}
11460
11461 ins_pipe(ialu_reg_shift);
11462 %}
11463
11464 // This pattern is automatically generated from aarch64_ad.m4
11465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11466 instruct AndI_reg_not_reg(iRegINoSp dst,
11467 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11468 match(Set dst (AndI src1 (XorI src2 m1)));
11469 ins_cost(INSN_COST);
11470 format %{ "bicw $dst, $src1, $src2" %}
11471
11472 ins_encode %{
11473 __ bicw(as_Register($dst$$reg),
11474 as_Register($src1$$reg),
11475 as_Register($src2$$reg),
11476 Assembler::LSL, 0);
11477 %}
11478
11479 ins_pipe(ialu_reg_reg);
11480 %}
11481
11482 // This pattern is automatically generated from aarch64_ad.m4
11483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11484 instruct AndL_reg_not_reg(iRegLNoSp dst,
11485 iRegL src1, iRegL src2, immL_M1 m1) %{
11486 match(Set dst (AndL src1 (XorL src2 m1)));
11487 ins_cost(INSN_COST);
11488 format %{ "bic $dst, $src1, $src2" %}
11489
11490 ins_encode %{
11491 __ bic(as_Register($dst$$reg),
11492 as_Register($src1$$reg),
11493 as_Register($src2$$reg),
11494 Assembler::LSL, 0);
11495 %}
11496
11497 ins_pipe(ialu_reg_reg);
11498 %}
11499
11500 // This pattern is automatically generated from aarch64_ad.m4
11501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11502 instruct OrI_reg_not_reg(iRegINoSp dst,
11503 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11504 match(Set dst (OrI src1 (XorI src2 m1)));
11505 ins_cost(INSN_COST);
11506 format %{ "ornw $dst, $src1, $src2" %}
11507
11508 ins_encode %{
11509 __ ornw(as_Register($dst$$reg),
11510 as_Register($src1$$reg),
11511 as_Register($src2$$reg),
11512 Assembler::LSL, 0);
11513 %}
11514
11515 ins_pipe(ialu_reg_reg);
11516 %}
11517
11518 // This pattern is automatically generated from aarch64_ad.m4
11519 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11520 instruct OrL_reg_not_reg(iRegLNoSp dst,
11521 iRegL src1, iRegL src2, immL_M1 m1) %{
11522 match(Set dst (OrL src1 (XorL src2 m1)));
11523 ins_cost(INSN_COST);
11524 format %{ "orn $dst, $src1, $src2" %}
11525
11526 ins_encode %{
11527 __ orn(as_Register($dst$$reg),
11528 as_Register($src1$$reg),
11529 as_Register($src2$$reg),
11530 Assembler::LSL, 0);
11531 %}
11532
11533 ins_pipe(ialu_reg_reg);
11534 %}
11535
11536 // This pattern is automatically generated from aarch64_ad.m4
11537 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11538 instruct XorI_reg_not_reg(iRegINoSp dst,
11539 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11540 match(Set dst (XorI m1 (XorI src2 src1)));
11541 ins_cost(INSN_COST);
11542 format %{ "eonw $dst, $src1, $src2" %}
11543
11544 ins_encode %{
11545 __ eonw(as_Register($dst$$reg),
11546 as_Register($src1$$reg),
11547 as_Register($src2$$reg),
11548 Assembler::LSL, 0);
11549 %}
11550
11551 ins_pipe(ialu_reg_reg);
11552 %}
11553
11554 // This pattern is automatically generated from aarch64_ad.m4
11555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11556 instruct XorL_reg_not_reg(iRegLNoSp dst,
11557 iRegL src1, iRegL src2, immL_M1 m1) %{
11558 match(Set dst (XorL m1 (XorL src2 src1)));
11559 ins_cost(INSN_COST);
11560 format %{ "eon $dst, $src1, $src2" %}
11561
11562 ins_encode %{
11563 __ eon(as_Register($dst$$reg),
11564 as_Register($src1$$reg),
11565 as_Register($src2$$reg),
11566 Assembler::LSL, 0);
11567 %}
11568
11569 ins_pipe(ialu_reg_reg);
11570 %}
11571
11572 // This pattern is automatically generated from aarch64_ad.m4
11573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11574 // val & (-1 ^ (val >>> shift)) ==> bicw
11575 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11576 iRegIorL2I src1, iRegIorL2I src2,
11577 immI src3, immI_M1 src4) %{
11578 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11579 ins_cost(1.9 * INSN_COST);
11580 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11581
11582 ins_encode %{
11583 __ bicw(as_Register($dst$$reg),
11584 as_Register($src1$$reg),
11585 as_Register($src2$$reg),
11586 Assembler::LSR,
11587 $src3$$constant & 0x1f);
11588 %}
11589
11590 ins_pipe(ialu_reg_reg_shift);
11591 %}
11592
11593 // This pattern is automatically generated from aarch64_ad.m4
11594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11595 // val & (-1 ^ (val >>> shift)) ==> bic
11596 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11597 iRegL src1, iRegL src2,
11598 immI src3, immL_M1 src4) %{
11599 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11600 ins_cost(1.9 * INSN_COST);
11601 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11602
11603 ins_encode %{
11604 __ bic(as_Register($dst$$reg),
11605 as_Register($src1$$reg),
11606 as_Register($src2$$reg),
11607 Assembler::LSR,
11608 $src3$$constant & 0x3f);
11609 %}
11610
11611 ins_pipe(ialu_reg_reg_shift);
11612 %}
11613
11614 // This pattern is automatically generated from aarch64_ad.m4
11615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11616 // val & (-1 ^ (val >> shift)) ==> bicw
11617 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11618 iRegIorL2I src1, iRegIorL2I src2,
11619 immI src3, immI_M1 src4) %{
11620 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11621 ins_cost(1.9 * INSN_COST);
11622 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11623
11624 ins_encode %{
11625 __ bicw(as_Register($dst$$reg),
11626 as_Register($src1$$reg),
11627 as_Register($src2$$reg),
11628 Assembler::ASR,
11629 $src3$$constant & 0x1f);
11630 %}
11631
11632 ins_pipe(ialu_reg_reg_shift);
11633 %}
11634
11635 // This pattern is automatically generated from aarch64_ad.m4
11636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11637 // val & (-1 ^ (val >> shift)) ==> bic
11638 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11639 iRegL src1, iRegL src2,
11640 immI src3, immL_M1 src4) %{
11641 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11642 ins_cost(1.9 * INSN_COST);
11643 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11644
11645 ins_encode %{
11646 __ bic(as_Register($dst$$reg),
11647 as_Register($src1$$reg),
11648 as_Register($src2$$reg),
11649 Assembler::ASR,
11650 $src3$$constant & 0x3f);
11651 %}
11652
11653 ins_pipe(ialu_reg_reg_shift);
11654 %}
11655
11656 // This pattern is automatically generated from aarch64_ad.m4
11657 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11658 // val & (-1 ^ (val ror shift)) ==> bicw
11659 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11660 iRegIorL2I src1, iRegIorL2I src2,
11661 immI src3, immI_M1 src4) %{
11662 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11663 ins_cost(1.9 * INSN_COST);
11664 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11665
11666 ins_encode %{
11667 __ bicw(as_Register($dst$$reg),
11668 as_Register($src1$$reg),
11669 as_Register($src2$$reg),
11670 Assembler::ROR,
11671 $src3$$constant & 0x1f);
11672 %}
11673
11674 ins_pipe(ialu_reg_reg_shift);
11675 %}
11676
11677 // This pattern is automatically generated from aarch64_ad.m4
11678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11679 // val & (-1 ^ (val ror shift)) ==> bic
11680 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11681 iRegL src1, iRegL src2,
11682 immI src3, immL_M1 src4) %{
11683 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11684 ins_cost(1.9 * INSN_COST);
11685 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11686
11687 ins_encode %{
11688 __ bic(as_Register($dst$$reg),
11689 as_Register($src1$$reg),
11690 as_Register($src2$$reg),
11691 Assembler::ROR,
11692 $src3$$constant & 0x3f);
11693 %}
11694
11695 ins_pipe(ialu_reg_reg_shift);
11696 %}
11697
11698 // This pattern is automatically generated from aarch64_ad.m4
11699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11700 // val & (-1 ^ (val << shift)) ==> bicw
11701 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11702 iRegIorL2I src1, iRegIorL2I src2,
11703 immI src3, immI_M1 src4) %{
11704 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11705 ins_cost(1.9 * INSN_COST);
11706 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11707
11708 ins_encode %{
11709 __ bicw(as_Register($dst$$reg),
11710 as_Register($src1$$reg),
11711 as_Register($src2$$reg),
11712 Assembler::LSL,
11713 $src3$$constant & 0x1f);
11714 %}
11715
11716 ins_pipe(ialu_reg_reg_shift);
11717 %}
11718
11719 // This pattern is automatically generated from aarch64_ad.m4
11720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11721 // val & (-1 ^ (val << shift)) ==> bic
11722 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11723 iRegL src1, iRegL src2,
11724 immI src3, immL_M1 src4) %{
11725 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11726 ins_cost(1.9 * INSN_COST);
11727 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11728
11729 ins_encode %{
11730 __ bic(as_Register($dst$$reg),
11731 as_Register($src1$$reg),
11732 as_Register($src2$$reg),
11733 Assembler::LSL,
11734 $src3$$constant & 0x3f);
11735 %}
11736
11737 ins_pipe(ialu_reg_reg_shift);
11738 %}
11739
11740 // This pattern is automatically generated from aarch64_ad.m4
11741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11742 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11743 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11744 iRegIorL2I src1, iRegIorL2I src2,
11745 immI src3, immI_M1 src4) %{
11746 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11747 ins_cost(1.9 * INSN_COST);
11748 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11749
11750 ins_encode %{
11751 __ eonw(as_Register($dst$$reg),
11752 as_Register($src1$$reg),
11753 as_Register($src2$$reg),
11754 Assembler::LSR,
11755 $src3$$constant & 0x1f);
11756 %}
11757
11758 ins_pipe(ialu_reg_reg_shift);
11759 %}
11760
11761 // This pattern is automatically generated from aarch64_ad.m4
11762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11763 // val ^ (-1 ^ (val >>> shift)) ==> eon
11764 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11765 iRegL src1, iRegL src2,
11766 immI src3, immL_M1 src4) %{
11767 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11768 ins_cost(1.9 * INSN_COST);
11769 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11770
11771 ins_encode %{
11772 __ eon(as_Register($dst$$reg),
11773 as_Register($src1$$reg),
11774 as_Register($src2$$reg),
11775 Assembler::LSR,
11776 $src3$$constant & 0x3f);
11777 %}
11778
11779 ins_pipe(ialu_reg_reg_shift);
11780 %}
11781
11782 // This pattern is automatically generated from aarch64_ad.m4
11783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11784 // val ^ (-1 ^ (val >> shift)) ==> eonw
11785 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11786 iRegIorL2I src1, iRegIorL2I src2,
11787 immI src3, immI_M1 src4) %{
11788 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11789 ins_cost(1.9 * INSN_COST);
11790 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11791
11792 ins_encode %{
11793 __ eonw(as_Register($dst$$reg),
11794 as_Register($src1$$reg),
11795 as_Register($src2$$reg),
11796 Assembler::ASR,
11797 $src3$$constant & 0x1f);
11798 %}
11799
11800 ins_pipe(ialu_reg_reg_shift);
11801 %}
11802
11803 // This pattern is automatically generated from aarch64_ad.m4
11804 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11805 // val ^ (-1 ^ (val >> shift)) ==> eon
11806 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11807 iRegL src1, iRegL src2,
11808 immI src3, immL_M1 src4) %{
11809 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11810 ins_cost(1.9 * INSN_COST);
11811 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11812
11813 ins_encode %{
11814 __ eon(as_Register($dst$$reg),
11815 as_Register($src1$$reg),
11816 as_Register($src2$$reg),
11817 Assembler::ASR,
11818 $src3$$constant & 0x3f);
11819 %}
11820
11821 ins_pipe(ialu_reg_reg_shift);
11822 %}
11823
11824 // This pattern is automatically generated from aarch64_ad.m4
11825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11826 // val ^ (-1 ^ (val ror shift)) ==> eonw
11827 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11828 iRegIorL2I src1, iRegIorL2I src2,
11829 immI src3, immI_M1 src4) %{
11830 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11831 ins_cost(1.9 * INSN_COST);
11832 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11833
11834 ins_encode %{
11835 __ eonw(as_Register($dst$$reg),
11836 as_Register($src1$$reg),
11837 as_Register($src2$$reg),
11838 Assembler::ROR,
11839 $src3$$constant & 0x1f);
11840 %}
11841
11842 ins_pipe(ialu_reg_reg_shift);
11843 %}
11844
11845 // This pattern is automatically generated from aarch64_ad.m4
11846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11847 // val ^ (-1 ^ (val ror shift)) ==> eon
11848 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11849 iRegL src1, iRegL src2,
11850 immI src3, immL_M1 src4) %{
11851 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11852 ins_cost(1.9 * INSN_COST);
11853 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11854
11855 ins_encode %{
11856 __ eon(as_Register($dst$$reg),
11857 as_Register($src1$$reg),
11858 as_Register($src2$$reg),
11859 Assembler::ROR,
11860 $src3$$constant & 0x3f);
11861 %}
11862
11863 ins_pipe(ialu_reg_reg_shift);
11864 %}
11865
11866 // This pattern is automatically generated from aarch64_ad.m4
11867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11868 // val ^ (-1 ^ (val << shift)) ==> eonw
11869 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11870 iRegIorL2I src1, iRegIorL2I src2,
11871 immI src3, immI_M1 src4) %{
11872 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11873 ins_cost(1.9 * INSN_COST);
11874 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11875
11876 ins_encode %{
11877 __ eonw(as_Register($dst$$reg),
11878 as_Register($src1$$reg),
11879 as_Register($src2$$reg),
11880 Assembler::LSL,
11881 $src3$$constant & 0x1f);
11882 %}
11883
11884 ins_pipe(ialu_reg_reg_shift);
11885 %}
11886
11887 // This pattern is automatically generated from aarch64_ad.m4
11888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11889 // val ^ (-1 ^ (val << shift)) ==> eon
11890 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11891 iRegL src1, iRegL src2,
11892 immI src3, immL_M1 src4) %{
11893 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11894 ins_cost(1.9 * INSN_COST);
11895 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11896
11897 ins_encode %{
11898 __ eon(as_Register($dst$$reg),
11899 as_Register($src1$$reg),
11900 as_Register($src2$$reg),
11901 Assembler::LSL,
11902 $src3$$constant & 0x3f);
11903 %}
11904
11905 ins_pipe(ialu_reg_reg_shift);
11906 %}
11907
11908 // This pattern is automatically generated from aarch64_ad.m4
11909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11910 // val | (-1 ^ (val >>> shift)) ==> ornw
11911 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11912 iRegIorL2I src1, iRegIorL2I src2,
11913 immI src3, immI_M1 src4) %{
11914 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11915 ins_cost(1.9 * INSN_COST);
11916 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11917
11918 ins_encode %{
11919 __ ornw(as_Register($dst$$reg),
11920 as_Register($src1$$reg),
11921 as_Register($src2$$reg),
11922 Assembler::LSR,
11923 $src3$$constant & 0x1f);
11924 %}
11925
11926 ins_pipe(ialu_reg_reg_shift);
11927 %}
11928
11929 // This pattern is automatically generated from aarch64_ad.m4
11930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11931 // val | (-1 ^ (val >>> shift)) ==> orn
11932 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11933 iRegL src1, iRegL src2,
11934 immI src3, immL_M1 src4) %{
11935 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11936 ins_cost(1.9 * INSN_COST);
11937 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11938
11939 ins_encode %{
11940 __ orn(as_Register($dst$$reg),
11941 as_Register($src1$$reg),
11942 as_Register($src2$$reg),
11943 Assembler::LSR,
11944 $src3$$constant & 0x3f);
11945 %}
11946
11947 ins_pipe(ialu_reg_reg_shift);
11948 %}
11949
11950 // This pattern is automatically generated from aarch64_ad.m4
11951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11952 // val | (-1 ^ (val >> shift)) ==> ornw
11953 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11954 iRegIorL2I src1, iRegIorL2I src2,
11955 immI src3, immI_M1 src4) %{
11956 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11957 ins_cost(1.9 * INSN_COST);
11958 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11959
11960 ins_encode %{
11961 __ ornw(as_Register($dst$$reg),
11962 as_Register($src1$$reg),
11963 as_Register($src2$$reg),
11964 Assembler::ASR,
11965 $src3$$constant & 0x1f);
11966 %}
11967
11968 ins_pipe(ialu_reg_reg_shift);
11969 %}
11970
11971 // This pattern is automatically generated from aarch64_ad.m4
11972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11973 // val | (-1 ^ (val >> shift)) ==> orn
11974 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11975 iRegL src1, iRegL src2,
11976 immI src3, immL_M1 src4) %{
11977 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11978 ins_cost(1.9 * INSN_COST);
11979 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11980
11981 ins_encode %{
11982 __ orn(as_Register($dst$$reg),
11983 as_Register($src1$$reg),
11984 as_Register($src2$$reg),
11985 Assembler::ASR,
11986 $src3$$constant & 0x3f);
11987 %}
11988
11989 ins_pipe(ialu_reg_reg_shift);
11990 %}
11991
11992 // This pattern is automatically generated from aarch64_ad.m4
11993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11994 // val | (-1 ^ (val ror shift)) ==> ornw
11995 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11996 iRegIorL2I src1, iRegIorL2I src2,
11997 immI src3, immI_M1 src4) %{
11998 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11999 ins_cost(1.9 * INSN_COST);
12000 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
12001
12002 ins_encode %{
12003 __ ornw(as_Register($dst$$reg),
12004 as_Register($src1$$reg),
12005 as_Register($src2$$reg),
12006 Assembler::ROR,
12007 $src3$$constant & 0x1f);
12008 %}
12009
12010 ins_pipe(ialu_reg_reg_shift);
12011 %}
12012
12013 // This pattern is automatically generated from aarch64_ad.m4
12014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12015 // val | (-1 ^ (val ror shift)) ==> orn
12016 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12017 iRegL src1, iRegL src2,
12018 immI src3, immL_M1 src4) %{
12019 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12020 ins_cost(1.9 * INSN_COST);
12021 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
12022
12023 ins_encode %{
12024 __ orn(as_Register($dst$$reg),
12025 as_Register($src1$$reg),
12026 as_Register($src2$$reg),
12027 Assembler::ROR,
12028 $src3$$constant & 0x3f);
12029 %}
12030
12031 ins_pipe(ialu_reg_reg_shift);
12032 %}
12033
12034 // This pattern is automatically generated from aarch64_ad.m4
12035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12036 // val | (-1 ^ (val << shift)) ==> ornw
12037 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12038 iRegIorL2I src1, iRegIorL2I src2,
12039 immI src3, immI_M1 src4) %{
12040 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12041 ins_cost(1.9 * INSN_COST);
12042 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
12043
12044 ins_encode %{
12045 __ ornw(as_Register($dst$$reg),
12046 as_Register($src1$$reg),
12047 as_Register($src2$$reg),
12048 Assembler::LSL,
12049 $src3$$constant & 0x1f);
12050 %}
12051
12052 ins_pipe(ialu_reg_reg_shift);
12053 %}
12054
12055 // This pattern is automatically generated from aarch64_ad.m4
12056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12057 // val | (-1 ^ (val << shift)) ==> orn
12058 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12059 iRegL src1, iRegL src2,
12060 immI src3, immL_M1 src4) %{
12061 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12062 ins_cost(1.9 * INSN_COST);
12063 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
12064
12065 ins_encode %{
12066 __ orn(as_Register($dst$$reg),
12067 as_Register($src1$$reg),
12068 as_Register($src2$$reg),
12069 Assembler::LSL,
12070 $src3$$constant & 0x3f);
12071 %}
12072
12073 ins_pipe(ialu_reg_reg_shift);
12074 %}
12075
12076 // This pattern is automatically generated from aarch64_ad.m4
12077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12078 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12079 iRegIorL2I src1, iRegIorL2I src2,
12080 immI src3) %{
12081 match(Set dst (AndI src1 (URShiftI src2 src3)));
12082
12083 ins_cost(1.9 * INSN_COST);
12084 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
12085
12086 ins_encode %{
12087 __ andw(as_Register($dst$$reg),
12088 as_Register($src1$$reg),
12089 as_Register($src2$$reg),
12090 Assembler::LSR,
12091 $src3$$constant & 0x1f);
12092 %}
12093
12094 ins_pipe(ialu_reg_reg_shift);
12095 %}
12096
12097 // This pattern is automatically generated from aarch64_ad.m4
12098 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12099 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12100 iRegL src1, iRegL src2,
12101 immI src3) %{
12102 match(Set dst (AndL src1 (URShiftL src2 src3)));
12103
12104 ins_cost(1.9 * INSN_COST);
12105 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
12106
12107 ins_encode %{
12108 __ andr(as_Register($dst$$reg),
12109 as_Register($src1$$reg),
12110 as_Register($src2$$reg),
12111 Assembler::LSR,
12112 $src3$$constant & 0x3f);
12113 %}
12114
12115 ins_pipe(ialu_reg_reg_shift);
12116 %}
12117
12118 // This pattern is automatically generated from aarch64_ad.m4
12119 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12120 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12121 iRegIorL2I src1, iRegIorL2I src2,
12122 immI src3) %{
12123 match(Set dst (AndI src1 (RShiftI src2 src3)));
12124
12125 ins_cost(1.9 * INSN_COST);
12126 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
12127
12128 ins_encode %{
12129 __ andw(as_Register($dst$$reg),
12130 as_Register($src1$$reg),
12131 as_Register($src2$$reg),
12132 Assembler::ASR,
12133 $src3$$constant & 0x1f);
12134 %}
12135
12136 ins_pipe(ialu_reg_reg_shift);
12137 %}
12138
12139 // This pattern is automatically generated from aarch64_ad.m4
12140 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12141 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12142 iRegL src1, iRegL src2,
12143 immI src3) %{
12144 match(Set dst (AndL src1 (RShiftL src2 src3)));
12145
12146 ins_cost(1.9 * INSN_COST);
12147 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
12148
12149 ins_encode %{
12150 __ andr(as_Register($dst$$reg),
12151 as_Register($src1$$reg),
12152 as_Register($src2$$reg),
12153 Assembler::ASR,
12154 $src3$$constant & 0x3f);
12155 %}
12156
12157 ins_pipe(ialu_reg_reg_shift);
12158 %}
12159
12160 // This pattern is automatically generated from aarch64_ad.m4
12161 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12162 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12163 iRegIorL2I src1, iRegIorL2I src2,
12164 immI src3) %{
12165 match(Set dst (AndI src1 (LShiftI src2 src3)));
12166
12167 ins_cost(1.9 * INSN_COST);
12168 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
12169
12170 ins_encode %{
12171 __ andw(as_Register($dst$$reg),
12172 as_Register($src1$$reg),
12173 as_Register($src2$$reg),
12174 Assembler::LSL,
12175 $src3$$constant & 0x1f);
12176 %}
12177
12178 ins_pipe(ialu_reg_reg_shift);
12179 %}
12180
12181 // This pattern is automatically generated from aarch64_ad.m4
12182 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12183 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12184 iRegL src1, iRegL src2,
12185 immI src3) %{
12186 match(Set dst (AndL src1 (LShiftL src2 src3)));
12187
12188 ins_cost(1.9 * INSN_COST);
12189 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
12190
12191 ins_encode %{
12192 __ andr(as_Register($dst$$reg),
12193 as_Register($src1$$reg),
12194 as_Register($src2$$reg),
12195 Assembler::LSL,
12196 $src3$$constant & 0x3f);
12197 %}
12198
12199 ins_pipe(ialu_reg_reg_shift);
12200 %}
12201
12202 // This pattern is automatically generated from aarch64_ad.m4
12203 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12204 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12205 iRegIorL2I src1, iRegIorL2I src2,
12206 immI src3) %{
12207 match(Set dst (AndI src1 (RotateRight src2 src3)));
12208
12209 ins_cost(1.9 * INSN_COST);
12210 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
12211
12212 ins_encode %{
12213 __ andw(as_Register($dst$$reg),
12214 as_Register($src1$$reg),
12215 as_Register($src2$$reg),
12216 Assembler::ROR,
12217 $src3$$constant & 0x1f);
12218 %}
12219
12220 ins_pipe(ialu_reg_reg_shift);
12221 %}
12222
12223 // This pattern is automatically generated from aarch64_ad.m4
12224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12225 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12226 iRegL src1, iRegL src2,
12227 immI src3) %{
12228 match(Set dst (AndL src1 (RotateRight src2 src3)));
12229
12230 ins_cost(1.9 * INSN_COST);
12231 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
12232
12233 ins_encode %{
12234 __ andr(as_Register($dst$$reg),
12235 as_Register($src1$$reg),
12236 as_Register($src2$$reg),
12237 Assembler::ROR,
12238 $src3$$constant & 0x3f);
12239 %}
12240
12241 ins_pipe(ialu_reg_reg_shift);
12242 %}
12243
12244 // This pattern is automatically generated from aarch64_ad.m4
12245 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12246 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12247 iRegIorL2I src1, iRegIorL2I src2,
12248 immI src3) %{
12249 match(Set dst (XorI src1 (URShiftI src2 src3)));
12250
12251 ins_cost(1.9 * INSN_COST);
12252 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
12253
12254 ins_encode %{
12255 __ eorw(as_Register($dst$$reg),
12256 as_Register($src1$$reg),
12257 as_Register($src2$$reg),
12258 Assembler::LSR,
12259 $src3$$constant & 0x1f);
12260 %}
12261
12262 ins_pipe(ialu_reg_reg_shift);
12263 %}
12264
12265 // This pattern is automatically generated from aarch64_ad.m4
12266 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12267 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12268 iRegL src1, iRegL src2,
12269 immI src3) %{
12270 match(Set dst (XorL src1 (URShiftL src2 src3)));
12271
12272 ins_cost(1.9 * INSN_COST);
12273 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
12274
12275 ins_encode %{
12276 __ eor(as_Register($dst$$reg),
12277 as_Register($src1$$reg),
12278 as_Register($src2$$reg),
12279 Assembler::LSR,
12280 $src3$$constant & 0x3f);
12281 %}
12282
12283 ins_pipe(ialu_reg_reg_shift);
12284 %}
12285
12286 // This pattern is automatically generated from aarch64_ad.m4
12287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12288 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12289 iRegIorL2I src1, iRegIorL2I src2,
12290 immI src3) %{
12291 match(Set dst (XorI src1 (RShiftI src2 src3)));
12292
12293 ins_cost(1.9 * INSN_COST);
12294 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
12295
12296 ins_encode %{
12297 __ eorw(as_Register($dst$$reg),
12298 as_Register($src1$$reg),
12299 as_Register($src2$$reg),
12300 Assembler::ASR,
12301 $src3$$constant & 0x1f);
12302 %}
12303
12304 ins_pipe(ialu_reg_reg_shift);
12305 %}
12306
12307 // This pattern is automatically generated from aarch64_ad.m4
12308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12309 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12310 iRegL src1, iRegL src2,
12311 immI src3) %{
12312 match(Set dst (XorL src1 (RShiftL src2 src3)));
12313
12314 ins_cost(1.9 * INSN_COST);
12315 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
12316
12317 ins_encode %{
12318 __ eor(as_Register($dst$$reg),
12319 as_Register($src1$$reg),
12320 as_Register($src2$$reg),
12321 Assembler::ASR,
12322 $src3$$constant & 0x3f);
12323 %}
12324
12325 ins_pipe(ialu_reg_reg_shift);
12326 %}
12327
12328 // This pattern is automatically generated from aarch64_ad.m4
12329 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12330 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12331 iRegIorL2I src1, iRegIorL2I src2,
12332 immI src3) %{
12333 match(Set dst (XorI src1 (LShiftI src2 src3)));
12334
12335 ins_cost(1.9 * INSN_COST);
12336 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
12337
12338 ins_encode %{
12339 __ eorw(as_Register($dst$$reg),
12340 as_Register($src1$$reg),
12341 as_Register($src2$$reg),
12342 Assembler::LSL,
12343 $src3$$constant & 0x1f);
12344 %}
12345
12346 ins_pipe(ialu_reg_reg_shift);
12347 %}
12348
12349 // This pattern is automatically generated from aarch64_ad.m4
12350 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12351 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12352 iRegL src1, iRegL src2,
12353 immI src3) %{
12354 match(Set dst (XorL src1 (LShiftL src2 src3)));
12355
12356 ins_cost(1.9 * INSN_COST);
12357 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
12358
12359 ins_encode %{
12360 __ eor(as_Register($dst$$reg),
12361 as_Register($src1$$reg),
12362 as_Register($src2$$reg),
12363 Assembler::LSL,
12364 $src3$$constant & 0x3f);
12365 %}
12366
12367 ins_pipe(ialu_reg_reg_shift);
12368 %}
12369
12370 // This pattern is automatically generated from aarch64_ad.m4
12371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12372 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12373 iRegIorL2I src1, iRegIorL2I src2,
12374 immI src3) %{
12375 match(Set dst (XorI src1 (RotateRight src2 src3)));
12376
12377 ins_cost(1.9 * INSN_COST);
12378 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
12379
12380 ins_encode %{
12381 __ eorw(as_Register($dst$$reg),
12382 as_Register($src1$$reg),
12383 as_Register($src2$$reg),
12384 Assembler::ROR,
12385 $src3$$constant & 0x1f);
12386 %}
12387
12388 ins_pipe(ialu_reg_reg_shift);
12389 %}
12390
12391 // This pattern is automatically generated from aarch64_ad.m4
12392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12393 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12394 iRegL src1, iRegL src2,
12395 immI src3) %{
12396 match(Set dst (XorL src1 (RotateRight src2 src3)));
12397
12398 ins_cost(1.9 * INSN_COST);
12399 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
12400
12401 ins_encode %{
12402 __ eor(as_Register($dst$$reg),
12403 as_Register($src1$$reg),
12404 as_Register($src2$$reg),
12405 Assembler::ROR,
12406 $src3$$constant & 0x3f);
12407 %}
12408
12409 ins_pipe(ialu_reg_reg_shift);
12410 %}
12411
12412 // This pattern is automatically generated from aarch64_ad.m4
12413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12414 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12415 iRegIorL2I src1, iRegIorL2I src2,
12416 immI src3) %{
12417 match(Set dst (OrI src1 (URShiftI src2 src3)));
12418
12419 ins_cost(1.9 * INSN_COST);
12420 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
12421
12422 ins_encode %{
12423 __ orrw(as_Register($dst$$reg),
12424 as_Register($src1$$reg),
12425 as_Register($src2$$reg),
12426 Assembler::LSR,
12427 $src3$$constant & 0x1f);
12428 %}
12429
12430 ins_pipe(ialu_reg_reg_shift);
12431 %}
12432
12433 // This pattern is automatically generated from aarch64_ad.m4
12434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12435 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12436 iRegL src1, iRegL src2,
12437 immI src3) %{
12438 match(Set dst (OrL src1 (URShiftL src2 src3)));
12439
12440 ins_cost(1.9 * INSN_COST);
12441 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12442
12443 ins_encode %{
12444 __ orr(as_Register($dst$$reg),
12445 as_Register($src1$$reg),
12446 as_Register($src2$$reg),
12447 Assembler::LSR,
12448 $src3$$constant & 0x3f);
12449 %}
12450
12451 ins_pipe(ialu_reg_reg_shift);
12452 %}
12453
12454 // This pattern is automatically generated from aarch64_ad.m4
12455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12456 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12457 iRegIorL2I src1, iRegIorL2I src2,
12458 immI src3) %{
12459 match(Set dst (OrI src1 (RShiftI src2 src3)));
12460
12461 ins_cost(1.9 * INSN_COST);
12462 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12463
12464 ins_encode %{
12465 __ orrw(as_Register($dst$$reg),
12466 as_Register($src1$$reg),
12467 as_Register($src2$$reg),
12468 Assembler::ASR,
12469 $src3$$constant & 0x1f);
12470 %}
12471
12472 ins_pipe(ialu_reg_reg_shift);
12473 %}
12474
12475 // This pattern is automatically generated from aarch64_ad.m4
12476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12477 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12478 iRegL src1, iRegL src2,
12479 immI src3) %{
12480 match(Set dst (OrL src1 (RShiftL src2 src3)));
12481
12482 ins_cost(1.9 * INSN_COST);
12483 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12484
12485 ins_encode %{
12486 __ orr(as_Register($dst$$reg),
12487 as_Register($src1$$reg),
12488 as_Register($src2$$reg),
12489 Assembler::ASR,
12490 $src3$$constant & 0x3f);
12491 %}
12492
12493 ins_pipe(ialu_reg_reg_shift);
12494 %}
12495
12496 // This pattern is automatically generated from aarch64_ad.m4
12497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12498 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12499 iRegIorL2I src1, iRegIorL2I src2,
12500 immI src3) %{
12501 match(Set dst (OrI src1 (LShiftI src2 src3)));
12502
12503 ins_cost(1.9 * INSN_COST);
12504 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12505
12506 ins_encode %{
12507 __ orrw(as_Register($dst$$reg),
12508 as_Register($src1$$reg),
12509 as_Register($src2$$reg),
12510 Assembler::LSL,
12511 $src3$$constant & 0x1f);
12512 %}
12513
12514 ins_pipe(ialu_reg_reg_shift);
12515 %}
12516
12517 // This pattern is automatically generated from aarch64_ad.m4
12518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12519 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12520 iRegL src1, iRegL src2,
12521 immI src3) %{
12522 match(Set dst (OrL src1 (LShiftL src2 src3)));
12523
12524 ins_cost(1.9 * INSN_COST);
12525 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12526
12527 ins_encode %{
12528 __ orr(as_Register($dst$$reg),
12529 as_Register($src1$$reg),
12530 as_Register($src2$$reg),
12531 Assembler::LSL,
12532 $src3$$constant & 0x3f);
12533 %}
12534
12535 ins_pipe(ialu_reg_reg_shift);
12536 %}
12537
12538 // This pattern is automatically generated from aarch64_ad.m4
12539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12540 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12541 iRegIorL2I src1, iRegIorL2I src2,
12542 immI src3) %{
12543 match(Set dst (OrI src1 (RotateRight src2 src3)));
12544
12545 ins_cost(1.9 * INSN_COST);
12546 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12547
12548 ins_encode %{
12549 __ orrw(as_Register($dst$$reg),
12550 as_Register($src1$$reg),
12551 as_Register($src2$$reg),
12552 Assembler::ROR,
12553 $src3$$constant & 0x1f);
12554 %}
12555
12556 ins_pipe(ialu_reg_reg_shift);
12557 %}
12558
12559 // This pattern is automatically generated from aarch64_ad.m4
12560 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12561 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12562 iRegL src1, iRegL src2,
12563 immI src3) %{
12564 match(Set dst (OrL src1 (RotateRight src2 src3)));
12565
12566 ins_cost(1.9 * INSN_COST);
12567 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12568
12569 ins_encode %{
12570 __ orr(as_Register($dst$$reg),
12571 as_Register($src1$$reg),
12572 as_Register($src2$$reg),
12573 Assembler::ROR,
12574 $src3$$constant & 0x3f);
12575 %}
12576
12577 ins_pipe(ialu_reg_reg_shift);
12578 %}
12579
12580 // This pattern is automatically generated from aarch64_ad.m4
12581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12582 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12583 iRegIorL2I src1, iRegIorL2I src2,
12584 immI src3) %{
12585 match(Set dst (AddI src1 (URShiftI src2 src3)));
12586
12587 ins_cost(1.9 * INSN_COST);
12588 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12589
12590 ins_encode %{
12591 __ addw(as_Register($dst$$reg),
12592 as_Register($src1$$reg),
12593 as_Register($src2$$reg),
12594 Assembler::LSR,
12595 $src3$$constant & 0x1f);
12596 %}
12597
12598 ins_pipe(ialu_reg_reg_shift);
12599 %}
12600
12601 // This pattern is automatically generated from aarch64_ad.m4
12602 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12603 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12604 iRegL src1, iRegL src2,
12605 immI src3) %{
12606 match(Set dst (AddL src1 (URShiftL src2 src3)));
12607
12608 ins_cost(1.9 * INSN_COST);
12609 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12610
12611 ins_encode %{
12612 __ add(as_Register($dst$$reg),
12613 as_Register($src1$$reg),
12614 as_Register($src2$$reg),
12615 Assembler::LSR,
12616 $src3$$constant & 0x3f);
12617 %}
12618
12619 ins_pipe(ialu_reg_reg_shift);
12620 %}
12621
12622 // This pattern is automatically generated from aarch64_ad.m4
12623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12624 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12625 iRegIorL2I src1, iRegIorL2I src2,
12626 immI src3) %{
12627 match(Set dst (AddI src1 (RShiftI src2 src3)));
12628
12629 ins_cost(1.9 * INSN_COST);
12630 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12631
12632 ins_encode %{
12633 __ addw(as_Register($dst$$reg),
12634 as_Register($src1$$reg),
12635 as_Register($src2$$reg),
12636 Assembler::ASR,
12637 $src3$$constant & 0x1f);
12638 %}
12639
12640 ins_pipe(ialu_reg_reg_shift);
12641 %}
12642
12643 // This pattern is automatically generated from aarch64_ad.m4
12644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12645 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12646 iRegL src1, iRegL src2,
12647 immI src3) %{
12648 match(Set dst (AddL src1 (RShiftL src2 src3)));
12649
12650 ins_cost(1.9 * INSN_COST);
12651 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12652
12653 ins_encode %{
12654 __ add(as_Register($dst$$reg),
12655 as_Register($src1$$reg),
12656 as_Register($src2$$reg),
12657 Assembler::ASR,
12658 $src3$$constant & 0x3f);
12659 %}
12660
12661 ins_pipe(ialu_reg_reg_shift);
12662 %}
12663
12664 // This pattern is automatically generated from aarch64_ad.m4
12665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12666 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12667 iRegIorL2I src1, iRegIorL2I src2,
12668 immI src3) %{
12669 match(Set dst (AddI src1 (LShiftI src2 src3)));
12670
12671 ins_cost(1.9 * INSN_COST);
12672 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12673
12674 ins_encode %{
12675 __ addw(as_Register($dst$$reg),
12676 as_Register($src1$$reg),
12677 as_Register($src2$$reg),
12678 Assembler::LSL,
12679 $src3$$constant & 0x1f);
12680 %}
12681
12682 ins_pipe(ialu_reg_reg_shift);
12683 %}
12684
12685 // This pattern is automatically generated from aarch64_ad.m4
12686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12687 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12688 iRegL src1, iRegL src2,
12689 immI src3) %{
12690 match(Set dst (AddL src1 (LShiftL src2 src3)));
12691
12692 ins_cost(1.9 * INSN_COST);
12693 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12694
12695 ins_encode %{
12696 __ add(as_Register($dst$$reg),
12697 as_Register($src1$$reg),
12698 as_Register($src2$$reg),
12699 Assembler::LSL,
12700 $src3$$constant & 0x3f);
12701 %}
12702
12703 ins_pipe(ialu_reg_reg_shift);
12704 %}
12705
12706 // This pattern is automatically generated from aarch64_ad.m4
12707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12708 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12709 iRegIorL2I src1, iRegIorL2I src2,
12710 immI src3) %{
12711 match(Set dst (SubI src1 (URShiftI src2 src3)));
12712
12713 ins_cost(1.9 * INSN_COST);
12714 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12715
12716 ins_encode %{
12717 __ subw(as_Register($dst$$reg),
12718 as_Register($src1$$reg),
12719 as_Register($src2$$reg),
12720 Assembler::LSR,
12721 $src3$$constant & 0x1f);
12722 %}
12723
12724 ins_pipe(ialu_reg_reg_shift);
12725 %}
12726
12727 // This pattern is automatically generated from aarch64_ad.m4
12728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12729 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12730 iRegL src1, iRegL src2,
12731 immI src3) %{
12732 match(Set dst (SubL src1 (URShiftL src2 src3)));
12733
12734 ins_cost(1.9 * INSN_COST);
12735 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12736
12737 ins_encode %{
12738 __ sub(as_Register($dst$$reg),
12739 as_Register($src1$$reg),
12740 as_Register($src2$$reg),
12741 Assembler::LSR,
12742 $src3$$constant & 0x3f);
12743 %}
12744
12745 ins_pipe(ialu_reg_reg_shift);
12746 %}
12747
12748 // This pattern is automatically generated from aarch64_ad.m4
12749 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12750 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12751 iRegIorL2I src1, iRegIorL2I src2,
12752 immI src3) %{
12753 match(Set dst (SubI src1 (RShiftI src2 src3)));
12754
12755 ins_cost(1.9 * INSN_COST);
12756 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12757
12758 ins_encode %{
12759 __ subw(as_Register($dst$$reg),
12760 as_Register($src1$$reg),
12761 as_Register($src2$$reg),
12762 Assembler::ASR,
12763 $src3$$constant & 0x1f);
12764 %}
12765
12766 ins_pipe(ialu_reg_reg_shift);
12767 %}
12768
12769 // This pattern is automatically generated from aarch64_ad.m4
12770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12771 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12772 iRegL src1, iRegL src2,
12773 immI src3) %{
12774 match(Set dst (SubL src1 (RShiftL src2 src3)));
12775
12776 ins_cost(1.9 * INSN_COST);
12777 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12778
12779 ins_encode %{
12780 __ sub(as_Register($dst$$reg),
12781 as_Register($src1$$reg),
12782 as_Register($src2$$reg),
12783 Assembler::ASR,
12784 $src3$$constant & 0x3f);
12785 %}
12786
12787 ins_pipe(ialu_reg_reg_shift);
12788 %}
12789
12790 // This pattern is automatically generated from aarch64_ad.m4
12791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12792 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12793 iRegIorL2I src1, iRegIorL2I src2,
12794 immI src3) %{
12795 match(Set dst (SubI src1 (LShiftI src2 src3)));
12796
12797 ins_cost(1.9 * INSN_COST);
12798 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12799
12800 ins_encode %{
12801 __ subw(as_Register($dst$$reg),
12802 as_Register($src1$$reg),
12803 as_Register($src2$$reg),
12804 Assembler::LSL,
12805 $src3$$constant & 0x1f);
12806 %}
12807
12808 ins_pipe(ialu_reg_reg_shift);
12809 %}
12810
12811 // This pattern is automatically generated from aarch64_ad.m4
12812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12813 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12814 iRegL src1, iRegL src2,
12815 immI src3) %{
12816 match(Set dst (SubL src1 (LShiftL src2 src3)));
12817
12818 ins_cost(1.9 * INSN_COST);
12819 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12820
12821 ins_encode %{
12822 __ sub(as_Register($dst$$reg),
12823 as_Register($src1$$reg),
12824 as_Register($src2$$reg),
12825 Assembler::LSL,
12826 $src3$$constant & 0x3f);
12827 %}
12828
12829 ins_pipe(ialu_reg_reg_shift);
12830 %}
12831
12832 // This pattern is automatically generated from aarch64_ad.m4
12833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12834
12835 // Shift Left followed by Shift Right.
12836 // This idiom is used by the compiler for the i2b bytecode etc.
12837 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12838 %{
12839 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12840 ins_cost(INSN_COST * 2);
12841 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12842 ins_encode %{
12843 int lshift = $lshift_count$$constant & 63;
12844 int rshift = $rshift_count$$constant & 63;
12845 int s = 63 - lshift;
12846 int r = (rshift - lshift) & 63;
12847 __ sbfm(as_Register($dst$$reg),
12848 as_Register($src$$reg),
12849 r, s);
12850 %}
12851
12852 ins_pipe(ialu_reg_shift);
12853 %}
12854
12855 // This pattern is automatically generated from aarch64_ad.m4
12856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12857
12858 // Shift Left followed by Shift Right.
12859 // This idiom is used by the compiler for the i2b bytecode etc.
12860 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12861 %{
12862 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12863 ins_cost(INSN_COST * 2);
12864 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12865 ins_encode %{
12866 int lshift = $lshift_count$$constant & 31;
12867 int rshift = $rshift_count$$constant & 31;
12868 int s = 31 - lshift;
12869 int r = (rshift - lshift) & 31;
12870 __ sbfmw(as_Register($dst$$reg),
12871 as_Register($src$$reg),
12872 r, s);
12873 %}
12874
12875 ins_pipe(ialu_reg_shift);
12876 %}
12877
12878 // This pattern is automatically generated from aarch64_ad.m4
12879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12880
12881 // Shift Left followed by Shift Right.
12882 // This idiom is used by the compiler for the i2b bytecode etc.
12883 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12884 %{
12885 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12886 ins_cost(INSN_COST * 2);
12887 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12888 ins_encode %{
12889 int lshift = $lshift_count$$constant & 63;
12890 int rshift = $rshift_count$$constant & 63;
12891 int s = 63 - lshift;
12892 int r = (rshift - lshift) & 63;
12893 __ ubfm(as_Register($dst$$reg),
12894 as_Register($src$$reg),
12895 r, s);
12896 %}
12897
12898 ins_pipe(ialu_reg_shift);
12899 %}
12900
12901 // This pattern is automatically generated from aarch64_ad.m4
12902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12903
12904 // Shift Left followed by Shift Right.
12905 // This idiom is used by the compiler for the i2b bytecode etc.
12906 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12907 %{
12908 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12909 ins_cost(INSN_COST * 2);
12910 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12911 ins_encode %{
12912 int lshift = $lshift_count$$constant & 31;
12913 int rshift = $rshift_count$$constant & 31;
12914 int s = 31 - lshift;
12915 int r = (rshift - lshift) & 31;
12916 __ ubfmw(as_Register($dst$$reg),
12917 as_Register($src$$reg),
12918 r, s);
12919 %}
12920
12921 ins_pipe(ialu_reg_shift);
12922 %}
12923
12924 // Bitfield extract with shift & mask
12925
12926 // This pattern is automatically generated from aarch64_ad.m4
12927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12928 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12929 %{
12930 match(Set dst (AndI (URShiftI src rshift) mask));
12931 // Make sure we are not going to exceed what ubfxw can do.
12932 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12933
12934 ins_cost(INSN_COST);
12935 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12936 ins_encode %{
12937 int rshift = $rshift$$constant & 31;
12938 intptr_t mask = $mask$$constant;
12939 int width = exact_log2(mask+1);
12940 __ ubfxw(as_Register($dst$$reg),
12941 as_Register($src$$reg), rshift, width);
12942 %}
12943 ins_pipe(ialu_reg_shift);
12944 %}
12945
12946 // This pattern is automatically generated from aarch64_ad.m4
12947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12948 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12949 %{
12950 match(Set dst (AndL (URShiftL src rshift) mask));
12951 // Make sure we are not going to exceed what ubfx can do.
12952 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12953
12954 ins_cost(INSN_COST);
12955 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12956 ins_encode %{
12957 int rshift = $rshift$$constant & 63;
12958 intptr_t mask = $mask$$constant;
12959 int width = exact_log2_long(mask+1);
12960 __ ubfx(as_Register($dst$$reg),
12961 as_Register($src$$reg), rshift, width);
12962 %}
12963 ins_pipe(ialu_reg_shift);
12964 %}
12965
12966
12967 // This pattern is automatically generated from aarch64_ad.m4
12968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12969
12970 // We can use ubfx when extending an And with a mask when we know mask
12971 // is positive. We know that because immI_bitmask guarantees it.
12972 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12973 %{
12974 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12975 // Make sure we are not going to exceed what ubfxw can do.
12976 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12977
12978 ins_cost(INSN_COST * 2);
12979 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12980 ins_encode %{
12981 int rshift = $rshift$$constant & 31;
12982 intptr_t mask = $mask$$constant;
12983 int width = exact_log2(mask+1);
12984 __ ubfx(as_Register($dst$$reg),
12985 as_Register($src$$reg), rshift, width);
12986 %}
12987 ins_pipe(ialu_reg_shift);
12988 %}
12989
12990
12991 // This pattern is automatically generated from aarch64_ad.m4
12992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12993
12994 // We can use ubfiz when masking by a positive number and then left shifting the result.
12995 // We know that the mask is positive because immI_bitmask guarantees it.
12996 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12997 %{
12998 match(Set dst (LShiftI (AndI src mask) lshift));
12999 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
13000
13001 ins_cost(INSN_COST);
13002 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13003 ins_encode %{
13004 int lshift = $lshift$$constant & 31;
13005 intptr_t mask = $mask$$constant;
13006 int width = exact_log2(mask+1);
13007 __ ubfizw(as_Register($dst$$reg),
13008 as_Register($src$$reg), lshift, width);
13009 %}
13010 ins_pipe(ialu_reg_shift);
13011 %}
13012
13013 // This pattern is automatically generated from aarch64_ad.m4
13014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13015
13016 // We can use ubfiz when masking by a positive number and then left shifting the result.
13017 // We know that the mask is positive because immL_bitmask guarantees it.
13018 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13019 %{
13020 match(Set dst (LShiftL (AndL src mask) lshift));
13021 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13022
13023 ins_cost(INSN_COST);
13024 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13025 ins_encode %{
13026 int lshift = $lshift$$constant & 63;
13027 intptr_t mask = $mask$$constant;
13028 int width = exact_log2_long(mask+1);
13029 __ ubfiz(as_Register($dst$$reg),
13030 as_Register($src$$reg), lshift, width);
13031 %}
13032 ins_pipe(ialu_reg_shift);
13033 %}
13034
13035 // This pattern is automatically generated from aarch64_ad.m4
13036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13037
13038 // We can use ubfiz when masking by a positive number and then left shifting the result.
13039 // We know that the mask is positive because immI_bitmask guarantees it.
13040 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13041 %{
13042 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13043 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13044
13045 ins_cost(INSN_COST);
13046 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13047 ins_encode %{
13048 int lshift = $lshift$$constant & 31;
13049 intptr_t mask = $mask$$constant;
13050 int width = exact_log2(mask+1);
13051 __ ubfizw(as_Register($dst$$reg),
13052 as_Register($src$$reg), lshift, width);
13053 %}
13054 ins_pipe(ialu_reg_shift);
13055 %}
13056
13057 // This pattern is automatically generated from aarch64_ad.m4
13058 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13059
13060 // We can use ubfiz when masking by a positive number and then left shifting the result.
13061 // We know that the mask is positive because immL_bitmask guarantees it.
13062 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13063 %{
13064 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13065 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13066
13067 ins_cost(INSN_COST);
13068 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13069 ins_encode %{
13070 int lshift = $lshift$$constant & 63;
13071 intptr_t mask = $mask$$constant;
13072 int width = exact_log2_long(mask+1);
13073 __ ubfiz(as_Register($dst$$reg),
13074 as_Register($src$$reg), lshift, width);
13075 %}
13076 ins_pipe(ialu_reg_shift);
13077 %}
13078
13079
13080 // This pattern is automatically generated from aarch64_ad.m4
13081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13082
13083 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13084 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13085 %{
13086 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13087 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13088
13089 ins_cost(INSN_COST);
13090 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13091 ins_encode %{
13092 int lshift = $lshift$$constant & 63;
13093 intptr_t mask = $mask$$constant;
13094 int width = exact_log2(mask+1);
13095 __ ubfiz(as_Register($dst$$reg),
13096 as_Register($src$$reg), lshift, width);
13097 %}
13098 ins_pipe(ialu_reg_shift);
13099 %}
13100
13101 // This pattern is automatically generated from aarch64_ad.m4
13102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13103
13104 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13105 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13106 %{
13107 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13108 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13109
13110 ins_cost(INSN_COST);
13111 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13112 ins_encode %{
13113 int lshift = $lshift$$constant & 31;
13114 intptr_t mask = $mask$$constant;
13115 int width = exact_log2(mask+1);
13116 __ ubfiz(as_Register($dst$$reg),
13117 as_Register($src$$reg), lshift, width);
13118 %}
13119 ins_pipe(ialu_reg_shift);
13120 %}
13121
13122 // This pattern is automatically generated from aarch64_ad.m4
13123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13124
13125 // Can skip int2long conversions after AND with small bitmask
13126 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13127 %{
13128 match(Set dst (ConvI2L (AndI src msk)));
13129 ins_cost(INSN_COST);
13130 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13131 ins_encode %{
13132 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13133 %}
13134 ins_pipe(ialu_reg_shift);
13135 %}
13136
13137
13138 // Rotations
13139
13140 // This pattern is automatically generated from aarch64_ad.m4
13141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13142 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13143 %{
13144 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13145 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13146
13147 ins_cost(INSN_COST);
13148 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13149
13150 ins_encode %{
13151 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13152 $rshift$$constant & 63);
13153 %}
13154 ins_pipe(ialu_reg_reg_extr);
13155 %}
13156
13157
13158 // This pattern is automatically generated from aarch64_ad.m4
13159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13160 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13161 %{
13162 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13163 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13164
13165 ins_cost(INSN_COST);
13166 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13167
13168 ins_encode %{
13169 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13170 $rshift$$constant & 31);
13171 %}
13172 ins_pipe(ialu_reg_reg_extr);
13173 %}
13174
13175
13176 // This pattern is automatically generated from aarch64_ad.m4
13177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13178 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13179 %{
13180 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13181 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13182
13183 ins_cost(INSN_COST);
13184 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13185
13186 ins_encode %{
13187 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13188 $rshift$$constant & 63);
13189 %}
13190 ins_pipe(ialu_reg_reg_extr);
13191 %}
13192
13193
13194 // This pattern is automatically generated from aarch64_ad.m4
13195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13196 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13197 %{
13198 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13199 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13200
13201 ins_cost(INSN_COST);
13202 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13203
13204 ins_encode %{
13205 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13206 $rshift$$constant & 31);
13207 %}
13208 ins_pipe(ialu_reg_reg_extr);
13209 %}
13210
13211 // This pattern is automatically generated from aarch64_ad.m4
13212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13213 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13214 %{
13215 match(Set dst (RotateRight src shift));
13216
13217 ins_cost(INSN_COST);
13218 format %{ "ror $dst, $src, $shift" %}
13219
13220 ins_encode %{
13221 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13222 $shift$$constant & 0x1f);
13223 %}
13224 ins_pipe(ialu_reg_reg_vshift);
13225 %}
13226
13227 // This pattern is automatically generated from aarch64_ad.m4
13228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13229 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13230 %{
13231 match(Set dst (RotateRight src shift));
13232
13233 ins_cost(INSN_COST);
13234 format %{ "ror $dst, $src, $shift" %}
13235
13236 ins_encode %{
13237 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13238 $shift$$constant & 0x3f);
13239 %}
13240 ins_pipe(ialu_reg_reg_vshift);
13241 %}
13242
13243 // This pattern is automatically generated from aarch64_ad.m4
13244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13245 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13246 %{
13247 match(Set dst (RotateRight src shift));
13248
13249 ins_cost(INSN_COST);
13250 format %{ "ror $dst, $src, $shift" %}
13251
13252 ins_encode %{
13253 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13254 %}
13255 ins_pipe(ialu_reg_reg_vshift);
13256 %}
13257
13258 // This pattern is automatically generated from aarch64_ad.m4
13259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13260 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13261 %{
13262 match(Set dst (RotateRight src shift));
13263
13264 ins_cost(INSN_COST);
13265 format %{ "ror $dst, $src, $shift" %}
13266
13267 ins_encode %{
13268 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13269 %}
13270 ins_pipe(ialu_reg_reg_vshift);
13271 %}
13272
13273 // This pattern is automatically generated from aarch64_ad.m4
13274 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13275 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13276 %{
13277 match(Set dst (RotateLeft src shift));
13278
13279 ins_cost(INSN_COST);
13280 format %{ "rol $dst, $src, $shift" %}
13281
13282 ins_encode %{
13283 __ subw(rscratch1, zr, as_Register($shift$$reg));
13284 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13285 %}
13286 ins_pipe(ialu_reg_reg_vshift);
13287 %}
13288
13289 // This pattern is automatically generated from aarch64_ad.m4
13290 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13291 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13292 %{
13293 match(Set dst (RotateLeft src shift));
13294
13295 ins_cost(INSN_COST);
13296 format %{ "rol $dst, $src, $shift" %}
13297
13298 ins_encode %{
13299 __ subw(rscratch1, zr, as_Register($shift$$reg));
13300 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13301 %}
13302 ins_pipe(ialu_reg_reg_vshift);
13303 %}
13304
13305
13306 // Add/subtract (extended)
13307
13308 // This pattern is automatically generated from aarch64_ad.m4
13309 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13310 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13311 %{
13312 match(Set dst (AddL src1 (ConvI2L src2)));
13313 ins_cost(INSN_COST);
13314 format %{ "add $dst, $src1, $src2, sxtw" %}
13315
13316 ins_encode %{
13317 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13318 as_Register($src2$$reg), ext::sxtw);
13319 %}
13320 ins_pipe(ialu_reg_reg);
13321 %}
13322
13323 // This pattern is automatically generated from aarch64_ad.m4
13324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13325 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13326 %{
13327 match(Set dst (SubL src1 (ConvI2L src2)));
13328 ins_cost(INSN_COST);
13329 format %{ "sub $dst, $src1, $src2, sxtw" %}
13330
13331 ins_encode %{
13332 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13333 as_Register($src2$$reg), ext::sxtw);
13334 %}
13335 ins_pipe(ialu_reg_reg);
13336 %}
13337
13338 // This pattern is automatically generated from aarch64_ad.m4
13339 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13340 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13341 %{
13342 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13343 ins_cost(INSN_COST);
13344 format %{ "add $dst, $src1, $src2, sxth" %}
13345
13346 ins_encode %{
13347 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13348 as_Register($src2$$reg), ext::sxth);
13349 %}
13350 ins_pipe(ialu_reg_reg);
13351 %}
13352
13353 // This pattern is automatically generated from aarch64_ad.m4
13354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13355 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13356 %{
13357 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13358 ins_cost(INSN_COST);
13359 format %{ "add $dst, $src1, $src2, sxtb" %}
13360
13361 ins_encode %{
13362 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13363 as_Register($src2$$reg), ext::sxtb);
13364 %}
13365 ins_pipe(ialu_reg_reg);
13366 %}
13367
13368 // This pattern is automatically generated from aarch64_ad.m4
13369 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13370 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13371 %{
13372 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13373 ins_cost(INSN_COST);
13374 format %{ "add $dst, $src1, $src2, uxtb" %}
13375
13376 ins_encode %{
13377 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13378 as_Register($src2$$reg), ext::uxtb);
13379 %}
13380 ins_pipe(ialu_reg_reg);
13381 %}
13382
13383 // This pattern is automatically generated from aarch64_ad.m4
13384 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13385 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13386 %{
13387 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13388 ins_cost(INSN_COST);
13389 format %{ "add $dst, $src1, $src2, sxth" %}
13390
13391 ins_encode %{
13392 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13393 as_Register($src2$$reg), ext::sxth);
13394 %}
13395 ins_pipe(ialu_reg_reg);
13396 %}
13397
13398 // This pattern is automatically generated from aarch64_ad.m4
13399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13400 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13401 %{
13402 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13403 ins_cost(INSN_COST);
13404 format %{ "add $dst, $src1, $src2, sxtw" %}
13405
13406 ins_encode %{
13407 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13408 as_Register($src2$$reg), ext::sxtw);
13409 %}
13410 ins_pipe(ialu_reg_reg);
13411 %}
13412
13413 // This pattern is automatically generated from aarch64_ad.m4
13414 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13415 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13416 %{
13417 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13418 ins_cost(INSN_COST);
13419 format %{ "add $dst, $src1, $src2, sxtb" %}
13420
13421 ins_encode %{
13422 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13423 as_Register($src2$$reg), ext::sxtb);
13424 %}
13425 ins_pipe(ialu_reg_reg);
13426 %}
13427
13428 // This pattern is automatically generated from aarch64_ad.m4
13429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13430 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13431 %{
13432 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13433 ins_cost(INSN_COST);
13434 format %{ "add $dst, $src1, $src2, uxtb" %}
13435
13436 ins_encode %{
13437 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13438 as_Register($src2$$reg), ext::uxtb);
13439 %}
13440 ins_pipe(ialu_reg_reg);
13441 %}
13442
13443 // This pattern is automatically generated from aarch64_ad.m4
13444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13445 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13446 %{
13447 match(Set dst (AddI src1 (AndI src2 mask)));
13448 ins_cost(INSN_COST);
13449 format %{ "addw $dst, $src1, $src2, uxtb" %}
13450
13451 ins_encode %{
13452 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13453 as_Register($src2$$reg), ext::uxtb);
13454 %}
13455 ins_pipe(ialu_reg_reg);
13456 %}
13457
13458 // This pattern is automatically generated from aarch64_ad.m4
13459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13460 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13461 %{
13462 match(Set dst (AddI src1 (AndI src2 mask)));
13463 ins_cost(INSN_COST);
13464 format %{ "addw $dst, $src1, $src2, uxth" %}
13465
13466 ins_encode %{
13467 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13468 as_Register($src2$$reg), ext::uxth);
13469 %}
13470 ins_pipe(ialu_reg_reg);
13471 %}
13472
13473 // This pattern is automatically generated from aarch64_ad.m4
13474 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13475 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13476 %{
13477 match(Set dst (AddL src1 (AndL src2 mask)));
13478 ins_cost(INSN_COST);
13479 format %{ "add $dst, $src1, $src2, uxtb" %}
13480
13481 ins_encode %{
13482 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13483 as_Register($src2$$reg), ext::uxtb);
13484 %}
13485 ins_pipe(ialu_reg_reg);
13486 %}
13487
13488 // This pattern is automatically generated from aarch64_ad.m4
13489 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13490 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13491 %{
13492 match(Set dst (AddL src1 (AndL src2 mask)));
13493 ins_cost(INSN_COST);
13494 format %{ "add $dst, $src1, $src2, uxth" %}
13495
13496 ins_encode %{
13497 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13498 as_Register($src2$$reg), ext::uxth);
13499 %}
13500 ins_pipe(ialu_reg_reg);
13501 %}
13502
13503 // This pattern is automatically generated from aarch64_ad.m4
13504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13505 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13506 %{
13507 match(Set dst (AddL src1 (AndL src2 mask)));
13508 ins_cost(INSN_COST);
13509 format %{ "add $dst, $src1, $src2, uxtw" %}
13510
13511 ins_encode %{
13512 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13513 as_Register($src2$$reg), ext::uxtw);
13514 %}
13515 ins_pipe(ialu_reg_reg);
13516 %}
13517
13518 // This pattern is automatically generated from aarch64_ad.m4
13519 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13520 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13521 %{
13522 match(Set dst (SubI src1 (AndI src2 mask)));
13523 ins_cost(INSN_COST);
13524 format %{ "subw $dst, $src1, $src2, uxtb" %}
13525
13526 ins_encode %{
13527 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13528 as_Register($src2$$reg), ext::uxtb);
13529 %}
13530 ins_pipe(ialu_reg_reg);
13531 %}
13532
13533 // This pattern is automatically generated from aarch64_ad.m4
13534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13535 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13536 %{
13537 match(Set dst (SubI src1 (AndI src2 mask)));
13538 ins_cost(INSN_COST);
13539 format %{ "subw $dst, $src1, $src2, uxth" %}
13540
13541 ins_encode %{
13542 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13543 as_Register($src2$$reg), ext::uxth);
13544 %}
13545 ins_pipe(ialu_reg_reg);
13546 %}
13547
13548 // This pattern is automatically generated from aarch64_ad.m4
13549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13550 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13551 %{
13552 match(Set dst (SubL src1 (AndL src2 mask)));
13553 ins_cost(INSN_COST);
13554 format %{ "sub $dst, $src1, $src2, uxtb" %}
13555
13556 ins_encode %{
13557 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13558 as_Register($src2$$reg), ext::uxtb);
13559 %}
13560 ins_pipe(ialu_reg_reg);
13561 %}
13562
13563 // This pattern is automatically generated from aarch64_ad.m4
13564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13565 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13566 %{
13567 match(Set dst (SubL src1 (AndL src2 mask)));
13568 ins_cost(INSN_COST);
13569 format %{ "sub $dst, $src1, $src2, uxth" %}
13570
13571 ins_encode %{
13572 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13573 as_Register($src2$$reg), ext::uxth);
13574 %}
13575 ins_pipe(ialu_reg_reg);
13576 %}
13577
13578 // This pattern is automatically generated from aarch64_ad.m4
13579 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13580 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13581 %{
13582 match(Set dst (SubL src1 (AndL src2 mask)));
13583 ins_cost(INSN_COST);
13584 format %{ "sub $dst, $src1, $src2, uxtw" %}
13585
13586 ins_encode %{
13587 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13588 as_Register($src2$$reg), ext::uxtw);
13589 %}
13590 ins_pipe(ialu_reg_reg);
13591 %}
13592
13593
13594 // This pattern is automatically generated from aarch64_ad.m4
13595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13596 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13597 %{
13598 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13599 ins_cost(1.9 * INSN_COST);
13600 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13601
13602 ins_encode %{
13603 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13604 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13605 %}
13606 ins_pipe(ialu_reg_reg_shift);
13607 %}
13608
13609 // This pattern is automatically generated from aarch64_ad.m4
13610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13611 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13612 %{
13613 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13614 ins_cost(1.9 * INSN_COST);
13615 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13616
13617 ins_encode %{
13618 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13619 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13620 %}
13621 ins_pipe(ialu_reg_reg_shift);
13622 %}
13623
13624 // This pattern is automatically generated from aarch64_ad.m4
13625 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13626 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13627 %{
13628 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13629 ins_cost(1.9 * INSN_COST);
13630 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13631
13632 ins_encode %{
13633 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13634 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13635 %}
13636 ins_pipe(ialu_reg_reg_shift);
13637 %}
13638
13639 // This pattern is automatically generated from aarch64_ad.m4
13640 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13641 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13642 %{
13643 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13644 ins_cost(1.9 * INSN_COST);
13645 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13646
13647 ins_encode %{
13648 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13649 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13650 %}
13651 ins_pipe(ialu_reg_reg_shift);
13652 %}
13653
13654 // This pattern is automatically generated from aarch64_ad.m4
13655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13656 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13657 %{
13658 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13659 ins_cost(1.9 * INSN_COST);
13660 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13661
13662 ins_encode %{
13663 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13664 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13665 %}
13666 ins_pipe(ialu_reg_reg_shift);
13667 %}
13668
13669 // This pattern is automatically generated from aarch64_ad.m4
13670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13671 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13672 %{
13673 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13674 ins_cost(1.9 * INSN_COST);
13675 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13676
13677 ins_encode %{
13678 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13679 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13680 %}
13681 ins_pipe(ialu_reg_reg_shift);
13682 %}
13683
13684 // This pattern is automatically generated from aarch64_ad.m4
13685 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13686 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13687 %{
13688 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13689 ins_cost(1.9 * INSN_COST);
13690 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13691
13692 ins_encode %{
13693 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13694 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13695 %}
13696 ins_pipe(ialu_reg_reg_shift);
13697 %}
13698
13699 // This pattern is automatically generated from aarch64_ad.m4
13700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13701 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13702 %{
13703 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13704 ins_cost(1.9 * INSN_COST);
13705 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13706
13707 ins_encode %{
13708 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13709 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13710 %}
13711 ins_pipe(ialu_reg_reg_shift);
13712 %}
13713
13714 // This pattern is automatically generated from aarch64_ad.m4
13715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13716 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13717 %{
13718 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13719 ins_cost(1.9 * INSN_COST);
13720 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13721
13722 ins_encode %{
13723 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13724 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13725 %}
13726 ins_pipe(ialu_reg_reg_shift);
13727 %}
13728
13729 // This pattern is automatically generated from aarch64_ad.m4
13730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13731 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13732 %{
13733 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13734 ins_cost(1.9 * INSN_COST);
13735 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13736
13737 ins_encode %{
13738 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13739 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13740 %}
13741 ins_pipe(ialu_reg_reg_shift);
13742 %}
13743
13744 // This pattern is automatically generated from aarch64_ad.m4
13745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13746 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13747 %{
13748 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13749 ins_cost(1.9 * INSN_COST);
13750 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13751
13752 ins_encode %{
13753 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13754 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13755 %}
13756 ins_pipe(ialu_reg_reg_shift);
13757 %}
13758
13759 // This pattern is automatically generated from aarch64_ad.m4
13760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13761 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13762 %{
13763 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13764 ins_cost(1.9 * INSN_COST);
13765 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13766
13767 ins_encode %{
13768 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13769 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13770 %}
13771 ins_pipe(ialu_reg_reg_shift);
13772 %}
13773
13774 // This pattern is automatically generated from aarch64_ad.m4
13775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13776 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13777 %{
13778 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13779 ins_cost(1.9 * INSN_COST);
13780 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13781
13782 ins_encode %{
13783 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13784 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13785 %}
13786 ins_pipe(ialu_reg_reg_shift);
13787 %}
13788
13789 // This pattern is automatically generated from aarch64_ad.m4
13790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13791 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13792 %{
13793 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13794 ins_cost(1.9 * INSN_COST);
13795 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13796
13797 ins_encode %{
13798 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13799 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13800 %}
13801 ins_pipe(ialu_reg_reg_shift);
13802 %}
13803
13804 // This pattern is automatically generated from aarch64_ad.m4
13805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13806 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13807 %{
13808 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13809 ins_cost(1.9 * INSN_COST);
13810 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13811
13812 ins_encode %{
13813 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13814 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13815 %}
13816 ins_pipe(ialu_reg_reg_shift);
13817 %}
13818
13819 // This pattern is automatically generated from aarch64_ad.m4
13820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13821 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13822 %{
13823 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13824 ins_cost(1.9 * INSN_COST);
13825 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13826
13827 ins_encode %{
13828 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13829 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13830 %}
13831 ins_pipe(ialu_reg_reg_shift);
13832 %}
13833
13834 // This pattern is automatically generated from aarch64_ad.m4
13835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13836 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13837 %{
13838 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13839 ins_cost(1.9 * INSN_COST);
13840 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13841
13842 ins_encode %{
13843 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13844 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13845 %}
13846 ins_pipe(ialu_reg_reg_shift);
13847 %}
13848
13849 // This pattern is automatically generated from aarch64_ad.m4
13850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13851 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13852 %{
13853 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13854 ins_cost(1.9 * INSN_COST);
13855 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13856
13857 ins_encode %{
13858 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13859 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13860 %}
13861 ins_pipe(ialu_reg_reg_shift);
13862 %}
13863
13864 // This pattern is automatically generated from aarch64_ad.m4
13865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13866 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13867 %{
13868 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13869 ins_cost(1.9 * INSN_COST);
13870 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13871
13872 ins_encode %{
13873 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13874 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13875 %}
13876 ins_pipe(ialu_reg_reg_shift);
13877 %}
13878
13879 // This pattern is automatically generated from aarch64_ad.m4
13880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13881 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13882 %{
13883 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13884 ins_cost(1.9 * INSN_COST);
13885 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13886
13887 ins_encode %{
13888 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13889 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13890 %}
13891 ins_pipe(ialu_reg_reg_shift);
13892 %}
13893
13894 // This pattern is automatically generated from aarch64_ad.m4
13895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13896 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13897 %{
13898 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13899 ins_cost(1.9 * INSN_COST);
13900 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13901
13902 ins_encode %{
13903 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13904 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13905 %}
13906 ins_pipe(ialu_reg_reg_shift);
13907 %}
13908
13909 // This pattern is automatically generated from aarch64_ad.m4
13910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13911 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13912 %{
13913 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13914 ins_cost(1.9 * INSN_COST);
13915 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13916
13917 ins_encode %{
13918 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13919 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13920 %}
13921 ins_pipe(ialu_reg_reg_shift);
13922 %}
13923
13924 // This pattern is automatically generated from aarch64_ad.m4
13925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13926 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13927 %{
13928 effect(DEF dst, USE src1, USE src2, USE cr);
13929 ins_cost(INSN_COST * 2);
13930 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13931
13932 ins_encode %{
13933 __ cselw($dst$$Register,
13934 $src1$$Register,
13935 $src2$$Register,
13936 Assembler::LT);
13937 %}
13938 ins_pipe(icond_reg_reg);
13939 %}
13940
13941 // This pattern is automatically generated from aarch64_ad.m4
13942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13943 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13944 %{
13945 effect(DEF dst, USE src1, USE src2, USE cr);
13946 ins_cost(INSN_COST * 2);
13947 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13948
13949 ins_encode %{
13950 __ cselw($dst$$Register,
13951 $src1$$Register,
13952 $src2$$Register,
13953 Assembler::GT);
13954 %}
13955 ins_pipe(icond_reg_reg);
13956 %}
13957
13958 // This pattern is automatically generated from aarch64_ad.m4
13959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13960 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13961 %{
13962 effect(DEF dst, USE src1, USE cr);
13963 ins_cost(INSN_COST * 2);
13964 format %{ "cselw $dst, $src1, zr lt\t" %}
13965
13966 ins_encode %{
13967 __ cselw($dst$$Register,
13968 $src1$$Register,
13969 zr,
13970 Assembler::LT);
13971 %}
13972 ins_pipe(icond_reg);
13973 %}
13974
13975 // This pattern is automatically generated from aarch64_ad.m4
13976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13977 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13978 %{
13979 effect(DEF dst, USE src1, USE cr);
13980 ins_cost(INSN_COST * 2);
13981 format %{ "cselw $dst, $src1, zr gt\t" %}
13982
13983 ins_encode %{
13984 __ cselw($dst$$Register,
13985 $src1$$Register,
13986 zr,
13987 Assembler::GT);
13988 %}
13989 ins_pipe(icond_reg);
13990 %}
13991
13992 // This pattern is automatically generated from aarch64_ad.m4
13993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13994 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13995 %{
13996 effect(DEF dst, USE src1, USE cr);
13997 ins_cost(INSN_COST * 2);
13998 format %{ "csincw $dst, $src1, zr le\t" %}
13999
14000 ins_encode %{
14001 __ csincw($dst$$Register,
14002 $src1$$Register,
14003 zr,
14004 Assembler::LE);
14005 %}
14006 ins_pipe(icond_reg);
14007 %}
14008
14009 // This pattern is automatically generated from aarch64_ad.m4
14010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14011 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14012 %{
14013 effect(DEF dst, USE src1, USE cr);
14014 ins_cost(INSN_COST * 2);
14015 format %{ "csincw $dst, $src1, zr gt\t" %}
14016
14017 ins_encode %{
14018 __ csincw($dst$$Register,
14019 $src1$$Register,
14020 zr,
14021 Assembler::GT);
14022 %}
14023 ins_pipe(icond_reg);
14024 %}
14025
14026 // This pattern is automatically generated from aarch64_ad.m4
14027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14028 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14029 %{
14030 effect(DEF dst, USE src1, USE cr);
14031 ins_cost(INSN_COST * 2);
14032 format %{ "csinvw $dst, $src1, zr lt\t" %}
14033
14034 ins_encode %{
14035 __ csinvw($dst$$Register,
14036 $src1$$Register,
14037 zr,
14038 Assembler::LT);
14039 %}
14040 ins_pipe(icond_reg);
14041 %}
14042
14043 // This pattern is automatically generated from aarch64_ad.m4
14044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14045 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14046 %{
14047 effect(DEF dst, USE src1, USE cr);
14048 ins_cost(INSN_COST * 2);
14049 format %{ "csinvw $dst, $src1, zr ge\t" %}
14050
14051 ins_encode %{
14052 __ csinvw($dst$$Register,
14053 $src1$$Register,
14054 zr,
14055 Assembler::GE);
14056 %}
14057 ins_pipe(icond_reg);
14058 %}
14059
14060 // This pattern is automatically generated from aarch64_ad.m4
14061 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14062 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14063 %{
14064 match(Set dst (MinI src imm));
14065 ins_cost(INSN_COST * 3);
14066 expand %{
14067 rFlagsReg cr;
14068 compI_reg_imm0(cr, src);
14069 cmovI_reg_imm0_lt(dst, src, cr);
14070 %}
14071 %}
14072
14073 // This pattern is automatically generated from aarch64_ad.m4
14074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14075 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14076 %{
14077 match(Set dst (MinI imm src));
14078 ins_cost(INSN_COST * 3);
14079 expand %{
14080 rFlagsReg cr;
14081 compI_reg_imm0(cr, src);
14082 cmovI_reg_imm0_lt(dst, src, cr);
14083 %}
14084 %}
14085
14086 // This pattern is automatically generated from aarch64_ad.m4
14087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14088 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14089 %{
14090 match(Set dst (MinI src imm));
14091 ins_cost(INSN_COST * 3);
14092 expand %{
14093 rFlagsReg cr;
14094 compI_reg_imm0(cr, src);
14095 cmovI_reg_imm1_le(dst, src, cr);
14096 %}
14097 %}
14098
14099 // This pattern is automatically generated from aarch64_ad.m4
14100 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14101 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14102 %{
14103 match(Set dst (MinI imm src));
14104 ins_cost(INSN_COST * 3);
14105 expand %{
14106 rFlagsReg cr;
14107 compI_reg_imm0(cr, src);
14108 cmovI_reg_imm1_le(dst, src, cr);
14109 %}
14110 %}
14111
14112 // This pattern is automatically generated from aarch64_ad.m4
14113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14114 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14115 %{
14116 match(Set dst (MinI src imm));
14117 ins_cost(INSN_COST * 3);
14118 expand %{
14119 rFlagsReg cr;
14120 compI_reg_imm0(cr, src);
14121 cmovI_reg_immM1_lt(dst, src, cr);
14122 %}
14123 %}
14124
14125 // This pattern is automatically generated from aarch64_ad.m4
14126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14127 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14128 %{
14129 match(Set dst (MinI imm src));
14130 ins_cost(INSN_COST * 3);
14131 expand %{
14132 rFlagsReg cr;
14133 compI_reg_imm0(cr, src);
14134 cmovI_reg_immM1_lt(dst, src, cr);
14135 %}
14136 %}
14137
14138 // This pattern is automatically generated from aarch64_ad.m4
14139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14140 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14141 %{
14142 match(Set dst (MaxI src imm));
14143 ins_cost(INSN_COST * 3);
14144 expand %{
14145 rFlagsReg cr;
14146 compI_reg_imm0(cr, src);
14147 cmovI_reg_imm0_gt(dst, src, cr);
14148 %}
14149 %}
14150
14151 // This pattern is automatically generated from aarch64_ad.m4
14152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14153 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14154 %{
14155 match(Set dst (MaxI imm src));
14156 ins_cost(INSN_COST * 3);
14157 expand %{
14158 rFlagsReg cr;
14159 compI_reg_imm0(cr, src);
14160 cmovI_reg_imm0_gt(dst, src, cr);
14161 %}
14162 %}
14163
14164 // This pattern is automatically generated from aarch64_ad.m4
14165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14166 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14167 %{
14168 match(Set dst (MaxI src imm));
14169 ins_cost(INSN_COST * 3);
14170 expand %{
14171 rFlagsReg cr;
14172 compI_reg_imm0(cr, src);
14173 cmovI_reg_imm1_gt(dst, src, cr);
14174 %}
14175 %}
14176
14177 // This pattern is automatically generated from aarch64_ad.m4
14178 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14179 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14180 %{
14181 match(Set dst (MaxI imm src));
14182 ins_cost(INSN_COST * 3);
14183 expand %{
14184 rFlagsReg cr;
14185 compI_reg_imm0(cr, src);
14186 cmovI_reg_imm1_gt(dst, src, cr);
14187 %}
14188 %}
14189
14190 // This pattern is automatically generated from aarch64_ad.m4
14191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14192 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14193 %{
14194 match(Set dst (MaxI src imm));
14195 ins_cost(INSN_COST * 3);
14196 expand %{
14197 rFlagsReg cr;
14198 compI_reg_imm0(cr, src);
14199 cmovI_reg_immM1_ge(dst, src, cr);
14200 %}
14201 %}
14202
14203 // This pattern is automatically generated from aarch64_ad.m4
14204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14205 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14206 %{
14207 match(Set dst (MaxI imm src));
14208 ins_cost(INSN_COST * 3);
14209 expand %{
14210 rFlagsReg cr;
14211 compI_reg_imm0(cr, src);
14212 cmovI_reg_immM1_ge(dst, src, cr);
14213 %}
14214 %}
14215
14216 // This pattern is automatically generated from aarch64_ad.m4
14217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14218 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
14219 %{
14220 match(Set dst (ReverseI src));
14221 ins_cost(INSN_COST);
14222 format %{ "rbitw $dst, $src" %}
14223 ins_encode %{
14224 __ rbitw($dst$$Register, $src$$Register);
14225 %}
14226 ins_pipe(ialu_reg);
14227 %}
14228
14229 // This pattern is automatically generated from aarch64_ad.m4
14230 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14231 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
14232 %{
14233 match(Set dst (ReverseL src));
14234 ins_cost(INSN_COST);
14235 format %{ "rbit $dst, $src" %}
14236 ins_encode %{
14237 __ rbit($dst$$Register, $src$$Register);
14238 %}
14239 ins_pipe(ialu_reg);
14240 %}
14241
14242
14243 // END This section of the file is automatically generated. Do not edit --------------
14244
14245
14246 // ============================================================================
14247 // Floating Point Arithmetic Instructions
14248
14249 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14250 match(Set dst (AddF src1 src2));
14251
14252 ins_cost(INSN_COST * 5);
14253 format %{ "fadds $dst, $src1, $src2" %}
14254
14255 ins_encode %{
14256 __ fadds(as_FloatRegister($dst$$reg),
14257 as_FloatRegister($src1$$reg),
14258 as_FloatRegister($src2$$reg));
14259 %}
14260
14261 ins_pipe(fp_dop_reg_reg_s);
14262 %}
14263
14264 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14265 match(Set dst (AddD src1 src2));
14266
14267 ins_cost(INSN_COST * 5);
14268 format %{ "faddd $dst, $src1, $src2" %}
14269
14270 ins_encode %{
14271 __ faddd(as_FloatRegister($dst$$reg),
14272 as_FloatRegister($src1$$reg),
14273 as_FloatRegister($src2$$reg));
14274 %}
14275
14276 ins_pipe(fp_dop_reg_reg_d);
14277 %}
14278
14279 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14280 match(Set dst (SubF src1 src2));
14281
14282 ins_cost(INSN_COST * 5);
14283 format %{ "fsubs $dst, $src1, $src2" %}
14284
14285 ins_encode %{
14286 __ fsubs(as_FloatRegister($dst$$reg),
14287 as_FloatRegister($src1$$reg),
14288 as_FloatRegister($src2$$reg));
14289 %}
14290
14291 ins_pipe(fp_dop_reg_reg_s);
14292 %}
14293
14294 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14295 match(Set dst (SubD src1 src2));
14296
14297 ins_cost(INSN_COST * 5);
14298 format %{ "fsubd $dst, $src1, $src2" %}
14299
14300 ins_encode %{
14301 __ fsubd(as_FloatRegister($dst$$reg),
14302 as_FloatRegister($src1$$reg),
14303 as_FloatRegister($src2$$reg));
14304 %}
14305
14306 ins_pipe(fp_dop_reg_reg_d);
14307 %}
14308
14309 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14310 match(Set dst (MulF src1 src2));
14311
14312 ins_cost(INSN_COST * 6);
14313 format %{ "fmuls $dst, $src1, $src2" %}
14314
14315 ins_encode %{
14316 __ fmuls(as_FloatRegister($dst$$reg),
14317 as_FloatRegister($src1$$reg),
14318 as_FloatRegister($src2$$reg));
14319 %}
14320
14321 ins_pipe(fp_dop_reg_reg_s);
14322 %}
14323
14324 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14325 match(Set dst (MulD src1 src2));
14326
14327 ins_cost(INSN_COST * 6);
14328 format %{ "fmuld $dst, $src1, $src2" %}
14329
14330 ins_encode %{
14331 __ fmuld(as_FloatRegister($dst$$reg),
14332 as_FloatRegister($src1$$reg),
14333 as_FloatRegister($src2$$reg));
14334 %}
14335
14336 ins_pipe(fp_dop_reg_reg_d);
14337 %}
14338
14339 // src1 * src2 + src3
14340 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14341 match(Set dst (FmaF src3 (Binary src1 src2)));
14342
14343 format %{ "fmadds $dst, $src1, $src2, $src3" %}
14344
14345 ins_encode %{
14346 assert(UseFMA, "Needs FMA instructions support.");
14347 __ fmadds(as_FloatRegister($dst$$reg),
14348 as_FloatRegister($src1$$reg),
14349 as_FloatRegister($src2$$reg),
14350 as_FloatRegister($src3$$reg));
14351 %}
14352
14353 ins_pipe(pipe_class_default);
14354 %}
14355
14356 // src1 * src2 + src3
14357 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14358 match(Set dst (FmaD src3 (Binary src1 src2)));
14359
14360 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
14361
14362 ins_encode %{
14363 assert(UseFMA, "Needs FMA instructions support.");
14364 __ fmaddd(as_FloatRegister($dst$$reg),
14365 as_FloatRegister($src1$$reg),
14366 as_FloatRegister($src2$$reg),
14367 as_FloatRegister($src3$$reg));
14368 %}
14369
14370 ins_pipe(pipe_class_default);
14371 %}
14372
14373 // src1 * (-src2) + src3
14374 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
14375 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14376 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14377
14378 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
14379
14380 ins_encode %{
14381 assert(UseFMA, "Needs FMA instructions support.");
14382 __ fmsubs(as_FloatRegister($dst$$reg),
14383 as_FloatRegister($src1$$reg),
14384 as_FloatRegister($src2$$reg),
14385 as_FloatRegister($src3$$reg));
14386 %}
14387
14388 ins_pipe(pipe_class_default);
14389 %}
14390
14391 // src1 * (-src2) + src3
14392 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
14393 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14394 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14395
14396 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
14397
14398 ins_encode %{
14399 assert(UseFMA, "Needs FMA instructions support.");
14400 __ fmsubd(as_FloatRegister($dst$$reg),
14401 as_FloatRegister($src1$$reg),
14402 as_FloatRegister($src2$$reg),
14403 as_FloatRegister($src3$$reg));
14404 %}
14405
14406 ins_pipe(pipe_class_default);
14407 %}
14408
14409 // src1 * (-src2) - src3
14410 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14411 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14412 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14413
14414 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
14415
14416 ins_encode %{
14417 assert(UseFMA, "Needs FMA instructions support.");
14418 __ fnmadds(as_FloatRegister($dst$$reg),
14419 as_FloatRegister($src1$$reg),
14420 as_FloatRegister($src2$$reg),
14421 as_FloatRegister($src3$$reg));
14422 %}
14423
14424 ins_pipe(pipe_class_default);
14425 %}
14426
14427 // src1 * (-src2) - src3
14428 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14429 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14430 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14431
14432 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14433
14434 ins_encode %{
14435 assert(UseFMA, "Needs FMA instructions support.");
14436 __ fnmaddd(as_FloatRegister($dst$$reg),
14437 as_FloatRegister($src1$$reg),
14438 as_FloatRegister($src2$$reg),
14439 as_FloatRegister($src3$$reg));
14440 %}
14441
14442 ins_pipe(pipe_class_default);
14443 %}
14444
14445 // src1 * src2 - src3
14446 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14447 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14448
14449 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14450
14451 ins_encode %{
14452 assert(UseFMA, "Needs FMA instructions support.");
14453 __ fnmsubs(as_FloatRegister($dst$$reg),
14454 as_FloatRegister($src1$$reg),
14455 as_FloatRegister($src2$$reg),
14456 as_FloatRegister($src3$$reg));
14457 %}
14458
14459 ins_pipe(pipe_class_default);
14460 %}
14461
14462 // src1 * src2 - src3
14463 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14464 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14465
14466 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14467
14468 ins_encode %{
14469 assert(UseFMA, "Needs FMA instructions support.");
14470 // n.b. insn name should be fnmsubd
14471 __ fnmsub(as_FloatRegister($dst$$reg),
14472 as_FloatRegister($src1$$reg),
14473 as_FloatRegister($src2$$reg),
14474 as_FloatRegister($src3$$reg));
14475 %}
14476
14477 ins_pipe(pipe_class_default);
14478 %}
14479
14480
14481 // Math.max(FF)F
14482 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14483 match(Set dst (MaxF src1 src2));
14484
14485 format %{ "fmaxs $dst, $src1, $src2" %}
14486 ins_encode %{
14487 __ fmaxs(as_FloatRegister($dst$$reg),
14488 as_FloatRegister($src1$$reg),
14489 as_FloatRegister($src2$$reg));
14490 %}
14491
14492 ins_pipe(fp_dop_reg_reg_s);
14493 %}
14494
14495 // Math.min(FF)F
14496 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14497 match(Set dst (MinF src1 src2));
14498
14499 format %{ "fmins $dst, $src1, $src2" %}
14500 ins_encode %{
14501 __ fmins(as_FloatRegister($dst$$reg),
14502 as_FloatRegister($src1$$reg),
14503 as_FloatRegister($src2$$reg));
14504 %}
14505
14506 ins_pipe(fp_dop_reg_reg_s);
14507 %}
14508
14509 // Math.max(DD)D
14510 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14511 match(Set dst (MaxD src1 src2));
14512
14513 format %{ "fmaxd $dst, $src1, $src2" %}
14514 ins_encode %{
14515 __ fmaxd(as_FloatRegister($dst$$reg),
14516 as_FloatRegister($src1$$reg),
14517 as_FloatRegister($src2$$reg));
14518 %}
14519
14520 ins_pipe(fp_dop_reg_reg_d);
14521 %}
14522
14523 // Math.min(DD)D
14524 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14525 match(Set dst (MinD src1 src2));
14526
14527 format %{ "fmind $dst, $src1, $src2" %}
14528 ins_encode %{
14529 __ fmind(as_FloatRegister($dst$$reg),
14530 as_FloatRegister($src1$$reg),
14531 as_FloatRegister($src2$$reg));
14532 %}
14533
14534 ins_pipe(fp_dop_reg_reg_d);
14535 %}
14536
14537
14538 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14539 match(Set dst (DivF src1 src2));
14540
14541 ins_cost(INSN_COST * 18);
14542 format %{ "fdivs $dst, $src1, $src2" %}
14543
14544 ins_encode %{
14545 __ fdivs(as_FloatRegister($dst$$reg),
14546 as_FloatRegister($src1$$reg),
14547 as_FloatRegister($src2$$reg));
14548 %}
14549
14550 ins_pipe(fp_div_s);
14551 %}
14552
14553 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14554 match(Set dst (DivD src1 src2));
14555
14556 ins_cost(INSN_COST * 32);
14557 format %{ "fdivd $dst, $src1, $src2" %}
14558
14559 ins_encode %{
14560 __ fdivd(as_FloatRegister($dst$$reg),
14561 as_FloatRegister($src1$$reg),
14562 as_FloatRegister($src2$$reg));
14563 %}
14564
14565 ins_pipe(fp_div_d);
14566 %}
14567
14568 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14569 match(Set dst (NegF src));
14570
14571 ins_cost(INSN_COST * 3);
14572 format %{ "fneg $dst, $src" %}
14573
14574 ins_encode %{
14575 __ fnegs(as_FloatRegister($dst$$reg),
14576 as_FloatRegister($src$$reg));
14577 %}
14578
14579 ins_pipe(fp_uop_s);
14580 %}
14581
14582 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14583 match(Set dst (NegD src));
14584
14585 ins_cost(INSN_COST * 3);
14586 format %{ "fnegd $dst, $src" %}
14587
14588 ins_encode %{
14589 __ fnegd(as_FloatRegister($dst$$reg),
14590 as_FloatRegister($src$$reg));
14591 %}
14592
14593 ins_pipe(fp_uop_d);
14594 %}
14595
14596 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14597 %{
14598 match(Set dst (AbsI src));
14599
14600 effect(KILL cr);
14601 ins_cost(INSN_COST * 2);
14602 format %{ "cmpw $src, zr\n\t"
14603 "cnegw $dst, $src, Assembler::LT\t# int abs"
14604 %}
14605
14606 ins_encode %{
14607 __ cmpw(as_Register($src$$reg), zr);
14608 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14609 %}
14610 ins_pipe(pipe_class_default);
14611 %}
14612
14613 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14614 %{
14615 match(Set dst (AbsL src));
14616
14617 effect(KILL cr);
14618 ins_cost(INSN_COST * 2);
14619 format %{ "cmp $src, zr\n\t"
14620 "cneg $dst, $src, Assembler::LT\t# long abs"
14621 %}
14622
14623 ins_encode %{
14624 __ cmp(as_Register($src$$reg), zr);
14625 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14626 %}
14627 ins_pipe(pipe_class_default);
14628 %}
14629
14630 instruct absF_reg(vRegF dst, vRegF src) %{
14631 match(Set dst (AbsF src));
14632
14633 ins_cost(INSN_COST * 3);
14634 format %{ "fabss $dst, $src" %}
14635 ins_encode %{
14636 __ fabss(as_FloatRegister($dst$$reg),
14637 as_FloatRegister($src$$reg));
14638 %}
14639
14640 ins_pipe(fp_uop_s);
14641 %}
14642
14643 instruct absD_reg(vRegD dst, vRegD src) %{
14644 match(Set dst (AbsD src));
14645
14646 ins_cost(INSN_COST * 3);
14647 format %{ "fabsd $dst, $src" %}
14648 ins_encode %{
14649 __ fabsd(as_FloatRegister($dst$$reg),
14650 as_FloatRegister($src$$reg));
14651 %}
14652
14653 ins_pipe(fp_uop_d);
14654 %}
14655
14656 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14657 match(Set dst (AbsF (SubF src1 src2)));
14658
14659 ins_cost(INSN_COST * 3);
14660 format %{ "fabds $dst, $src1, $src2" %}
14661 ins_encode %{
14662 __ fabds(as_FloatRegister($dst$$reg),
14663 as_FloatRegister($src1$$reg),
14664 as_FloatRegister($src2$$reg));
14665 %}
14666
14667 ins_pipe(fp_uop_s);
14668 %}
14669
14670 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14671 match(Set dst (AbsD (SubD src1 src2)));
14672
14673 ins_cost(INSN_COST * 3);
14674 format %{ "fabdd $dst, $src1, $src2" %}
14675 ins_encode %{
14676 __ fabdd(as_FloatRegister($dst$$reg),
14677 as_FloatRegister($src1$$reg),
14678 as_FloatRegister($src2$$reg));
14679 %}
14680
14681 ins_pipe(fp_uop_d);
14682 %}
14683
14684 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14685 match(Set dst (SqrtD src));
14686
14687 ins_cost(INSN_COST * 50);
14688 format %{ "fsqrtd $dst, $src" %}
14689 ins_encode %{
14690 __ fsqrtd(as_FloatRegister($dst$$reg),
14691 as_FloatRegister($src$$reg));
14692 %}
14693
14694 ins_pipe(fp_div_s);
14695 %}
14696
14697 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14698 match(Set dst (SqrtF src));
14699
14700 ins_cost(INSN_COST * 50);
14701 format %{ "fsqrts $dst, $src" %}
14702 ins_encode %{
14703 __ fsqrts(as_FloatRegister($dst$$reg),
14704 as_FloatRegister($src$$reg));
14705 %}
14706
14707 ins_pipe(fp_div_d);
14708 %}
14709
14710 // Math.rint, floor, ceil
14711 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14712 match(Set dst (RoundDoubleMode src rmode));
14713 format %{ "frint $dst, $src, $rmode" %}
14714 ins_encode %{
14715 switch ($rmode$$constant) {
14716 case RoundDoubleModeNode::rmode_rint:
14717 __ frintnd(as_FloatRegister($dst$$reg),
14718 as_FloatRegister($src$$reg));
14719 break;
14720 case RoundDoubleModeNode::rmode_floor:
14721 __ frintmd(as_FloatRegister($dst$$reg),
14722 as_FloatRegister($src$$reg));
14723 break;
14724 case RoundDoubleModeNode::rmode_ceil:
14725 __ frintpd(as_FloatRegister($dst$$reg),
14726 as_FloatRegister($src$$reg));
14727 break;
14728 }
14729 %}
14730 ins_pipe(fp_uop_d);
14731 %}
14732
14733 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14734 match(Set dst (CopySignD src1 (Binary src2 zero)));
14735 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14736 format %{ "CopySignD $dst $src1 $src2" %}
14737 ins_encode %{
14738 FloatRegister dst = as_FloatRegister($dst$$reg),
14739 src1 = as_FloatRegister($src1$$reg),
14740 src2 = as_FloatRegister($src2$$reg),
14741 zero = as_FloatRegister($zero$$reg);
14742 __ fnegd(dst, zero);
14743 __ bsl(dst, __ T8B, src2, src1);
14744 %}
14745 ins_pipe(fp_uop_d);
14746 %}
14747
14748 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14749 match(Set dst (CopySignF src1 src2));
14750 effect(TEMP_DEF dst, USE src1, USE src2);
14751 format %{ "CopySignF $dst $src1 $src2" %}
14752 ins_encode %{
14753 FloatRegister dst = as_FloatRegister($dst$$reg),
14754 src1 = as_FloatRegister($src1$$reg),
14755 src2 = as_FloatRegister($src2$$reg);
14756 __ movi(dst, __ T2S, 0x80, 24);
14757 __ bsl(dst, __ T8B, src2, src1);
14758 %}
14759 ins_pipe(fp_uop_d);
14760 %}
14761
14762 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14763 match(Set dst (SignumD src (Binary zero one)));
14764 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14765 format %{ "signumD $dst, $src" %}
14766 ins_encode %{
14767 FloatRegister src = as_FloatRegister($src$$reg),
14768 dst = as_FloatRegister($dst$$reg),
14769 zero = as_FloatRegister($zero$$reg),
14770 one = as_FloatRegister($one$$reg);
14771 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14772 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14773 // Bit selection instruction gets bit from "one" for each enabled bit in
14774 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14775 // NaN the whole "src" will be copied because "dst" is zero. For all other
14776 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14777 // from "src", and all other bits are copied from 1.0.
14778 __ bsl(dst, __ T8B, one, src);
14779 %}
14780 ins_pipe(fp_uop_d);
14781 %}
14782
14783 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14784 match(Set dst (SignumF src (Binary zero one)));
14785 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14786 format %{ "signumF $dst, $src" %}
14787 ins_encode %{
14788 FloatRegister src = as_FloatRegister($src$$reg),
14789 dst = as_FloatRegister($dst$$reg),
14790 zero = as_FloatRegister($zero$$reg),
14791 one = as_FloatRegister($one$$reg);
14792 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14793 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14794 // Bit selection instruction gets bit from "one" for each enabled bit in
14795 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14796 // NaN the whole "src" will be copied because "dst" is zero. For all other
14797 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14798 // from "src", and all other bits are copied from 1.0.
14799 __ bsl(dst, __ T8B, one, src);
14800 %}
14801 ins_pipe(fp_uop_d);
14802 %}
14803
14804 instruct onspinwait() %{
14805 match(OnSpinWait);
14806 ins_cost(INSN_COST);
14807
14808 format %{ "onspinwait" %}
14809
14810 ins_encode %{
14811 __ spin_wait();
14812 %}
14813 ins_pipe(pipe_class_empty);
14814 %}
14815
14816 // ============================================================================
14817 // Logical Instructions
14818
14819 // Integer Logical Instructions
14820
14821 // And Instructions
14822
14823
14824 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14825 match(Set dst (AndI src1 src2));
14826
14827 format %{ "andw $dst, $src1, $src2\t# int" %}
14828
14829 ins_cost(INSN_COST);
14830 ins_encode %{
14831 __ andw(as_Register($dst$$reg),
14832 as_Register($src1$$reg),
14833 as_Register($src2$$reg));
14834 %}
14835
14836 ins_pipe(ialu_reg_reg);
14837 %}
14838
14839 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14840 match(Set dst (AndI src1 src2));
14841
14842 format %{ "andsw $dst, $src1, $src2\t# int" %}
14843
14844 ins_cost(INSN_COST);
14845 ins_encode %{
14846 __ andw(as_Register($dst$$reg),
14847 as_Register($src1$$reg),
14848 (uint64_t)($src2$$constant));
14849 %}
14850
14851 ins_pipe(ialu_reg_imm);
14852 %}
14853
14854 // Or Instructions
14855
14856 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14857 match(Set dst (OrI src1 src2));
14858
14859 format %{ "orrw $dst, $src1, $src2\t# int" %}
14860
14861 ins_cost(INSN_COST);
14862 ins_encode %{
14863 __ orrw(as_Register($dst$$reg),
14864 as_Register($src1$$reg),
14865 as_Register($src2$$reg));
14866 %}
14867
14868 ins_pipe(ialu_reg_reg);
14869 %}
14870
14871 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14872 match(Set dst (OrI src1 src2));
14873
14874 format %{ "orrw $dst, $src1, $src2\t# int" %}
14875
14876 ins_cost(INSN_COST);
14877 ins_encode %{
14878 __ orrw(as_Register($dst$$reg),
14879 as_Register($src1$$reg),
14880 (uint64_t)($src2$$constant));
14881 %}
14882
14883 ins_pipe(ialu_reg_imm);
14884 %}
14885
14886 // Xor Instructions
14887
14888 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14889 match(Set dst (XorI src1 src2));
14890
14891 format %{ "eorw $dst, $src1, $src2\t# int" %}
14892
14893 ins_cost(INSN_COST);
14894 ins_encode %{
14895 __ eorw(as_Register($dst$$reg),
14896 as_Register($src1$$reg),
14897 as_Register($src2$$reg));
14898 %}
14899
14900 ins_pipe(ialu_reg_reg);
14901 %}
14902
14903 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14904 match(Set dst (XorI src1 src2));
14905
14906 format %{ "eorw $dst, $src1, $src2\t# int" %}
14907
14908 ins_cost(INSN_COST);
14909 ins_encode %{
14910 __ eorw(as_Register($dst$$reg),
14911 as_Register($src1$$reg),
14912 (uint64_t)($src2$$constant));
14913 %}
14914
14915 ins_pipe(ialu_reg_imm);
14916 %}
14917
14918 // Long Logical Instructions
14919 // TODO
14920
14921 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14922 match(Set dst (AndL src1 src2));
14923
14924 format %{ "and $dst, $src1, $src2\t# int" %}
14925
14926 ins_cost(INSN_COST);
14927 ins_encode %{
14928 __ andr(as_Register($dst$$reg),
14929 as_Register($src1$$reg),
14930 as_Register($src2$$reg));
14931 %}
14932
14933 ins_pipe(ialu_reg_reg);
14934 %}
14935
14936 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14937 match(Set dst (AndL src1 src2));
14938
14939 format %{ "and $dst, $src1, $src2\t# int" %}
14940
14941 ins_cost(INSN_COST);
14942 ins_encode %{
14943 __ andr(as_Register($dst$$reg),
14944 as_Register($src1$$reg),
14945 (uint64_t)($src2$$constant));
14946 %}
14947
14948 ins_pipe(ialu_reg_imm);
14949 %}
14950
14951 // Or Instructions
14952
14953 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14954 match(Set dst (OrL src1 src2));
14955
14956 format %{ "orr $dst, $src1, $src2\t# int" %}
14957
14958 ins_cost(INSN_COST);
14959 ins_encode %{
14960 __ orr(as_Register($dst$$reg),
14961 as_Register($src1$$reg),
14962 as_Register($src2$$reg));
14963 %}
14964
14965 ins_pipe(ialu_reg_reg);
14966 %}
14967
14968 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14969 match(Set dst (OrL src1 src2));
14970
14971 format %{ "orr $dst, $src1, $src2\t# int" %}
14972
14973 ins_cost(INSN_COST);
14974 ins_encode %{
14975 __ orr(as_Register($dst$$reg),
14976 as_Register($src1$$reg),
14977 (uint64_t)($src2$$constant));
14978 %}
14979
14980 ins_pipe(ialu_reg_imm);
14981 %}
14982
14983 // Xor Instructions
14984
14985 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14986 match(Set dst (XorL src1 src2));
14987
14988 format %{ "eor $dst, $src1, $src2\t# int" %}
14989
14990 ins_cost(INSN_COST);
14991 ins_encode %{
14992 __ eor(as_Register($dst$$reg),
14993 as_Register($src1$$reg),
14994 as_Register($src2$$reg));
14995 %}
14996
14997 ins_pipe(ialu_reg_reg);
14998 %}
14999
15000 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
15001 match(Set dst (XorL src1 src2));
15002
15003 ins_cost(INSN_COST);
15004 format %{ "eor $dst, $src1, $src2\t# int" %}
15005
15006 ins_encode %{
15007 __ eor(as_Register($dst$$reg),
15008 as_Register($src1$$reg),
15009 (uint64_t)($src2$$constant));
15010 %}
15011
15012 ins_pipe(ialu_reg_imm);
15013 %}
15014
15015 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
15016 %{
15017 match(Set dst (ConvI2L src));
15018
15019 ins_cost(INSN_COST);
15020 format %{ "sxtw $dst, $src\t# i2l" %}
15021 ins_encode %{
15022 __ sbfm($dst$$Register, $src$$Register, 0, 31);
15023 %}
15024 ins_pipe(ialu_reg_shift);
15025 %}
15026
15027 // this pattern occurs in bigmath arithmetic
15028 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
15029 %{
15030 match(Set dst (AndL (ConvI2L src) mask));
15031
15032 ins_cost(INSN_COST);
15033 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
15034 ins_encode %{
15035 __ ubfm($dst$$Register, $src$$Register, 0, 31);
15036 %}
15037
15038 ins_pipe(ialu_reg_shift);
15039 %}
15040
15041 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
15042 match(Set dst (ConvL2I src));
15043
15044 ins_cost(INSN_COST);
15045 format %{ "movw $dst, $src \t// l2i" %}
15046
15047 ins_encode %{
15048 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
15049 %}
15050
15051 ins_pipe(ialu_reg);
15052 %}
15053
15054 instruct convD2F_reg(vRegF dst, vRegD src) %{
15055 match(Set dst (ConvD2F src));
15056
15057 ins_cost(INSN_COST * 5);
15058 format %{ "fcvtd $dst, $src \t// d2f" %}
15059
15060 ins_encode %{
15061 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15062 %}
15063
15064 ins_pipe(fp_d2f);
15065 %}
15066
15067 instruct convF2D_reg(vRegD dst, vRegF src) %{
15068 match(Set dst (ConvF2D src));
15069
15070 ins_cost(INSN_COST * 5);
15071 format %{ "fcvts $dst, $src \t// f2d" %}
15072
15073 ins_encode %{
15074 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15075 %}
15076
15077 ins_pipe(fp_f2d);
15078 %}
15079
15080 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15081 match(Set dst (ConvF2I src));
15082
15083 ins_cost(INSN_COST * 5);
15084 format %{ "fcvtzsw $dst, $src \t// f2i" %}
15085
15086 ins_encode %{
15087 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15088 %}
15089
15090 ins_pipe(fp_f2i);
15091 %}
15092
15093 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
15094 match(Set dst (ConvF2L src));
15095
15096 ins_cost(INSN_COST * 5);
15097 format %{ "fcvtzs $dst, $src \t// f2l" %}
15098
15099 ins_encode %{
15100 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15101 %}
15102
15103 ins_pipe(fp_f2l);
15104 %}
15105
15106 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
15107 match(Set dst (ConvF2HF src));
15108 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
15109 "smov $dst, $tmp\t# move result from $tmp to $dst"
15110 %}
15111 effect(TEMP tmp);
15112 ins_encode %{
15113 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
15114 %}
15115 ins_pipe(pipe_slow);
15116 %}
15117
15118 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
15119 match(Set dst (ConvHF2F src));
15120 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
15121 "fcvt $dst, $tmp\t# convert half to single precision"
15122 %}
15123 effect(TEMP tmp);
15124 ins_encode %{
15125 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
15126 %}
15127 ins_pipe(pipe_slow);
15128 %}
15129
15130 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
15131 match(Set dst (ConvI2F src));
15132
15133 ins_cost(INSN_COST * 5);
15134 format %{ "scvtfws $dst, $src \t// i2f" %}
15135
15136 ins_encode %{
15137 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15138 %}
15139
15140 ins_pipe(fp_i2f);
15141 %}
15142
15143 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15144 match(Set dst (ConvL2F src));
15145
15146 ins_cost(INSN_COST * 5);
15147 format %{ "scvtfs $dst, $src \t// l2f" %}
15148
15149 ins_encode %{
15150 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15151 %}
15152
15153 ins_pipe(fp_l2f);
15154 %}
15155
15156 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15157 match(Set dst (ConvD2I src));
15158
15159 ins_cost(INSN_COST * 5);
15160 format %{ "fcvtzdw $dst, $src \t// d2i" %}
15161
15162 ins_encode %{
15163 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15164 %}
15165
15166 ins_pipe(fp_d2i);
15167 %}
15168
15169 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15170 match(Set dst (ConvD2L src));
15171
15172 ins_cost(INSN_COST * 5);
15173 format %{ "fcvtzd $dst, $src \t// d2l" %}
15174
15175 ins_encode %{
15176 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15177 %}
15178
15179 ins_pipe(fp_d2l);
15180 %}
15181
15182 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15183 match(Set dst (ConvI2D src));
15184
15185 ins_cost(INSN_COST * 5);
15186 format %{ "scvtfwd $dst, $src \t// i2d" %}
15187
15188 ins_encode %{
15189 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15190 %}
15191
15192 ins_pipe(fp_i2d);
15193 %}
15194
15195 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15196 match(Set dst (ConvL2D src));
15197
15198 ins_cost(INSN_COST * 5);
15199 format %{ "scvtfd $dst, $src \t// l2d" %}
15200
15201 ins_encode %{
15202 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15203 %}
15204
15205 ins_pipe(fp_l2d);
15206 %}
15207
15208 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
15209 %{
15210 match(Set dst (RoundD src));
15211 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15212 format %{ "java_round_double $dst,$src"%}
15213 ins_encode %{
15214 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
15215 as_FloatRegister($ftmp$$reg));
15216 %}
15217 ins_pipe(pipe_slow);
15218 %}
15219
15220 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
15221 %{
15222 match(Set dst (RoundF src));
15223 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15224 format %{ "java_round_float $dst,$src"%}
15225 ins_encode %{
15226 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
15227 as_FloatRegister($ftmp$$reg));
15228 %}
15229 ins_pipe(pipe_slow);
15230 %}
15231
15232 // stack <-> reg and reg <-> reg shuffles with no conversion
15233
15234 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15235
15236 match(Set dst (MoveF2I src));
15237
15238 effect(DEF dst, USE src);
15239
15240 ins_cost(4 * INSN_COST);
15241
15242 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15243
15244 ins_encode %{
15245 __ ldrw($dst$$Register, Address(sp, $src$$disp));
15246 %}
15247
15248 ins_pipe(iload_reg_reg);
15249
15250 %}
15251
15252 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15253
15254 match(Set dst (MoveI2F src));
15255
15256 effect(DEF dst, USE src);
15257
15258 ins_cost(4 * INSN_COST);
15259
15260 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15261
15262 ins_encode %{
15263 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15264 %}
15265
15266 ins_pipe(pipe_class_memory);
15267
15268 %}
15269
15270 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15271
15272 match(Set dst (MoveD2L src));
15273
15274 effect(DEF dst, USE src);
15275
15276 ins_cost(4 * INSN_COST);
15277
15278 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15279
15280 ins_encode %{
15281 __ ldr($dst$$Register, Address(sp, $src$$disp));
15282 %}
15283
15284 ins_pipe(iload_reg_reg);
15285
15286 %}
15287
15288 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15289
15290 match(Set dst (MoveL2D src));
15291
15292 effect(DEF dst, USE src);
15293
15294 ins_cost(4 * INSN_COST);
15295
15296 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15297
15298 ins_encode %{
15299 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15300 %}
15301
15302 ins_pipe(pipe_class_memory);
15303
15304 %}
15305
15306 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15307
15308 match(Set dst (MoveF2I src));
15309
15310 effect(DEF dst, USE src);
15311
15312 ins_cost(INSN_COST);
15313
15314 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15315
15316 ins_encode %{
15317 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15318 %}
15319
15320 ins_pipe(pipe_class_memory);
15321
15322 %}
15323
15324 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15325
15326 match(Set dst (MoveI2F src));
15327
15328 effect(DEF dst, USE src);
15329
15330 ins_cost(INSN_COST);
15331
15332 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15333
15334 ins_encode %{
15335 __ strw($src$$Register, Address(sp, $dst$$disp));
15336 %}
15337
15338 ins_pipe(istore_reg_reg);
15339
15340 %}
15341
15342 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15343
15344 match(Set dst (MoveD2L src));
15345
15346 effect(DEF dst, USE src);
15347
15348 ins_cost(INSN_COST);
15349
15350 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15351
15352 ins_encode %{
15353 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15354 %}
15355
15356 ins_pipe(pipe_class_memory);
15357
15358 %}
15359
15360 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15361
15362 match(Set dst (MoveL2D src));
15363
15364 effect(DEF dst, USE src);
15365
15366 ins_cost(INSN_COST);
15367
15368 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15369
15370 ins_encode %{
15371 __ str($src$$Register, Address(sp, $dst$$disp));
15372 %}
15373
15374 ins_pipe(istore_reg_reg);
15375
15376 %}
15377
15378 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15379
15380 match(Set dst (MoveF2I src));
15381
15382 effect(DEF dst, USE src);
15383
15384 ins_cost(INSN_COST);
15385
15386 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15387
15388 ins_encode %{
15389 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15390 %}
15391
15392 ins_pipe(fp_f2i);
15393
15394 %}
15395
15396 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15397
15398 match(Set dst (MoveI2F src));
15399
15400 effect(DEF dst, USE src);
15401
15402 ins_cost(INSN_COST);
15403
15404 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15405
15406 ins_encode %{
15407 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15408 %}
15409
15410 ins_pipe(fp_i2f);
15411
15412 %}
15413
15414 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15415
15416 match(Set dst (MoveD2L src));
15417
15418 effect(DEF dst, USE src);
15419
15420 ins_cost(INSN_COST);
15421
15422 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15423
15424 ins_encode %{
15425 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15426 %}
15427
15428 ins_pipe(fp_d2l);
15429
15430 %}
15431
15432 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15433
15434 match(Set dst (MoveL2D src));
15435
15436 effect(DEF dst, USE src);
15437
15438 ins_cost(INSN_COST);
15439
15440 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15441
15442 ins_encode %{
15443 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15444 %}
15445
15446 ins_pipe(fp_l2d);
15447
15448 %}
15449
15450 // ============================================================================
15451 // clearing of an array
15452
15453 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15454 %{
15455 match(Set dummy (ClearArray cnt base));
15456 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15457
15458 ins_cost(4 * INSN_COST);
15459 format %{ "ClearArray $cnt, $base" %}
15460
15461 ins_encode %{
15462 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15463 if (tpc == NULL) {
15464 ciEnv::current()->record_failure("CodeCache is full");
15465 return;
15466 }
15467 %}
15468
15469 ins_pipe(pipe_class_memory);
15470 %}
15471
15472 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15473 %{
15474 predicate((uint64_t)n->in(2)->get_long()
15475 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15476 match(Set dummy (ClearArray cnt base));
15477 effect(TEMP temp, USE_KILL base, KILL cr);
15478
15479 ins_cost(4 * INSN_COST);
15480 format %{ "ClearArray $cnt, $base" %}
15481
15482 ins_encode %{
15483 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15484 if (tpc == NULL) {
15485 ciEnv::current()->record_failure("CodeCache is full");
15486 return;
15487 }
15488 %}
15489
15490 ins_pipe(pipe_class_memory);
15491 %}
15492
15493 // ============================================================================
15494 // Overflow Math Instructions
15495
15496 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15497 %{
15498 match(Set cr (OverflowAddI op1 op2));
15499
15500 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15501 ins_cost(INSN_COST);
15502 ins_encode %{
15503 __ cmnw($op1$$Register, $op2$$Register);
15504 %}
15505
15506 ins_pipe(icmp_reg_reg);
15507 %}
15508
15509 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15510 %{
15511 match(Set cr (OverflowAddI op1 op2));
15512
15513 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15514 ins_cost(INSN_COST);
15515 ins_encode %{
15516 __ cmnw($op1$$Register, $op2$$constant);
15517 %}
15518
15519 ins_pipe(icmp_reg_imm);
15520 %}
15521
15522 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15523 %{
15524 match(Set cr (OverflowAddL op1 op2));
15525
15526 format %{ "cmn $op1, $op2\t# overflow check long" %}
15527 ins_cost(INSN_COST);
15528 ins_encode %{
15529 __ cmn($op1$$Register, $op2$$Register);
15530 %}
15531
15532 ins_pipe(icmp_reg_reg);
15533 %}
15534
15535 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15536 %{
15537 match(Set cr (OverflowAddL op1 op2));
15538
15539 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15540 ins_cost(INSN_COST);
15541 ins_encode %{
15542 __ adds(zr, $op1$$Register, $op2$$constant);
15543 %}
15544
15545 ins_pipe(icmp_reg_imm);
15546 %}
15547
15548 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15549 %{
15550 match(Set cr (OverflowSubI op1 op2));
15551
15552 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15553 ins_cost(INSN_COST);
15554 ins_encode %{
15555 __ cmpw($op1$$Register, $op2$$Register);
15556 %}
15557
15558 ins_pipe(icmp_reg_reg);
15559 %}
15560
15561 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15562 %{
15563 match(Set cr (OverflowSubI op1 op2));
15564
15565 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15566 ins_cost(INSN_COST);
15567 ins_encode %{
15568 __ cmpw($op1$$Register, $op2$$constant);
15569 %}
15570
15571 ins_pipe(icmp_reg_imm);
15572 %}
15573
15574 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15575 %{
15576 match(Set cr (OverflowSubL op1 op2));
15577
15578 format %{ "cmp $op1, $op2\t# overflow check long" %}
15579 ins_cost(INSN_COST);
15580 ins_encode %{
15581 __ cmp($op1$$Register, $op2$$Register);
15582 %}
15583
15584 ins_pipe(icmp_reg_reg);
15585 %}
15586
15587 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15588 %{
15589 match(Set cr (OverflowSubL op1 op2));
15590
15591 format %{ "cmp $op1, $op2\t# overflow check long" %}
15592 ins_cost(INSN_COST);
15593 ins_encode %{
15594 __ subs(zr, $op1$$Register, $op2$$constant);
15595 %}
15596
15597 ins_pipe(icmp_reg_imm);
15598 %}
15599
15600 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15601 %{
15602 match(Set cr (OverflowSubI zero op1));
15603
15604 format %{ "cmpw zr, $op1\t# overflow check int" %}
15605 ins_cost(INSN_COST);
15606 ins_encode %{
15607 __ cmpw(zr, $op1$$Register);
15608 %}
15609
15610 ins_pipe(icmp_reg_imm);
15611 %}
15612
15613 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15614 %{
15615 match(Set cr (OverflowSubL zero op1));
15616
15617 format %{ "cmp zr, $op1\t# overflow check long" %}
15618 ins_cost(INSN_COST);
15619 ins_encode %{
15620 __ cmp(zr, $op1$$Register);
15621 %}
15622
15623 ins_pipe(icmp_reg_imm);
15624 %}
15625
15626 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15627 %{
15628 match(Set cr (OverflowMulI op1 op2));
15629
15630 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15631 "cmp rscratch1, rscratch1, sxtw\n\t"
15632 "movw rscratch1, #0x80000000\n\t"
15633 "cselw rscratch1, rscratch1, zr, NE\n\t"
15634 "cmpw rscratch1, #1" %}
15635 ins_cost(5 * INSN_COST);
15636 ins_encode %{
15637 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15638 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15639 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15640 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15641 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15642 %}
15643
15644 ins_pipe(pipe_slow);
15645 %}
15646
15647 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15648 %{
15649 match(If cmp (OverflowMulI op1 op2));
15650 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15651 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15652 effect(USE labl, KILL cr);
15653
15654 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15655 "cmp rscratch1, rscratch1, sxtw\n\t"
15656 "b$cmp $labl" %}
15657 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15658 ins_encode %{
15659 Label* L = $labl$$label;
15660 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15661 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15662 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15663 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15664 %}
15665
15666 ins_pipe(pipe_serial);
15667 %}
15668
15669 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15670 %{
15671 match(Set cr (OverflowMulL op1 op2));
15672
15673 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15674 "smulh rscratch2, $op1, $op2\n\t"
15675 "cmp rscratch2, rscratch1, ASR #63\n\t"
15676 "movw rscratch1, #0x80000000\n\t"
15677 "cselw rscratch1, rscratch1, zr, NE\n\t"
15678 "cmpw rscratch1, #1" %}
15679 ins_cost(6 * INSN_COST);
15680 ins_encode %{
15681 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15682 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15683 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15684 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15685 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15686 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15687 %}
15688
15689 ins_pipe(pipe_slow);
15690 %}
15691
15692 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15693 %{
15694 match(If cmp (OverflowMulL op1 op2));
15695 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15696 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15697 effect(USE labl, KILL cr);
15698
15699 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15700 "smulh rscratch2, $op1, $op2\n\t"
15701 "cmp rscratch2, rscratch1, ASR #63\n\t"
15702 "b$cmp $labl" %}
15703 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15704 ins_encode %{
15705 Label* L = $labl$$label;
15706 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15707 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15708 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15709 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15710 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15711 %}
15712
15713 ins_pipe(pipe_serial);
15714 %}
15715
15716 // ============================================================================
15717 // Compare Instructions
15718
15719 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15720 %{
15721 match(Set cr (CmpI op1 op2));
15722
15723 effect(DEF cr, USE op1, USE op2);
15724
15725 ins_cost(INSN_COST);
15726 format %{ "cmpw $op1, $op2" %}
15727
15728 ins_encode(aarch64_enc_cmpw(op1, op2));
15729
15730 ins_pipe(icmp_reg_reg);
15731 %}
15732
15733 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15734 %{
15735 match(Set cr (CmpI op1 zero));
15736
15737 effect(DEF cr, USE op1);
15738
15739 ins_cost(INSN_COST);
15740 format %{ "cmpw $op1, 0" %}
15741
15742 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15743
15744 ins_pipe(icmp_reg_imm);
15745 %}
15746
15747 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15748 %{
15749 match(Set cr (CmpI op1 op2));
15750
15751 effect(DEF cr, USE op1);
15752
15753 ins_cost(INSN_COST);
15754 format %{ "cmpw $op1, $op2" %}
15755
15756 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15757
15758 ins_pipe(icmp_reg_imm);
15759 %}
15760
15761 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15762 %{
15763 match(Set cr (CmpI op1 op2));
15764
15765 effect(DEF cr, USE op1);
15766
15767 ins_cost(INSN_COST * 2);
15768 format %{ "cmpw $op1, $op2" %}
15769
15770 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15771
15772 ins_pipe(icmp_reg_imm);
15773 %}
15774
15775 // Unsigned compare Instructions; really, same as signed compare
15776 // except it should only be used to feed an If or a CMovI which takes a
15777 // cmpOpU.
15778
15779 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15780 %{
15781 match(Set cr (CmpU op1 op2));
15782
15783 effect(DEF cr, USE op1, USE op2);
15784
15785 ins_cost(INSN_COST);
15786 format %{ "cmpw $op1, $op2\t# unsigned" %}
15787
15788 ins_encode(aarch64_enc_cmpw(op1, op2));
15789
15790 ins_pipe(icmp_reg_reg);
15791 %}
15792
15793 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15794 %{
15795 match(Set cr (CmpU op1 zero));
15796
15797 effect(DEF cr, USE op1);
15798
15799 ins_cost(INSN_COST);
15800 format %{ "cmpw $op1, #0\t# unsigned" %}
15801
15802 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15803
15804 ins_pipe(icmp_reg_imm);
15805 %}
15806
15807 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15808 %{
15809 match(Set cr (CmpU op1 op2));
15810
15811 effect(DEF cr, USE op1);
15812
15813 ins_cost(INSN_COST);
15814 format %{ "cmpw $op1, $op2\t# unsigned" %}
15815
15816 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15817
15818 ins_pipe(icmp_reg_imm);
15819 %}
15820
15821 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15822 %{
15823 match(Set cr (CmpU op1 op2));
15824
15825 effect(DEF cr, USE op1);
15826
15827 ins_cost(INSN_COST * 2);
15828 format %{ "cmpw $op1, $op2\t# unsigned" %}
15829
15830 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15831
15832 ins_pipe(icmp_reg_imm);
15833 %}
15834
15835 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15836 %{
15837 match(Set cr (CmpL op1 op2));
15838
15839 effect(DEF cr, USE op1, USE op2);
15840
15841 ins_cost(INSN_COST);
15842 format %{ "cmp $op1, $op2" %}
15843
15844 ins_encode(aarch64_enc_cmp(op1, op2));
15845
15846 ins_pipe(icmp_reg_reg);
15847 %}
15848
15849 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15850 %{
15851 match(Set cr (CmpL op1 zero));
15852
15853 effect(DEF cr, USE op1);
15854
15855 ins_cost(INSN_COST);
15856 format %{ "tst $op1" %}
15857
15858 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15859
15860 ins_pipe(icmp_reg_imm);
15861 %}
15862
15863 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15864 %{
15865 match(Set cr (CmpL op1 op2));
15866
15867 effect(DEF cr, USE op1);
15868
15869 ins_cost(INSN_COST);
15870 format %{ "cmp $op1, $op2" %}
15871
15872 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15873
15874 ins_pipe(icmp_reg_imm);
15875 %}
15876
15877 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15878 %{
15879 match(Set cr (CmpL op1 op2));
15880
15881 effect(DEF cr, USE op1);
15882
15883 ins_cost(INSN_COST * 2);
15884 format %{ "cmp $op1, $op2" %}
15885
15886 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15887
15888 ins_pipe(icmp_reg_imm);
15889 %}
15890
15891 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15892 %{
15893 match(Set cr (CmpUL op1 op2));
15894
15895 effect(DEF cr, USE op1, USE op2);
15896
15897 ins_cost(INSN_COST);
15898 format %{ "cmp $op1, $op2" %}
15899
15900 ins_encode(aarch64_enc_cmp(op1, op2));
15901
15902 ins_pipe(icmp_reg_reg);
15903 %}
15904
15905 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15906 %{
15907 match(Set cr (CmpUL op1 zero));
15908
15909 effect(DEF cr, USE op1);
15910
15911 ins_cost(INSN_COST);
15912 format %{ "tst $op1" %}
15913
15914 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15915
15916 ins_pipe(icmp_reg_imm);
15917 %}
15918
15919 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15920 %{
15921 match(Set cr (CmpUL op1 op2));
15922
15923 effect(DEF cr, USE op1);
15924
15925 ins_cost(INSN_COST);
15926 format %{ "cmp $op1, $op2" %}
15927
15928 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15929
15930 ins_pipe(icmp_reg_imm);
15931 %}
15932
15933 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15934 %{
15935 match(Set cr (CmpUL op1 op2));
15936
15937 effect(DEF cr, USE op1);
15938
15939 ins_cost(INSN_COST * 2);
15940 format %{ "cmp $op1, $op2" %}
15941
15942 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15943
15944 ins_pipe(icmp_reg_imm);
15945 %}
15946
15947 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15948 %{
15949 match(Set cr (CmpP op1 op2));
15950
15951 effect(DEF cr, USE op1, USE op2);
15952
15953 ins_cost(INSN_COST);
15954 format %{ "cmp $op1, $op2\t // ptr" %}
15955
15956 ins_encode(aarch64_enc_cmpp(op1, op2));
15957
15958 ins_pipe(icmp_reg_reg);
15959 %}
15960
15961 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15962 %{
15963 match(Set cr (CmpN op1 op2));
15964
15965 effect(DEF cr, USE op1, USE op2);
15966
15967 ins_cost(INSN_COST);
15968 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15969
15970 ins_encode(aarch64_enc_cmpn(op1, op2));
15971
15972 ins_pipe(icmp_reg_reg);
15973 %}
15974
15975 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15976 %{
15977 match(Set cr (CmpP op1 zero));
15978
15979 effect(DEF cr, USE op1, USE zero);
15980
15981 ins_cost(INSN_COST);
15982 format %{ "cmp $op1, 0\t // ptr" %}
15983
15984 ins_encode(aarch64_enc_testp(op1));
15985
15986 ins_pipe(icmp_reg_imm);
15987 %}
15988
15989 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15990 %{
15991 match(Set cr (CmpN op1 zero));
15992
15993 effect(DEF cr, USE op1, USE zero);
15994
15995 ins_cost(INSN_COST);
15996 format %{ "cmp $op1, 0\t // compressed ptr" %}
15997
15998 ins_encode(aarch64_enc_testn(op1));
15999
16000 ins_pipe(icmp_reg_imm);
16001 %}
16002
16003 // FP comparisons
16004 //
16005 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
16006 // using normal cmpOp. See declaration of rFlagsReg for details.
16007
16008 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
16009 %{
16010 match(Set cr (CmpF src1 src2));
16011
16012 ins_cost(3 * INSN_COST);
16013 format %{ "fcmps $src1, $src2" %}
16014
16015 ins_encode %{
16016 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16017 %}
16018
16019 ins_pipe(pipe_class_compare);
16020 %}
16021
16022 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
16023 %{
16024 match(Set cr (CmpF src1 src2));
16025
16026 ins_cost(3 * INSN_COST);
16027 format %{ "fcmps $src1, 0.0" %}
16028
16029 ins_encode %{
16030 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
16031 %}
16032
16033 ins_pipe(pipe_class_compare);
16034 %}
16035 // FROM HERE
16036
16037 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
16038 %{
16039 match(Set cr (CmpD src1 src2));
16040
16041 ins_cost(3 * INSN_COST);
16042 format %{ "fcmpd $src1, $src2" %}
16043
16044 ins_encode %{
16045 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16046 %}
16047
16048 ins_pipe(pipe_class_compare);
16049 %}
16050
16051 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
16052 %{
16053 match(Set cr (CmpD src1 src2));
16054
16055 ins_cost(3 * INSN_COST);
16056 format %{ "fcmpd $src1, 0.0" %}
16057
16058 ins_encode %{
16059 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
16060 %}
16061
16062 ins_pipe(pipe_class_compare);
16063 %}
16064
16065 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
16066 %{
16067 match(Set dst (CmpF3 src1 src2));
16068 effect(KILL cr);
16069
16070 ins_cost(5 * INSN_COST);
16071 format %{ "fcmps $src1, $src2\n\t"
16072 "csinvw($dst, zr, zr, eq\n\t"
16073 "csnegw($dst, $dst, $dst, lt)"
16074 %}
16075
16076 ins_encode %{
16077 Label done;
16078 FloatRegister s1 = as_FloatRegister($src1$$reg);
16079 FloatRegister s2 = as_FloatRegister($src2$$reg);
16080 Register d = as_Register($dst$$reg);
16081 __ fcmps(s1, s2);
16082 // installs 0 if EQ else -1
16083 __ csinvw(d, zr, zr, Assembler::EQ);
16084 // keeps -1 if less or unordered else installs 1
16085 __ csnegw(d, d, d, Assembler::LT);
16086 __ bind(done);
16087 %}
16088
16089 ins_pipe(pipe_class_default);
16090
16091 %}
16092
16093 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
16094 %{
16095 match(Set dst (CmpD3 src1 src2));
16096 effect(KILL cr);
16097
16098 ins_cost(5 * INSN_COST);
16099 format %{ "fcmpd $src1, $src2\n\t"
16100 "csinvw($dst, zr, zr, eq\n\t"
16101 "csnegw($dst, $dst, $dst, lt)"
16102 %}
16103
16104 ins_encode %{
16105 Label done;
16106 FloatRegister s1 = as_FloatRegister($src1$$reg);
16107 FloatRegister s2 = as_FloatRegister($src2$$reg);
16108 Register d = as_Register($dst$$reg);
16109 __ fcmpd(s1, s2);
16110 // installs 0 if EQ else -1
16111 __ csinvw(d, zr, zr, Assembler::EQ);
16112 // keeps -1 if less or unordered else installs 1
16113 __ csnegw(d, d, d, Assembler::LT);
16114 __ bind(done);
16115 %}
16116 ins_pipe(pipe_class_default);
16117
16118 %}
16119
16120 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
16121 %{
16122 match(Set dst (CmpF3 src1 zero));
16123 effect(KILL cr);
16124
16125 ins_cost(5 * INSN_COST);
16126 format %{ "fcmps $src1, 0.0\n\t"
16127 "csinvw($dst, zr, zr, eq\n\t"
16128 "csnegw($dst, $dst, $dst, lt)"
16129 %}
16130
16131 ins_encode %{
16132 Label done;
16133 FloatRegister s1 = as_FloatRegister($src1$$reg);
16134 Register d = as_Register($dst$$reg);
16135 __ fcmps(s1, 0.0);
16136 // installs 0 if EQ else -1
16137 __ csinvw(d, zr, zr, Assembler::EQ);
16138 // keeps -1 if less or unordered else installs 1
16139 __ csnegw(d, d, d, Assembler::LT);
16140 __ bind(done);
16141 %}
16142
16143 ins_pipe(pipe_class_default);
16144
16145 %}
16146
16147 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16148 %{
16149 match(Set dst (CmpD3 src1 zero));
16150 effect(KILL cr);
16151
16152 ins_cost(5 * INSN_COST);
16153 format %{ "fcmpd $src1, 0.0\n\t"
16154 "csinvw($dst, zr, zr, eq\n\t"
16155 "csnegw($dst, $dst, $dst, lt)"
16156 %}
16157
16158 ins_encode %{
16159 Label done;
16160 FloatRegister s1 = as_FloatRegister($src1$$reg);
16161 Register d = as_Register($dst$$reg);
16162 __ fcmpd(s1, 0.0);
16163 // installs 0 if EQ else -1
16164 __ csinvw(d, zr, zr, Assembler::EQ);
16165 // keeps -1 if less or unordered else installs 1
16166 __ csnegw(d, d, d, Assembler::LT);
16167 __ bind(done);
16168 %}
16169 ins_pipe(pipe_class_default);
16170
16171 %}
16172
16173 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16174 %{
16175 match(Set dst (CmpLTMask p q));
16176 effect(KILL cr);
16177
16178 ins_cost(3 * INSN_COST);
16179
16180 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16181 "csetw $dst, lt\n\t"
16182 "subw $dst, zr, $dst"
16183 %}
16184
16185 ins_encode %{
16186 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16187 __ csetw(as_Register($dst$$reg), Assembler::LT);
16188 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16189 %}
16190
16191 ins_pipe(ialu_reg_reg);
16192 %}
16193
16194 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16195 %{
16196 match(Set dst (CmpLTMask src zero));
16197 effect(KILL cr);
16198
16199 ins_cost(INSN_COST);
16200
16201 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16202
16203 ins_encode %{
16204 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16205 %}
16206
16207 ins_pipe(ialu_reg_shift);
16208 %}
16209
16210 // ============================================================================
16211 // Max and Min
16212
16213 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
16214
16215 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
16216 %{
16217 effect(DEF cr, USE src);
16218 ins_cost(INSN_COST);
16219 format %{ "cmpw $src, 0" %}
16220
16221 ins_encode %{
16222 __ cmpw($src$$Register, 0);
16223 %}
16224 ins_pipe(icmp_reg_imm);
16225 %}
16226
16227 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16228 %{
16229 match(Set dst (MinI src1 src2));
16230 ins_cost(INSN_COST * 3);
16231
16232 expand %{
16233 rFlagsReg cr;
16234 compI_reg_reg(cr, src1, src2);
16235 cmovI_reg_reg_lt(dst, src1, src2, cr);
16236 %}
16237 %}
16238
16239 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16240 %{
16241 match(Set dst (MaxI src1 src2));
16242 ins_cost(INSN_COST * 3);
16243
16244 expand %{
16245 rFlagsReg cr;
16246 compI_reg_reg(cr, src1, src2);
16247 cmovI_reg_reg_gt(dst, src1, src2, cr);
16248 %}
16249 %}
16250
16251
16252 // ============================================================================
16253 // Branch Instructions
16254
16255 // Direct Branch.
16256 instruct branch(label lbl)
16257 %{
16258 match(Goto);
16259
16260 effect(USE lbl);
16261
16262 ins_cost(BRANCH_COST);
16263 format %{ "b $lbl" %}
16264
16265 ins_encode(aarch64_enc_b(lbl));
16266
16267 ins_pipe(pipe_branch);
16268 %}
16269
16270 // Conditional Near Branch
16271 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16272 %{
16273 // Same match rule as `branchConFar'.
16274 match(If cmp cr);
16275
16276 effect(USE lbl);
16277
16278 ins_cost(BRANCH_COST);
16279 // If set to 1 this indicates that the current instruction is a
16280 // short variant of a long branch. This avoids using this
16281 // instruction in first-pass matching. It will then only be used in
16282 // the `Shorten_branches' pass.
16283 // ins_short_branch(1);
16284 format %{ "b$cmp $lbl" %}
16285
16286 ins_encode(aarch64_enc_br_con(cmp, lbl));
16287
16288 ins_pipe(pipe_branch_cond);
16289 %}
16290
16291 // Conditional Near Branch Unsigned
16292 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16293 %{
16294 // Same match rule as `branchConFar'.
16295 match(If cmp cr);
16296
16297 effect(USE lbl);
16298
16299 ins_cost(BRANCH_COST);
16300 // If set to 1 this indicates that the current instruction is a
16301 // short variant of a long branch. This avoids using this
16302 // instruction in first-pass matching. It will then only be used in
16303 // the `Shorten_branches' pass.
16304 // ins_short_branch(1);
16305 format %{ "b$cmp $lbl\t# unsigned" %}
16306
16307 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16308
16309 ins_pipe(pipe_branch_cond);
16310 %}
16311
16312 // Make use of CBZ and CBNZ. These instructions, as well as being
16313 // shorter than (cmp; branch), have the additional benefit of not
16314 // killing the flags.
16315
16316 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16317 match(If cmp (CmpI op1 op2));
16318 effect(USE labl);
16319
16320 ins_cost(BRANCH_COST);
16321 format %{ "cbw$cmp $op1, $labl" %}
16322 ins_encode %{
16323 Label* L = $labl$$label;
16324 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16325 if (cond == Assembler::EQ)
16326 __ cbzw($op1$$Register, *L);
16327 else
16328 __ cbnzw($op1$$Register, *L);
16329 %}
16330 ins_pipe(pipe_cmp_branch);
16331 %}
16332
16333 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16334 match(If cmp (CmpL op1 op2));
16335 effect(USE labl);
16336
16337 ins_cost(BRANCH_COST);
16338 format %{ "cb$cmp $op1, $labl" %}
16339 ins_encode %{
16340 Label* L = $labl$$label;
16341 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16342 if (cond == Assembler::EQ)
16343 __ cbz($op1$$Register, *L);
16344 else
16345 __ cbnz($op1$$Register, *L);
16346 %}
16347 ins_pipe(pipe_cmp_branch);
16348 %}
16349
16350 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16351 match(If cmp (CmpP op1 op2));
16352 effect(USE labl);
16353
16354 ins_cost(BRANCH_COST);
16355 format %{ "cb$cmp $op1, $labl" %}
16356 ins_encode %{
16357 Label* L = $labl$$label;
16358 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16359 if (cond == Assembler::EQ)
16360 __ cbz($op1$$Register, *L);
16361 else
16362 __ cbnz($op1$$Register, *L);
16363 %}
16364 ins_pipe(pipe_cmp_branch);
16365 %}
16366
16367 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16368 match(If cmp (CmpN op1 op2));
16369 effect(USE labl);
16370
16371 ins_cost(BRANCH_COST);
16372 format %{ "cbw$cmp $op1, $labl" %}
16373 ins_encode %{
16374 Label* L = $labl$$label;
16375 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16376 if (cond == Assembler::EQ)
16377 __ cbzw($op1$$Register, *L);
16378 else
16379 __ cbnzw($op1$$Register, *L);
16380 %}
16381 ins_pipe(pipe_cmp_branch);
16382 %}
16383
16384 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16385 match(If cmp (CmpP (DecodeN oop) zero));
16386 effect(USE labl);
16387
16388 ins_cost(BRANCH_COST);
16389 format %{ "cb$cmp $oop, $labl" %}
16390 ins_encode %{
16391 Label* L = $labl$$label;
16392 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16393 if (cond == Assembler::EQ)
16394 __ cbzw($oop$$Register, *L);
16395 else
16396 __ cbnzw($oop$$Register, *L);
16397 %}
16398 ins_pipe(pipe_cmp_branch);
16399 %}
16400
16401 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16402 match(If cmp (CmpU op1 op2));
16403 effect(USE labl);
16404
16405 ins_cost(BRANCH_COST);
16406 format %{ "cbw$cmp $op1, $labl" %}
16407 ins_encode %{
16408 Label* L = $labl$$label;
16409 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16410 if (cond == Assembler::EQ || cond == Assembler::LS)
16411 __ cbzw($op1$$Register, *L);
16412 else
16413 __ cbnzw($op1$$Register, *L);
16414 %}
16415 ins_pipe(pipe_cmp_branch);
16416 %}
16417
16418 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16419 match(If cmp (CmpUL op1 op2));
16420 effect(USE labl);
16421
16422 ins_cost(BRANCH_COST);
16423 format %{ "cb$cmp $op1, $labl" %}
16424 ins_encode %{
16425 Label* L = $labl$$label;
16426 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16427 if (cond == Assembler::EQ || cond == Assembler::LS)
16428 __ cbz($op1$$Register, *L);
16429 else
16430 __ cbnz($op1$$Register, *L);
16431 %}
16432 ins_pipe(pipe_cmp_branch);
16433 %}
16434
16435 // Test bit and Branch
16436
16437 // Patterns for short (< 32KiB) variants
16438 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16439 match(If cmp (CmpL op1 op2));
16440 effect(USE labl);
16441
16442 ins_cost(BRANCH_COST);
16443 format %{ "cb$cmp $op1, $labl # long" %}
16444 ins_encode %{
16445 Label* L = $labl$$label;
16446 Assembler::Condition cond =
16447 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16448 __ tbr(cond, $op1$$Register, 63, *L);
16449 %}
16450 ins_pipe(pipe_cmp_branch);
16451 ins_short_branch(1);
16452 %}
16453
16454 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16455 match(If cmp (CmpI op1 op2));
16456 effect(USE labl);
16457
16458 ins_cost(BRANCH_COST);
16459 format %{ "cb$cmp $op1, $labl # int" %}
16460 ins_encode %{
16461 Label* L = $labl$$label;
16462 Assembler::Condition cond =
16463 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16464 __ tbr(cond, $op1$$Register, 31, *L);
16465 %}
16466 ins_pipe(pipe_cmp_branch);
16467 ins_short_branch(1);
16468 %}
16469
16470 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16471 match(If cmp (CmpL (AndL op1 op2) op3));
16472 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16473 effect(USE labl);
16474
16475 ins_cost(BRANCH_COST);
16476 format %{ "tb$cmp $op1, $op2, $labl" %}
16477 ins_encode %{
16478 Label* L = $labl$$label;
16479 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16480 int bit = exact_log2_long($op2$$constant);
16481 __ tbr(cond, $op1$$Register, bit, *L);
16482 %}
16483 ins_pipe(pipe_cmp_branch);
16484 ins_short_branch(1);
16485 %}
16486
16487 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16488 match(If cmp (CmpI (AndI op1 op2) op3));
16489 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16490 effect(USE labl);
16491
16492 ins_cost(BRANCH_COST);
16493 format %{ "tb$cmp $op1, $op2, $labl" %}
16494 ins_encode %{
16495 Label* L = $labl$$label;
16496 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16497 int bit = exact_log2((juint)$op2$$constant);
16498 __ tbr(cond, $op1$$Register, bit, *L);
16499 %}
16500 ins_pipe(pipe_cmp_branch);
16501 ins_short_branch(1);
16502 %}
16503
16504 // And far variants
16505 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16506 match(If cmp (CmpL op1 op2));
16507 effect(USE labl);
16508
16509 ins_cost(BRANCH_COST);
16510 format %{ "cb$cmp $op1, $labl # long" %}
16511 ins_encode %{
16512 Label* L = $labl$$label;
16513 Assembler::Condition cond =
16514 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16515 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16516 %}
16517 ins_pipe(pipe_cmp_branch);
16518 %}
16519
16520 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16521 match(If cmp (CmpI op1 op2));
16522 effect(USE labl);
16523
16524 ins_cost(BRANCH_COST);
16525 format %{ "cb$cmp $op1, $labl # int" %}
16526 ins_encode %{
16527 Label* L = $labl$$label;
16528 Assembler::Condition cond =
16529 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16530 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16531 %}
16532 ins_pipe(pipe_cmp_branch);
16533 %}
16534
16535 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16536 match(If cmp (CmpL (AndL op1 op2) op3));
16537 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16538 effect(USE labl);
16539
16540 ins_cost(BRANCH_COST);
16541 format %{ "tb$cmp $op1, $op2, $labl" %}
16542 ins_encode %{
16543 Label* L = $labl$$label;
16544 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16545 int bit = exact_log2_long($op2$$constant);
16546 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16547 %}
16548 ins_pipe(pipe_cmp_branch);
16549 %}
16550
16551 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16552 match(If cmp (CmpI (AndI op1 op2) op3));
16553 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16554 effect(USE labl);
16555
16556 ins_cost(BRANCH_COST);
16557 format %{ "tb$cmp $op1, $op2, $labl" %}
16558 ins_encode %{
16559 Label* L = $labl$$label;
16560 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16561 int bit = exact_log2((juint)$op2$$constant);
16562 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16563 %}
16564 ins_pipe(pipe_cmp_branch);
16565 %}
16566
16567 // Test bits
16568
16569 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16570 match(Set cr (CmpL (AndL op1 op2) op3));
16571 predicate(Assembler::operand_valid_for_logical_immediate
16572 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16573
16574 ins_cost(INSN_COST);
16575 format %{ "tst $op1, $op2 # long" %}
16576 ins_encode %{
16577 __ tst($op1$$Register, $op2$$constant);
16578 %}
16579 ins_pipe(ialu_reg_reg);
16580 %}
16581
16582 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16583 match(Set cr (CmpI (AndI op1 op2) op3));
16584 predicate(Assembler::operand_valid_for_logical_immediate
16585 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16586
16587 ins_cost(INSN_COST);
16588 format %{ "tst $op1, $op2 # int" %}
16589 ins_encode %{
16590 __ tstw($op1$$Register, $op2$$constant);
16591 %}
16592 ins_pipe(ialu_reg_reg);
16593 %}
16594
16595 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16596 match(Set cr (CmpL (AndL op1 op2) op3));
16597
16598 ins_cost(INSN_COST);
16599 format %{ "tst $op1, $op2 # long" %}
16600 ins_encode %{
16601 __ tst($op1$$Register, $op2$$Register);
16602 %}
16603 ins_pipe(ialu_reg_reg);
16604 %}
16605
16606 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16607 match(Set cr (CmpI (AndI op1 op2) op3));
16608
16609 ins_cost(INSN_COST);
16610 format %{ "tstw $op1, $op2 # int" %}
16611 ins_encode %{
16612 __ tstw($op1$$Register, $op2$$Register);
16613 %}
16614 ins_pipe(ialu_reg_reg);
16615 %}
16616
16617
16618 // Conditional Far Branch
16619 // Conditional Far Branch Unsigned
16620 // TODO: fixme
16621
16622 // counted loop end branch near
16623 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16624 %{
16625 match(CountedLoopEnd cmp cr);
16626
16627 effect(USE lbl);
16628
16629 ins_cost(BRANCH_COST);
16630 // short variant.
16631 // ins_short_branch(1);
16632 format %{ "b$cmp $lbl \t// counted loop end" %}
16633
16634 ins_encode(aarch64_enc_br_con(cmp, lbl));
16635
16636 ins_pipe(pipe_branch);
16637 %}
16638
16639 // counted loop end branch far
16640 // TODO: fixme
16641
16642 // ============================================================================
16643 // inlined locking and unlocking
16644
16645 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16646 %{
16647 match(Set cr (FastLock object box));
16648 effect(TEMP tmp, TEMP tmp2);
16649
16650 // TODO
16651 // identify correct cost
16652 ins_cost(5 * INSN_COST);
16653 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16654
16655 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16656
16657 ins_pipe(pipe_serial);
16658 %}
16659
16660 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16661 %{
16662 match(Set cr (FastUnlock object box));
16663 effect(TEMP tmp, TEMP tmp2);
16664
16665 ins_cost(5 * INSN_COST);
16666 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16667
16668 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16669
16670 ins_pipe(pipe_serial);
16671 %}
16672
16673
16674 // ============================================================================
16675 // Safepoint Instructions
16676
16677 // TODO
16678 // provide a near and far version of this code
16679
16680 instruct safePoint(rFlagsReg cr, iRegP poll)
16681 %{
16682 match(SafePoint poll);
16683 effect(KILL cr);
16684
16685 format %{
16686 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16687 %}
16688 ins_encode %{
16689 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16690 %}
16691 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16692 %}
16693
16694
16695 // ============================================================================
16696 // Procedure Call/Return Instructions
16697
16698 // Call Java Static Instruction
16699
16700 instruct CallStaticJavaDirect(method meth)
16701 %{
16702 match(CallStaticJava);
16703
16704 effect(USE meth);
16705
16706 ins_cost(CALL_COST);
16707
16708 format %{ "call,static $meth \t// ==> " %}
16709
16710 ins_encode(aarch64_enc_java_static_call(meth),
16711 aarch64_enc_call_epilog);
16712
16713 ins_pipe(pipe_class_call);
16714 %}
16715
16716 // TO HERE
16717
16718 // Call Java Dynamic Instruction
16719 instruct CallDynamicJavaDirect(method meth)
16720 %{
16721 match(CallDynamicJava);
16722
16723 effect(USE meth);
16724
16725 ins_cost(CALL_COST);
16726
16727 format %{ "CALL,dynamic $meth \t// ==> " %}
16728
16729 ins_encode(aarch64_enc_java_dynamic_call(meth),
16730 aarch64_enc_call_epilog);
16731
16732 ins_pipe(pipe_class_call);
16733 %}
16734
16735 // Call Runtime Instruction
16736
16737 instruct CallRuntimeDirect(method meth)
16738 %{
16739 match(CallRuntime);
16740
16741 effect(USE meth);
16742
16743 ins_cost(CALL_COST);
16744
16745 format %{ "CALL, runtime $meth" %}
16746
16747 ins_encode( aarch64_enc_java_to_runtime(meth) );
16748
16749 ins_pipe(pipe_class_call);
16750 %}
16751
16752 // Call Runtime Instruction
16753
16754 instruct CallLeafDirect(method meth)
16755 %{
16756 match(CallLeaf);
16757
16758 effect(USE meth);
16759
16760 ins_cost(CALL_COST);
16761
16762 format %{ "CALL, runtime leaf $meth" %}
16763
16764 ins_encode( aarch64_enc_java_to_runtime(meth) );
16765
16766 ins_pipe(pipe_class_call);
16767 %}
16768
16769 // Call Runtime Instruction
16770
16771 instruct CallLeafNoFPDirect(method meth)
16772 %{
16773 match(CallLeafNoFP);
16774
16775 effect(USE meth);
16776
16777 ins_cost(CALL_COST);
16778
16779 format %{ "CALL, runtime leaf nofp $meth" %}
16780
16781 ins_encode( aarch64_enc_java_to_runtime(meth) );
16782
16783 ins_pipe(pipe_class_call);
16784 %}
16785
16786 // Tail Call; Jump from runtime stub to Java code.
16787 // Also known as an 'interprocedural jump'.
16788 // Target of jump will eventually return to caller.
16789 // TailJump below removes the return address.
16790 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16791 %{
16792 match(TailCall jump_target method_ptr);
16793
16794 ins_cost(CALL_COST);
16795
16796 format %{ "br $jump_target\t# $method_ptr holds method" %}
16797
16798 ins_encode(aarch64_enc_tail_call(jump_target));
16799
16800 ins_pipe(pipe_class_call);
16801 %}
16802
16803 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16804 %{
16805 match(TailJump jump_target ex_oop);
16806
16807 ins_cost(CALL_COST);
16808
16809 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16810
16811 ins_encode(aarch64_enc_tail_jmp(jump_target));
16812
16813 ins_pipe(pipe_class_call);
16814 %}
16815
16816 // Create exception oop: created by stack-crawling runtime code.
16817 // Created exception is now available to this handler, and is setup
16818 // just prior to jumping to this handler. No code emitted.
16819 // TODO check
16820 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16821 instruct CreateException(iRegP_R0 ex_oop)
16822 %{
16823 match(Set ex_oop (CreateEx));
16824
16825 format %{ " -- \t// exception oop; no code emitted" %}
16826
16827 size(0);
16828
16829 ins_encode( /*empty*/ );
16830
16831 ins_pipe(pipe_class_empty);
16832 %}
16833
16834 // Rethrow exception: The exception oop will come in the first
16835 // argument position. Then JUMP (not call) to the rethrow stub code.
16836 instruct RethrowException() %{
16837 match(Rethrow);
16838 ins_cost(CALL_COST);
16839
16840 format %{ "b rethrow_stub" %}
16841
16842 ins_encode( aarch64_enc_rethrow() );
16843
16844 ins_pipe(pipe_class_call);
16845 %}
16846
16847
16848 // Return Instruction
16849 // epilog node loads ret address into lr as part of frame pop
16850 instruct Ret()
16851 %{
16852 match(Return);
16853
16854 format %{ "ret\t// return register" %}
16855
16856 ins_encode( aarch64_enc_ret() );
16857
16858 ins_pipe(pipe_branch);
16859 %}
16860
16861 // Die now.
16862 instruct ShouldNotReachHere() %{
16863 match(Halt);
16864
16865 ins_cost(CALL_COST);
16866 format %{ "ShouldNotReachHere" %}
16867
16868 ins_encode %{
16869 if (is_reachable()) {
16870 __ stop(_halt_reason);
16871 }
16872 %}
16873
16874 ins_pipe(pipe_class_default);
16875 %}
16876
16877 // ============================================================================
16878 // Partial Subtype Check
16879 //
16880 // superklass array for an instance of the superklass. Set a hidden
16881 // internal cache on a hit (cache is checked with exposed code in
16882 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16883 // encoding ALSO sets flags.
16884
16885 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16886 %{
16887 match(Set result (PartialSubtypeCheck sub super));
16888 effect(KILL cr, KILL temp);
16889
16890 ins_cost(1100); // slightly larger than the next version
16891 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16892
16893 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16894
16895 opcode(0x1); // Force zero of result reg on hit
16896
16897 ins_pipe(pipe_class_memory);
16898 %}
16899
16900 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16901 %{
16902 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16903 effect(KILL temp, KILL result);
16904
16905 ins_cost(1100); // slightly larger than the next version
16906 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16907
16908 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16909
16910 opcode(0x0); // Don't zero result reg on hit
16911
16912 ins_pipe(pipe_class_memory);
16913 %}
16914
16915 // Intrisics for String.compareTo()
16916
16917 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16918 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16919 %{
16920 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16921 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16922 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16923
16924 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16925 ins_encode %{
16926 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16927 __ string_compare($str1$$Register, $str2$$Register,
16928 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16929 $tmp1$$Register, $tmp2$$Register,
16930 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16931 %}
16932 ins_pipe(pipe_class_memory);
16933 %}
16934
16935 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16936 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16937 %{
16938 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16939 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16940 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16941
16942 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16943 ins_encode %{
16944 __ string_compare($str1$$Register, $str2$$Register,
16945 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16946 $tmp1$$Register, $tmp2$$Register,
16947 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16948 %}
16949 ins_pipe(pipe_class_memory);
16950 %}
16951
16952 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16953 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16954 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16955 %{
16956 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16957 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16958 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16959 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16960
16961 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16962 ins_encode %{
16963 __ string_compare($str1$$Register, $str2$$Register,
16964 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16965 $tmp1$$Register, $tmp2$$Register,
16966 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16967 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16968 %}
16969 ins_pipe(pipe_class_memory);
16970 %}
16971
16972 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16973 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16974 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16975 %{
16976 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16977 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16978 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16979 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16980
16981 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16982 ins_encode %{
16983 __ string_compare($str1$$Register, $str2$$Register,
16984 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16985 $tmp1$$Register, $tmp2$$Register,
16986 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16987 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16988 %}
16989 ins_pipe(pipe_class_memory);
16990 %}
16991
16992 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16993 // these string_compare variants as NEON register type for convenience so that the prototype of
16994 // string_compare can be shared with all variants.
16995
16996 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16997 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16998 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16999 pRegGov_P1 pgtmp2, rFlagsReg cr)
17000 %{
17001 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
17002 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17003 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17004 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17005
17006 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
17007 ins_encode %{
17008 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17009 __ string_compare($str1$$Register, $str2$$Register,
17010 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17011 $tmp1$$Register, $tmp2$$Register,
17012 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17013 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17014 StrIntrinsicNode::LL);
17015 %}
17016 ins_pipe(pipe_class_memory);
17017 %}
17018
17019 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17020 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17021 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17022 pRegGov_P1 pgtmp2, rFlagsReg cr)
17023 %{
17024 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
17025 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17026 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17027 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17028
17029 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
17030 ins_encode %{
17031 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17032 __ string_compare($str1$$Register, $str2$$Register,
17033 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17034 $tmp1$$Register, $tmp2$$Register,
17035 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17036 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17037 StrIntrinsicNode::LU);
17038 %}
17039 ins_pipe(pipe_class_memory);
17040 %}
17041
17042 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17043 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17044 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17045 pRegGov_P1 pgtmp2, rFlagsReg cr)
17046 %{
17047 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
17048 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17049 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17050 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17051
17052 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
17053 ins_encode %{
17054 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17055 __ string_compare($str1$$Register, $str2$$Register,
17056 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17057 $tmp1$$Register, $tmp2$$Register,
17058 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17059 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17060 StrIntrinsicNode::UL);
17061 %}
17062 ins_pipe(pipe_class_memory);
17063 %}
17064
17065 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17066 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17067 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17068 pRegGov_P1 pgtmp2, rFlagsReg cr)
17069 %{
17070 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
17071 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17072 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17073 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17074
17075 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
17076 ins_encode %{
17077 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17078 __ string_compare($str1$$Register, $str2$$Register,
17079 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17080 $tmp1$$Register, $tmp2$$Register,
17081 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17082 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17083 StrIntrinsicNode::UU);
17084 %}
17085 ins_pipe(pipe_class_memory);
17086 %}
17087
17088 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17089 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17090 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17091 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17092 %{
17093 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17094 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17095 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17096 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
17097 TEMP vtmp0, TEMP vtmp1, KILL cr);
17098 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
17099 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17100
17101 ins_encode %{
17102 __ string_indexof($str1$$Register, $str2$$Register,
17103 $cnt1$$Register, $cnt2$$Register,
17104 $tmp1$$Register, $tmp2$$Register,
17105 $tmp3$$Register, $tmp4$$Register,
17106 $tmp5$$Register, $tmp6$$Register,
17107 -1, $result$$Register, StrIntrinsicNode::UU);
17108 %}
17109 ins_pipe(pipe_class_memory);
17110 %}
17111
17112 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17113 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17114 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17115 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17116 %{
17117 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17118 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17119 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17120 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
17121 TEMP vtmp0, TEMP vtmp1, KILL cr);
17122 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
17123 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17124
17125 ins_encode %{
17126 __ string_indexof($str1$$Register, $str2$$Register,
17127 $cnt1$$Register, $cnt2$$Register,
17128 $tmp1$$Register, $tmp2$$Register,
17129 $tmp3$$Register, $tmp4$$Register,
17130 $tmp5$$Register, $tmp6$$Register,
17131 -1, $result$$Register, StrIntrinsicNode::LL);
17132 %}
17133 ins_pipe(pipe_class_memory);
17134 %}
17135
17136 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17137 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
17138 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17139 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17140 %{
17141 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17142 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17143 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17144 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
17145 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
17146 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
17147 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17148
17149 ins_encode %{
17150 __ string_indexof($str1$$Register, $str2$$Register,
17151 $cnt1$$Register, $cnt2$$Register,
17152 $tmp1$$Register, $tmp2$$Register,
17153 $tmp3$$Register, $tmp4$$Register,
17154 $tmp5$$Register, $tmp6$$Register,
17155 -1, $result$$Register, StrIntrinsicNode::UL);
17156 %}
17157 ins_pipe(pipe_class_memory);
17158 %}
17159
17160 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17161 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17162 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17163 %{
17164 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17165 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17166 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17167 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17168 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
17169 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17170
17171 ins_encode %{
17172 int icnt2 = (int)$int_cnt2$$constant;
17173 __ string_indexof($str1$$Register, $str2$$Register,
17174 $cnt1$$Register, zr,
17175 $tmp1$$Register, $tmp2$$Register,
17176 $tmp3$$Register, $tmp4$$Register, zr, zr,
17177 icnt2, $result$$Register, StrIntrinsicNode::UU);
17178 %}
17179 ins_pipe(pipe_class_memory);
17180 %}
17181
17182 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17183 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17184 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17185 %{
17186 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17187 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17188 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17189 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17190 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
17191 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17192
17193 ins_encode %{
17194 int icnt2 = (int)$int_cnt2$$constant;
17195 __ string_indexof($str1$$Register, $str2$$Register,
17196 $cnt1$$Register, zr,
17197 $tmp1$$Register, $tmp2$$Register,
17198 $tmp3$$Register, $tmp4$$Register, zr, zr,
17199 icnt2, $result$$Register, StrIntrinsicNode::LL);
17200 %}
17201 ins_pipe(pipe_class_memory);
17202 %}
17203
17204 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17205 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17206 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17207 %{
17208 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17209 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17210 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17211 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17212 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
17213 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17214
17215 ins_encode %{
17216 int icnt2 = (int)$int_cnt2$$constant;
17217 __ string_indexof($str1$$Register, $str2$$Register,
17218 $cnt1$$Register, zr,
17219 $tmp1$$Register, $tmp2$$Register,
17220 $tmp3$$Register, $tmp4$$Register, zr, zr,
17221 icnt2, $result$$Register, StrIntrinsicNode::UL);
17222 %}
17223 ins_pipe(pipe_class_memory);
17224 %}
17225
17226 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17227 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17228 iRegINoSp tmp3, rFlagsReg cr)
17229 %{
17230 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17231 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17232 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17233 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17234
17235 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17236
17237 ins_encode %{
17238 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17239 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17240 $tmp3$$Register);
17241 %}
17242 ins_pipe(pipe_class_memory);
17243 %}
17244
17245 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17246 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17247 iRegINoSp tmp3, rFlagsReg cr)
17248 %{
17249 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17250 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17251 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17252 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17253
17254 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17255
17256 ins_encode %{
17257 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17258 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17259 $tmp3$$Register);
17260 %}
17261 ins_pipe(pipe_class_memory);
17262 %}
17263
17264 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17265 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17266 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17267 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17268 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17269 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17270 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17271 ins_encode %{
17272 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17273 $result$$Register, $ztmp1$$FloatRegister,
17274 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17275 $ptmp$$PRegister, true /* isL */);
17276 %}
17277 ins_pipe(pipe_class_memory);
17278 %}
17279
17280 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17281 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17282 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17283 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17284 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17285 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17286 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17287 ins_encode %{
17288 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17289 $result$$Register, $ztmp1$$FloatRegister,
17290 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17291 $ptmp$$PRegister, false /* isL */);
17292 %}
17293 ins_pipe(pipe_class_memory);
17294 %}
17295
17296 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17297 iRegI_R0 result, rFlagsReg cr)
17298 %{
17299 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17300 match(Set result (StrEquals (Binary str1 str2) cnt));
17301 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17302
17303 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17304 ins_encode %{
17305 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17306 __ string_equals($str1$$Register, $str2$$Register,
17307 $result$$Register, $cnt$$Register, 1);
17308 %}
17309 ins_pipe(pipe_class_memory);
17310 %}
17311
17312 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17313 iRegI_R0 result, rFlagsReg cr)
17314 %{
17315 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17316 match(Set result (StrEquals (Binary str1 str2) cnt));
17317 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17318
17319 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17320 ins_encode %{
17321 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17322 __ string_equals($str1$$Register, $str2$$Register,
17323 $result$$Register, $cnt$$Register, 2);
17324 %}
17325 ins_pipe(pipe_class_memory);
17326 %}
17327
17328 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17329 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17330 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17331 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17332 iRegP_R10 tmp, rFlagsReg cr)
17333 %{
17334 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17335 match(Set result (AryEq ary1 ary2));
17336 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17337 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17338 TEMP vtmp6, TEMP vtmp7, KILL cr);
17339
17340 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17341 ins_encode %{
17342 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17343 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17344 $result$$Register, $tmp$$Register, 1);
17345 if (tpc == NULL) {
17346 ciEnv::current()->record_failure("CodeCache is full");
17347 return;
17348 }
17349 %}
17350 ins_pipe(pipe_class_memory);
17351 %}
17352
17353 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17354 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17355 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17356 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17357 iRegP_R10 tmp, rFlagsReg cr)
17358 %{
17359 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17360 match(Set result (AryEq ary1 ary2));
17361 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17362 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17363 TEMP vtmp6, TEMP vtmp7, KILL cr);
17364
17365 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17366 ins_encode %{
17367 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17368 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17369 $result$$Register, $tmp$$Register, 2);
17370 if (tpc == NULL) {
17371 ciEnv::current()->record_failure("CodeCache is full");
17372 return;
17373 }
17374 %}
17375 ins_pipe(pipe_class_memory);
17376 %}
17377
17378 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17379 %{
17380 match(Set result (CountPositives ary1 len));
17381 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17382 format %{ "count positives byte[] $ary1,$len -> $result" %}
17383 ins_encode %{
17384 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17385 if (tpc == NULL) {
17386 ciEnv::current()->record_failure("CodeCache is full");
17387 return;
17388 }
17389 %}
17390 ins_pipe( pipe_slow );
17391 %}
17392
17393 // fast char[] to byte[] compression
17394 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17395 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17396 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17397 iRegI_R0 result, rFlagsReg cr)
17398 %{
17399 match(Set result (StrCompressedCopy src (Binary dst len)));
17400 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17401 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17402
17403 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17404 ins_encode %{
17405 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17406 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17407 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17408 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17409 %}
17410 ins_pipe(pipe_slow);
17411 %}
17412
17413 // fast byte[] to char[] inflation
17414 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17415 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17416 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17417 %{
17418 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17419 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17420 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17421 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17422
17423 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17424 ins_encode %{
17425 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17426 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17427 $vtmp2$$FloatRegister, $tmp$$Register);
17428 if (tpc == NULL) {
17429 ciEnv::current()->record_failure("CodeCache is full");
17430 return;
17431 }
17432 %}
17433 ins_pipe(pipe_class_memory);
17434 %}
17435
17436 // encode char[] to byte[] in ISO_8859_1
17437 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17438 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17439 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17440 iRegI_R0 result, rFlagsReg cr)
17441 %{
17442 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17443 match(Set result (EncodeISOArray src (Binary dst len)));
17444 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17445 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17446
17447 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17448 ins_encode %{
17449 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17450 $result$$Register, false,
17451 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17452 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17453 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17454 %}
17455 ins_pipe(pipe_class_memory);
17456 %}
17457
17458 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17459 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17460 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17461 iRegI_R0 result, rFlagsReg cr)
17462 %{
17463 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17464 match(Set result (EncodeISOArray src (Binary dst len)));
17465 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17466 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17467
17468 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17469 ins_encode %{
17470 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17471 $result$$Register, true,
17472 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17473 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17474 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17475 %}
17476 ins_pipe(pipe_class_memory);
17477 %}
17478
17479 //----------------------------- CompressBits/ExpandBits ------------------------
17480
17481 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17482 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17483 match(Set dst (CompressBits src mask));
17484 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17485 format %{ "mov $tsrc, $src\n\t"
17486 "mov $tmask, $mask\n\t"
17487 "bext $tdst, $tsrc, $tmask\n\t"
17488 "mov $dst, $tdst"
17489 %}
17490 ins_encode %{
17491 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17492 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17493 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17494 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17495 %}
17496 ins_pipe(pipe_slow);
17497 %}
17498
17499 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17500 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17501 match(Set dst (CompressBits (LoadI mem) mask));
17502 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17503 format %{ "ldrs $tsrc, $mem\n\t"
17504 "ldrs $tmask, $mask\n\t"
17505 "bext $tdst, $tsrc, $tmask\n\t"
17506 "mov $dst, $tdst"
17507 %}
17508 ins_encode %{
17509 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17510 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17511 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17512 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17513 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17514 %}
17515 ins_pipe(pipe_slow);
17516 %}
17517
17518 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17519 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17520 match(Set dst (CompressBits src mask));
17521 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17522 format %{ "mov $tsrc, $src\n\t"
17523 "mov $tmask, $mask\n\t"
17524 "bext $tdst, $tsrc, $tmask\n\t"
17525 "mov $dst, $tdst"
17526 %}
17527 ins_encode %{
17528 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17529 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17530 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17531 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17532 %}
17533 ins_pipe(pipe_slow);
17534 %}
17535
17536 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17537 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17538 match(Set dst (CompressBits (LoadL mem) mask));
17539 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17540 format %{ "ldrd $tsrc, $mem\n\t"
17541 "ldrd $tmask, $mask\n\t"
17542 "bext $tdst, $tsrc, $tmask\n\t"
17543 "mov $dst, $tdst"
17544 %}
17545 ins_encode %{
17546 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17547 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17548 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17549 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17550 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17551 %}
17552 ins_pipe(pipe_slow);
17553 %}
17554
17555 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17556 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17557 match(Set dst (ExpandBits src mask));
17558 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17559 format %{ "mov $tsrc, $src\n\t"
17560 "mov $tmask, $mask\n\t"
17561 "bdep $tdst, $tsrc, $tmask\n\t"
17562 "mov $dst, $tdst"
17563 %}
17564 ins_encode %{
17565 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17566 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17567 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17568 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17569 %}
17570 ins_pipe(pipe_slow);
17571 %}
17572
17573 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17574 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17575 match(Set dst (ExpandBits (LoadI mem) mask));
17576 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17577 format %{ "ldrs $tsrc, $mem\n\t"
17578 "ldrs $tmask, $mask\n\t"
17579 "bdep $tdst, $tsrc, $tmask\n\t"
17580 "mov $dst, $tdst"
17581 %}
17582 ins_encode %{
17583 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17584 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17585 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17586 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17587 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17588 %}
17589 ins_pipe(pipe_slow);
17590 %}
17591
17592 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17593 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17594 match(Set dst (ExpandBits src mask));
17595 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17596 format %{ "mov $tsrc, $src\n\t"
17597 "mov $tmask, $mask\n\t"
17598 "bdep $tdst, $tsrc, $tmask\n\t"
17599 "mov $dst, $tdst"
17600 %}
17601 ins_encode %{
17602 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17603 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17604 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17605 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17606 %}
17607 ins_pipe(pipe_slow);
17608 %}
17609
17610
17611 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17612 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17613 match(Set dst (ExpandBits (LoadL mem) mask));
17614 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17615 format %{ "ldrd $tsrc, $mem\n\t"
17616 "ldrd $tmask, $mask\n\t"
17617 "bdep $tdst, $tsrc, $tmask\n\t"
17618 "mov $dst, $tdst"
17619 %}
17620 ins_encode %{
17621 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17622 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17623 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17624 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17625 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17626 %}
17627 ins_pipe(pipe_slow);
17628 %}
17629
17630 // ============================================================================
17631 // This name is KNOWN by the ADLC and cannot be changed.
17632 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17633 // for this guy.
17634 instruct tlsLoadP(thread_RegP dst)
17635 %{
17636 match(Set dst (ThreadLocal));
17637
17638 ins_cost(0);
17639
17640 format %{ " -- \t// $dst=Thread::current(), empty" %}
17641
17642 size(0);
17643
17644 ins_encode( /*empty*/ );
17645
17646 ins_pipe(pipe_class_empty);
17647 %}
17648
17649 //----------PEEPHOLE RULES-----------------------------------------------------
17650 // These must follow all instruction definitions as they use the names
17651 // defined in the instructions definitions.
17652 //
17653 // peepmatch ( root_instr_name [preceding_instruction]* );
17654 //
17655 // peepconstraint %{
17656 // (instruction_number.operand_name relational_op instruction_number.operand_name
17657 // [, ...] );
17658 // // instruction numbers are zero-based using left to right order in peepmatch
17659 //
17660 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17661 // // provide an instruction_number.operand_name for each operand that appears
17662 // // in the replacement instruction's match rule
17663 //
17664 // ---------VM FLAGS---------------------------------------------------------
17665 //
17666 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17667 //
17668 // Each peephole rule is given an identifying number starting with zero and
17669 // increasing by one in the order seen by the parser. An individual peephole
17670 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17671 // on the command-line.
17672 //
17673 // ---------CURRENT LIMITATIONS----------------------------------------------
17674 //
17675 // Only match adjacent instructions in same basic block
17676 // Only equality constraints
17677 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17678 // Only one replacement instruction
17679 //
17680 // ---------EXAMPLE----------------------------------------------------------
17681 //
17682 // // pertinent parts of existing instructions in architecture description
17683 // instruct movI(iRegINoSp dst, iRegI src)
17684 // %{
17685 // match(Set dst (CopyI src));
17686 // %}
17687 //
17688 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17689 // %{
17690 // match(Set dst (AddI dst src));
17691 // effect(KILL cr);
17692 // %}
17693 //
17694 // // Change (inc mov) to lea
17695 // peephole %{
17696 // // increment preceded by register-register move
17697 // peepmatch ( incI_iReg movI );
17698 // // require that the destination register of the increment
17699 // // match the destination register of the move
17700 // peepconstraint ( 0.dst == 1.dst );
17701 // // construct a replacement instruction that sets
17702 // // the destination to ( move's source register + one )
17703 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17704 // %}
17705 //
17706
17707 // Implementation no longer uses movX instructions since
17708 // machine-independent system no longer uses CopyX nodes.
17709 //
17710 // peephole
17711 // %{
17712 // peepmatch (incI_iReg movI);
17713 // peepconstraint (0.dst == 1.dst);
17714 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17715 // %}
17716
17717 // peephole
17718 // %{
17719 // peepmatch (decI_iReg movI);
17720 // peepconstraint (0.dst == 1.dst);
17721 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17722 // %}
17723
17724 // peephole
17725 // %{
17726 // peepmatch (addI_iReg_imm movI);
17727 // peepconstraint (0.dst == 1.dst);
17728 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17729 // %}
17730
17731 // peephole
17732 // %{
17733 // peepmatch (incL_iReg movL);
17734 // peepconstraint (0.dst == 1.dst);
17735 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17736 // %}
17737
17738 // peephole
17739 // %{
17740 // peepmatch (decL_iReg movL);
17741 // peepconstraint (0.dst == 1.dst);
17742 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17743 // %}
17744
17745 // peephole
17746 // %{
17747 // peepmatch (addL_iReg_imm movL);
17748 // peepconstraint (0.dst == 1.dst);
17749 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17750 // %}
17751
17752 // peephole
17753 // %{
17754 // peepmatch (addP_iReg_imm movP);
17755 // peepconstraint (0.dst == 1.dst);
17756 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17757 // %}
17758
17759 // // Change load of spilled value to only a spill
17760 // instruct storeI(memory mem, iRegI src)
17761 // %{
17762 // match(Set mem (StoreI mem src));
17763 // %}
17764 //
17765 // instruct loadI(iRegINoSp dst, memory mem)
17766 // %{
17767 // match(Set dst (LoadI mem));
17768 // %}
17769 //
17770
17771 //----------SMARTSPILL RULES---------------------------------------------------
17772 // These must follow all instruction definitions as they use the names
17773 // defined in the instructions definitions.
17774
17775 // Local Variables:
17776 // mode: c++
17777 // End: