1 //
2 // Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all float registers
698 reg_class float_reg(
699 V0,
700 V1,
701 V2,
702 V3,
703 V4,
704 V5,
705 V6,
706 V7,
707 V8,
708 V9,
709 V10,
710 V11,
711 V12,
712 V13,
713 V14,
714 V15,
715 V16,
716 V17,
717 V18,
718 V19,
719 V20,
720 V21,
721 V22,
722 V23,
723 V24,
724 V25,
725 V26,
726 V27,
727 V28,
728 V29,
729 V30,
730 V31
731 );
732
733 // Double precision float registers have virtual `high halves' that
734 // are needed by the allocator.
735 // Class for all double registers
736 reg_class double_reg(
737 V0, V0_H,
738 V1, V1_H,
739 V2, V2_H,
740 V3, V3_H,
741 V4, V4_H,
742 V5, V5_H,
743 V6, V6_H,
744 V7, V7_H,
745 V8, V8_H,
746 V9, V9_H,
747 V10, V10_H,
748 V11, V11_H,
749 V12, V12_H,
750 V13, V13_H,
751 V14, V14_H,
752 V15, V15_H,
753 V16, V16_H,
754 V17, V17_H,
755 V18, V18_H,
756 V19, V19_H,
757 V20, V20_H,
758 V21, V21_H,
759 V22, V22_H,
760 V23, V23_H,
761 V24, V24_H,
762 V25, V25_H,
763 V26, V26_H,
764 V27, V27_H,
765 V28, V28_H,
766 V29, V29_H,
767 V30, V30_H,
768 V31, V31_H
769 );
770
771 // Class for all SVE vector registers.
772 reg_class vectora_reg (
773 V0, V0_H, V0_J, V0_K,
774 V1, V1_H, V1_J, V1_K,
775 V2, V2_H, V2_J, V2_K,
776 V3, V3_H, V3_J, V3_K,
777 V4, V4_H, V4_J, V4_K,
778 V5, V5_H, V5_J, V5_K,
779 V6, V6_H, V6_J, V6_K,
780 V7, V7_H, V7_J, V7_K,
781 V8, V8_H, V8_J, V8_K,
782 V9, V9_H, V9_J, V9_K,
783 V10, V10_H, V10_J, V10_K,
784 V11, V11_H, V11_J, V11_K,
785 V12, V12_H, V12_J, V12_K,
786 V13, V13_H, V13_J, V13_K,
787 V14, V14_H, V14_J, V14_K,
788 V15, V15_H, V15_J, V15_K,
789 V16, V16_H, V16_J, V16_K,
790 V17, V17_H, V17_J, V17_K,
791 V18, V18_H, V18_J, V18_K,
792 V19, V19_H, V19_J, V19_K,
793 V20, V20_H, V20_J, V20_K,
794 V21, V21_H, V21_J, V21_K,
795 V22, V22_H, V22_J, V22_K,
796 V23, V23_H, V23_J, V23_K,
797 V24, V24_H, V24_J, V24_K,
798 V25, V25_H, V25_J, V25_K,
799 V26, V26_H, V26_J, V26_K,
800 V27, V27_H, V27_J, V27_K,
801 V28, V28_H, V28_J, V28_K,
802 V29, V29_H, V29_J, V29_K,
803 V30, V30_H, V30_J, V30_K,
804 V31, V31_H, V31_J, V31_K,
805 );
806
807 // Class for all 64bit vector registers
808 reg_class vectord_reg(
809 V0, V0_H,
810 V1, V1_H,
811 V2, V2_H,
812 V3, V3_H,
813 V4, V4_H,
814 V5, V5_H,
815 V6, V6_H,
816 V7, V7_H,
817 V8, V8_H,
818 V9, V9_H,
819 V10, V10_H,
820 V11, V11_H,
821 V12, V12_H,
822 V13, V13_H,
823 V14, V14_H,
824 V15, V15_H,
825 V16, V16_H,
826 V17, V17_H,
827 V18, V18_H,
828 V19, V19_H,
829 V20, V20_H,
830 V21, V21_H,
831 V22, V22_H,
832 V23, V23_H,
833 V24, V24_H,
834 V25, V25_H,
835 V26, V26_H,
836 V27, V27_H,
837 V28, V28_H,
838 V29, V29_H,
839 V30, V30_H,
840 V31, V31_H
841 );
842
843 // Class for all 128bit vector registers
844 reg_class vectorx_reg(
845 V0, V0_H, V0_J, V0_K,
846 V1, V1_H, V1_J, V1_K,
847 V2, V2_H, V2_J, V2_K,
848 V3, V3_H, V3_J, V3_K,
849 V4, V4_H, V4_J, V4_K,
850 V5, V5_H, V5_J, V5_K,
851 V6, V6_H, V6_J, V6_K,
852 V7, V7_H, V7_J, V7_K,
853 V8, V8_H, V8_J, V8_K,
854 V9, V9_H, V9_J, V9_K,
855 V10, V10_H, V10_J, V10_K,
856 V11, V11_H, V11_J, V11_K,
857 V12, V12_H, V12_J, V12_K,
858 V13, V13_H, V13_J, V13_K,
859 V14, V14_H, V14_J, V14_K,
860 V15, V15_H, V15_J, V15_K,
861 V16, V16_H, V16_J, V16_K,
862 V17, V17_H, V17_J, V17_K,
863 V18, V18_H, V18_J, V18_K,
864 V19, V19_H, V19_J, V19_K,
865 V20, V20_H, V20_J, V20_K,
866 V21, V21_H, V21_J, V21_K,
867 V22, V22_H, V22_J, V22_K,
868 V23, V23_H, V23_J, V23_K,
869 V24, V24_H, V24_J, V24_K,
870 V25, V25_H, V25_J, V25_K,
871 V26, V26_H, V26_J, V26_K,
872 V27, V27_H, V27_J, V27_K,
873 V28, V28_H, V28_J, V28_K,
874 V29, V29_H, V29_J, V29_K,
875 V30, V30_H, V30_J, V30_K,
876 V31, V31_H, V31_J, V31_K
877 );
878
879 // Class for 128 bit register v0
880 reg_class v0_reg(
881 V0, V0_H
882 );
883
884 // Class for 128 bit register v1
885 reg_class v1_reg(
886 V1, V1_H
887 );
888
889 // Class for 128 bit register v2
890 reg_class v2_reg(
891 V2, V2_H
892 );
893
894 // Class for 128 bit register v3
895 reg_class v3_reg(
896 V3, V3_H
897 );
898
899 // Class for 128 bit register v4
900 reg_class v4_reg(
901 V4, V4_H
902 );
903
904 // Class for 128 bit register v5
905 reg_class v5_reg(
906 V5, V5_H
907 );
908
909 // Class for 128 bit register v6
910 reg_class v6_reg(
911 V6, V6_H
912 );
913
914 // Class for 128 bit register v7
915 reg_class v7_reg(
916 V7, V7_H
917 );
918
919 // Class for 128 bit register v8
920 reg_class v8_reg(
921 V8, V8_H
922 );
923
924 // Class for 128 bit register v9
925 reg_class v9_reg(
926 V9, V9_H
927 );
928
929 // Class for 128 bit register v10
930 reg_class v10_reg(
931 V10, V10_H
932 );
933
934 // Class for 128 bit register v11
935 reg_class v11_reg(
936 V11, V11_H
937 );
938
939 // Class for 128 bit register v12
940 reg_class v12_reg(
941 V12, V12_H
942 );
943
944 // Class for 128 bit register v13
945 reg_class v13_reg(
946 V13, V13_H
947 );
948
949 // Class for 128 bit register v14
950 reg_class v14_reg(
951 V14, V14_H
952 );
953
954 // Class for 128 bit register v15
955 reg_class v15_reg(
956 V15, V15_H
957 );
958
959 // Class for 128 bit register v16
960 reg_class v16_reg(
961 V16, V16_H
962 );
963
964 // Class for 128 bit register v17
965 reg_class v17_reg(
966 V17, V17_H
967 );
968
969 // Class for 128 bit register v18
970 reg_class v18_reg(
971 V18, V18_H
972 );
973
974 // Class for 128 bit register v19
975 reg_class v19_reg(
976 V19, V19_H
977 );
978
979 // Class for 128 bit register v20
980 reg_class v20_reg(
981 V20, V20_H
982 );
983
984 // Class for 128 bit register v21
985 reg_class v21_reg(
986 V21, V21_H
987 );
988
989 // Class for 128 bit register v22
990 reg_class v22_reg(
991 V22, V22_H
992 );
993
994 // Class for 128 bit register v23
995 reg_class v23_reg(
996 V23, V23_H
997 );
998
999 // Class for 128 bit register v24
1000 reg_class v24_reg(
1001 V24, V24_H
1002 );
1003
1004 // Class for 128 bit register v25
1005 reg_class v25_reg(
1006 V25, V25_H
1007 );
1008
1009 // Class for 128 bit register v26
1010 reg_class v26_reg(
1011 V26, V26_H
1012 );
1013
1014 // Class for 128 bit register v27
1015 reg_class v27_reg(
1016 V27, V27_H
1017 );
1018
1019 // Class for 128 bit register v28
1020 reg_class v28_reg(
1021 V28, V28_H
1022 );
1023
1024 // Class for 128 bit register v29
1025 reg_class v29_reg(
1026 V29, V29_H
1027 );
1028
1029 // Class for 128 bit register v30
1030 reg_class v30_reg(
1031 V30, V30_H
1032 );
1033
1034 // Class for 128 bit register v31
1035 reg_class v31_reg(
1036 V31, V31_H
1037 );
1038
1039 // Class for all SVE predicate registers.
1040 reg_class pr_reg (
1041 P0,
1042 P1,
1043 P2,
1044 P3,
1045 P4,
1046 P5,
1047 P6,
1048 // P7, non-allocatable, preserved with all elements preset to TRUE.
1049 P8,
1050 P9,
1051 P10,
1052 P11,
1053 P12,
1054 P13,
1055 P14,
1056 P15
1057 );
1058
1059 // Class for SVE governing predicate registers, which are used
1060 // to determine the active elements of a predicated instruction.
1061 reg_class gov_pr (
1062 P0,
1063 P1,
1064 P2,
1065 P3,
1066 P4,
1067 P5,
1068 P6,
1069 // P7, non-allocatable, preserved with all elements preset to TRUE.
1070 );
1071
1072 reg_class p0_reg(P0);
1073 reg_class p1_reg(P1);
1074
1075 // Singleton class for condition codes
1076 reg_class int_flags(RFLAGS);
1077
1078 %}
1079
1080 //----------DEFINITION BLOCK---------------------------------------------------
1081 // Define name --> value mappings to inform the ADLC of an integer valued name
1082 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1083 // Format:
1084 // int_def <name> ( <int_value>, <expression>);
1085 // Generated Code in ad_<arch>.hpp
1086 // #define <name> (<expression>)
1087 // // value == <int_value>
1088 // Generated code in ad_<arch>.cpp adlc_verification()
1089 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1090 //
1091
1092 // we follow the ppc-aix port in using a simple cost model which ranks
1093 // register operations as cheap, memory ops as more expensive and
1094 // branches as most expensive. the first two have a low as well as a
1095 // normal cost. huge cost appears to be a way of saying don't do
1096 // something
1097
1098 definitions %{
1099 // The default cost (of a register move instruction).
1100 int_def INSN_COST ( 100, 100);
1101 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1102 int_def CALL_COST ( 200, 2 * INSN_COST);
1103 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1104 %}
1105
1106
1107 //----------SOURCE BLOCK-------------------------------------------------------
1108 // This is a block of C++ code which provides values, functions, and
1109 // definitions necessary in the rest of the architecture description
1110
1111 source_hpp %{
1112
1113 #include "asm/macroAssembler.hpp"
1114 #include "gc/shared/barrierSetAssembler.hpp"
1115 #include "gc/shared/cardTable.hpp"
1116 #include "gc/shared/cardTableBarrierSet.hpp"
1117 #include "gc/shared/collectedHeap.hpp"
1118 #include "opto/addnode.hpp"
1119 #include "opto/convertnode.hpp"
1120 #include "runtime/objectMonitor.hpp"
1121
1122 extern RegMask _ANY_REG32_mask;
1123 extern RegMask _ANY_REG_mask;
1124 extern RegMask _PTR_REG_mask;
1125 extern RegMask _NO_SPECIAL_REG32_mask;
1126 extern RegMask _NO_SPECIAL_REG_mask;
1127 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1128
1129 class CallStubImpl {
1130
1131 //--------------------------------------------------------------
1132 //---< Used for optimization in Compile::shorten_branches >---
1133 //--------------------------------------------------------------
1134
1135 public:
1136 // Size of call trampoline stub.
1137 static uint size_call_trampoline() {
1138 return 0; // no call trampolines on this platform
1139 }
1140
1141 // number of relocations needed by a call trampoline stub
1142 static uint reloc_call_trampoline() {
1143 return 0; // no call trampolines on this platform
1144 }
1145 };
1146
1147 class HandlerImpl {
1148
1149 public:
1150
1151 static int emit_exception_handler(CodeBuffer &cbuf);
1152 static int emit_deopt_handler(CodeBuffer& cbuf);
1153
1154 static uint size_exception_handler() {
1155 return MacroAssembler::far_codestub_branch_size();
1156 }
1157
1158 static uint size_deopt_handler() {
1159 // count one adr and one far branch instruction
1160 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1161 }
1162 };
1163
1164 class Node::PD {
1165 public:
1166 enum NodeFlags {
1167 _last_flag = Node::_last_flag
1168 };
1169 };
1170
1171 bool is_CAS(int opcode, bool maybe_volatile);
1172
1173 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1174
1175 bool unnecessary_acquire(const Node *barrier);
1176 bool needs_acquiring_load(const Node *load);
1177
1178 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1179
1180 bool unnecessary_release(const Node *barrier);
1181 bool unnecessary_volatile(const Node *barrier);
1182 bool needs_releasing_store(const Node *store);
1183
1184 // predicate controlling translation of CompareAndSwapX
1185 bool needs_acquiring_load_exclusive(const Node *load);
1186
1187 // predicate controlling addressing modes
1188 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1189
1190 // Convert BootTest condition to Assembler condition.
1191 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1192 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1193 %}
1194
1195 source %{
1196
1197 // Derived RegMask with conditionally allocatable registers
1198
1199 void PhaseOutput::pd_perform_mach_node_analysis() {
1200 }
1201
1202 int MachNode::pd_alignment_required() const {
1203 return 1;
1204 }
1205
1206 int MachNode::compute_padding(int current_offset) const {
1207 return 0;
1208 }
1209
1210 RegMask _ANY_REG32_mask;
1211 RegMask _ANY_REG_mask;
1212 RegMask _PTR_REG_mask;
1213 RegMask _NO_SPECIAL_REG32_mask;
1214 RegMask _NO_SPECIAL_REG_mask;
1215 RegMask _NO_SPECIAL_PTR_REG_mask;
1216
1217 void reg_mask_init() {
1218 // We derive below RegMask(s) from the ones which are auto-generated from
1219 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1220 // registers conditionally reserved.
1221
1222 _ANY_REG32_mask = _ALL_REG32_mask;
1223 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1224
1225 _ANY_REG_mask = _ALL_REG_mask;
1226
1227 _PTR_REG_mask = _ALL_REG_mask;
1228
1229 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1230 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1231
1232 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1233 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1234
1235 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1236 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1237
1238 // r27 is not allocatable when compressed oops is on and heapbase is not
1239 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1240 if (UseCompressedOops && (CompressedOops::ptrs_base() != nullptr)) {
1241 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1242 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1243 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1244 }
1245
1246 // r29 is not allocatable when PreserveFramePointer is on
1247 if (PreserveFramePointer) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1251 }
1252 }
1253
1254 // Optimizaton of volatile gets and puts
1255 // -------------------------------------
1256 //
1257 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1258 // use to implement volatile reads and writes. For a volatile read
1259 // we simply need
1260 //
1261 // ldar<x>
1262 //
1263 // and for a volatile write we need
1264 //
1265 // stlr<x>
1266 //
1267 // Alternatively, we can implement them by pairing a normal
1268 // load/store with a memory barrier. For a volatile read we need
1269 //
1270 // ldr<x>
1271 // dmb ishld
1272 //
1273 // for a volatile write
1274 //
1275 // dmb ish
1276 // str<x>
1277 // dmb ish
1278 //
1279 // We can also use ldaxr and stlxr to implement compare and swap CAS
1280 // sequences. These are normally translated to an instruction
1281 // sequence like the following
1282 //
1283 // dmb ish
1284 // retry:
1285 // ldxr<x> rval raddr
1286 // cmp rval rold
1287 // b.ne done
1288 // stlxr<x> rval, rnew, rold
1289 // cbnz rval retry
1290 // done:
1291 // cset r0, eq
1292 // dmb ishld
1293 //
1294 // Note that the exclusive store is already using an stlxr
1295 // instruction. That is required to ensure visibility to other
1296 // threads of the exclusive write (assuming it succeeds) before that
1297 // of any subsequent writes.
1298 //
1299 // The following instruction sequence is an improvement on the above
1300 //
1301 // retry:
1302 // ldaxr<x> rval raddr
1303 // cmp rval rold
1304 // b.ne done
1305 // stlxr<x> rval, rnew, rold
1306 // cbnz rval retry
1307 // done:
1308 // cset r0, eq
1309 //
1310 // We don't need the leading dmb ish since the stlxr guarantees
1311 // visibility of prior writes in the case that the swap is
1312 // successful. Crucially we don't have to worry about the case where
1313 // the swap is not successful since no valid program should be
1314 // relying on visibility of prior changes by the attempting thread
1315 // in the case where the CAS fails.
1316 //
1317 // Similarly, we don't need the trailing dmb ishld if we substitute
1318 // an ldaxr instruction since that will provide all the guarantees we
1319 // require regarding observation of changes made by other threads
1320 // before any change to the CAS address observed by the load.
1321 //
1322 // In order to generate the desired instruction sequence we need to
1323 // be able to identify specific 'signature' ideal graph node
1324 // sequences which i) occur as a translation of a volatile reads or
1325 // writes or CAS operations and ii) do not occur through any other
1326 // translation or graph transformation. We can then provide
1327 // alternative aldc matching rules which translate these node
1328 // sequences to the desired machine code sequences. Selection of the
1329 // alternative rules can be implemented by predicates which identify
1330 // the relevant node sequences.
1331 //
1332 // The ideal graph generator translates a volatile read to the node
1333 // sequence
1334 //
1335 // LoadX[mo_acquire]
1336 // MemBarAcquire
1337 //
1338 // As a special case when using the compressed oops optimization we
1339 // may also see this variant
1340 //
1341 // LoadN[mo_acquire]
1342 // DecodeN
1343 // MemBarAcquire
1344 //
1345 // A volatile write is translated to the node sequence
1346 //
1347 // MemBarRelease
1348 // StoreX[mo_release] {CardMark}-optional
1349 // MemBarVolatile
1350 //
1351 // n.b. the above node patterns are generated with a strict
1352 // 'signature' configuration of input and output dependencies (see
1353 // the predicates below for exact details). The card mark may be as
1354 // simple as a few extra nodes or, in a few GC configurations, may
1355 // include more complex control flow between the leading and
1356 // trailing memory barriers. However, whatever the card mark
1357 // configuration these signatures are unique to translated volatile
1358 // reads/stores -- they will not appear as a result of any other
1359 // bytecode translation or inlining nor as a consequence of
1360 // optimizing transforms.
1361 //
1362 // We also want to catch inlined unsafe volatile gets and puts and
1363 // be able to implement them using either ldar<x>/stlr<x> or some
1364 // combination of ldr<x>/stlr<x> and dmb instructions.
1365 //
1366 // Inlined unsafe volatiles puts manifest as a minor variant of the
1367 // normal volatile put node sequence containing an extra cpuorder
1368 // membar
1369 //
1370 // MemBarRelease
1371 // MemBarCPUOrder
1372 // StoreX[mo_release] {CardMark}-optional
1373 // MemBarCPUOrder
1374 // MemBarVolatile
1375 //
1376 // n.b. as an aside, a cpuorder membar is not itself subject to
1377 // matching and translation by adlc rules. However, the rule
1378 // predicates need to detect its presence in order to correctly
1379 // select the desired adlc rules.
1380 //
1381 // Inlined unsafe volatile gets manifest as a slightly different
1382 // node sequence to a normal volatile get because of the
1383 // introduction of some CPUOrder memory barriers to bracket the
1384 // Load. However, but the same basic skeleton of a LoadX feeding a
1385 // MemBarAcquire, possibly through an optional DecodeN, is still
1386 // present
1387 //
1388 // MemBarCPUOrder
1389 // || \\
1390 // MemBarCPUOrder LoadX[mo_acquire]
1391 // || |
1392 // || {DecodeN} optional
1393 // || /
1394 // MemBarAcquire
1395 //
1396 // In this case the acquire membar does not directly depend on the
1397 // load. However, we can be sure that the load is generated from an
1398 // inlined unsafe volatile get if we see it dependent on this unique
1399 // sequence of membar nodes. Similarly, given an acquire membar we
1400 // can know that it was added because of an inlined unsafe volatile
1401 // get if it is fed and feeds a cpuorder membar and if its feed
1402 // membar also feeds an acquiring load.
1403 //
1404 // Finally an inlined (Unsafe) CAS operation is translated to the
1405 // following ideal graph
1406 //
1407 // MemBarRelease
1408 // MemBarCPUOrder
1409 // CompareAndSwapX {CardMark}-optional
1410 // MemBarCPUOrder
1411 // MemBarAcquire
1412 //
1413 // So, where we can identify these volatile read and write
1414 // signatures we can choose to plant either of the above two code
1415 // sequences. For a volatile read we can simply plant a normal
1416 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1417 // also choose to inhibit translation of the MemBarAcquire and
1418 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1419 //
1420 // When we recognise a volatile store signature we can choose to
1421 // plant at a dmb ish as a translation for the MemBarRelease, a
1422 // normal str<x> and then a dmb ish for the MemBarVolatile.
1423 // Alternatively, we can inhibit translation of the MemBarRelease
1424 // and MemBarVolatile and instead plant a simple stlr<x>
1425 // instruction.
1426 //
1427 // when we recognise a CAS signature we can choose to plant a dmb
1428 // ish as a translation for the MemBarRelease, the conventional
1429 // macro-instruction sequence for the CompareAndSwap node (which
1430 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1431 // Alternatively, we can elide generation of the dmb instructions
1432 // and plant the alternative CompareAndSwap macro-instruction
1433 // sequence (which uses ldaxr<x>).
1434 //
1435 // Of course, the above only applies when we see these signature
1436 // configurations. We still want to plant dmb instructions in any
1437 // other cases where we may see a MemBarAcquire, MemBarRelease or
1438 // MemBarVolatile. For example, at the end of a constructor which
1439 // writes final/volatile fields we will see a MemBarRelease
1440 // instruction and this needs a 'dmb ish' lest we risk the
1441 // constructed object being visible without making the
1442 // final/volatile field writes visible.
1443 //
1444 // n.b. the translation rules below which rely on detection of the
1445 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1446 // If we see anything other than the signature configurations we
1447 // always just translate the loads and stores to ldr<x> and str<x>
1448 // and translate acquire, release and volatile membars to the
1449 // relevant dmb instructions.
1450 //
1451
1452 // is_CAS(int opcode, bool maybe_volatile)
1453 //
1454 // return true if opcode is one of the possible CompareAndSwapX
1455 // values otherwise false.
1456
1457 bool is_CAS(int opcode, bool maybe_volatile)
1458 {
1459 switch(opcode) {
1460 // We handle these
1461 case Op_CompareAndSwapI:
1462 case Op_CompareAndSwapL:
1463 case Op_CompareAndSwapP:
1464 case Op_CompareAndSwapN:
1465 case Op_ShenandoahCompareAndSwapP:
1466 case Op_ShenandoahCompareAndSwapN:
1467 case Op_CompareAndSwapB:
1468 case Op_CompareAndSwapS:
1469 case Op_GetAndSetI:
1470 case Op_GetAndSetL:
1471 case Op_GetAndSetP:
1472 case Op_GetAndSetN:
1473 case Op_GetAndAddI:
1474 case Op_GetAndAddL:
1475 return true;
1476 case Op_CompareAndExchangeI:
1477 case Op_CompareAndExchangeN:
1478 case Op_CompareAndExchangeB:
1479 case Op_CompareAndExchangeS:
1480 case Op_CompareAndExchangeL:
1481 case Op_CompareAndExchangeP:
1482 case Op_WeakCompareAndSwapB:
1483 case Op_WeakCompareAndSwapS:
1484 case Op_WeakCompareAndSwapI:
1485 case Op_WeakCompareAndSwapL:
1486 case Op_WeakCompareAndSwapP:
1487 case Op_WeakCompareAndSwapN:
1488 case Op_ShenandoahWeakCompareAndSwapP:
1489 case Op_ShenandoahWeakCompareAndSwapN:
1490 case Op_ShenandoahCompareAndExchangeP:
1491 case Op_ShenandoahCompareAndExchangeN:
1492 return maybe_volatile;
1493 default:
1494 return false;
1495 }
1496 }
1497
1498 // helper to determine the maximum number of Phi nodes we may need to
1499 // traverse when searching from a card mark membar for the merge mem
1500 // feeding a trailing membar or vice versa
1501
1502 // predicates controlling emit of ldr<x>/ldar<x>
1503
1504 bool unnecessary_acquire(const Node *barrier)
1505 {
1506 assert(barrier->is_MemBar(), "expecting a membar");
1507
1508 MemBarNode* mb = barrier->as_MemBar();
1509
1510 if (mb->trailing_load()) {
1511 return true;
1512 }
1513
1514 if (mb->trailing_load_store()) {
1515 Node* load_store = mb->in(MemBarNode::Precedent);
1516 assert(load_store->is_LoadStore(), "unexpected graph shape");
1517 return is_CAS(load_store->Opcode(), true);
1518 }
1519
1520 return false;
1521 }
1522
1523 bool needs_acquiring_load(const Node *n)
1524 {
1525 assert(n->is_Load(), "expecting a load");
1526 LoadNode *ld = n->as_Load();
1527 return ld->is_acquire();
1528 }
1529
1530 bool unnecessary_release(const Node *n)
1531 {
1532 assert((n->is_MemBar() &&
1533 n->Opcode() == Op_MemBarRelease),
1534 "expecting a release membar");
1535
1536 MemBarNode *barrier = n->as_MemBar();
1537 if (!barrier->leading()) {
1538 return false;
1539 } else {
1540 Node* trailing = barrier->trailing_membar();
1541 MemBarNode* trailing_mb = trailing->as_MemBar();
1542 assert(trailing_mb->trailing(), "Not a trailing membar?");
1543 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1544
1545 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1546 if (mem->is_Store()) {
1547 assert(mem->as_Store()->is_release(), "");
1548 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1549 return true;
1550 } else {
1551 assert(mem->is_LoadStore(), "");
1552 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1553 return is_CAS(mem->Opcode(), true);
1554 }
1555 }
1556 return false;
1557 }
1558
1559 bool unnecessary_volatile(const Node *n)
1560 {
1561 // assert n->is_MemBar();
1562 MemBarNode *mbvol = n->as_MemBar();
1563
1564 bool release = mbvol->trailing_store();
1565 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1566 #ifdef ASSERT
1567 if (release) {
1568 Node* leading = mbvol->leading_membar();
1569 assert(leading->Opcode() == Op_MemBarRelease, "");
1570 assert(leading->as_MemBar()->leading_store(), "");
1571 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1572 }
1573 #endif
1574
1575 return release;
1576 }
1577
1578 // predicates controlling emit of str<x>/stlr<x>
1579
1580 bool needs_releasing_store(const Node *n)
1581 {
1582 // assert n->is_Store();
1583 StoreNode *st = n->as_Store();
1584 return st->trailing_membar() != nullptr;
1585 }
1586
1587 // predicate controlling translation of CAS
1588 //
1589 // returns true if CAS needs to use an acquiring load otherwise false
1590
1591 bool needs_acquiring_load_exclusive(const Node *n)
1592 {
1593 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1594 LoadStoreNode* ldst = n->as_LoadStore();
1595 if (is_CAS(n->Opcode(), false)) {
1596 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1597 } else {
1598 return ldst->trailing_membar() != nullptr;
1599 }
1600
1601 // so we can just return true here
1602 return true;
1603 }
1604
1605 #define __ _masm.
1606
1607 // advance declarations for helper functions to convert register
1608 // indices to register objects
1609
1610 // the ad file has to provide implementations of certain methods
1611 // expected by the generic code
1612 //
1613 // REQUIRED FUNCTIONALITY
1614
1615 //=============================================================================
1616
1617 // !!!!! Special hack to get all types of calls to specify the byte offset
1618 // from the start of the call to the point where the return address
1619 // will point.
1620
1621 int MachCallStaticJavaNode::ret_addr_offset()
1622 {
1623 // call should be a simple bl
1624 int off = 4;
1625 return off;
1626 }
1627
1628 int MachCallDynamicJavaNode::ret_addr_offset()
1629 {
1630 return 16; // movz, movk, movk, bl
1631 }
1632
1633 int MachCallRuntimeNode::ret_addr_offset() {
1634 // for generated stubs the call will be
1635 // bl(addr)
1636 // or with far branches
1637 // bl(trampoline_stub)
1638 // for real runtime callouts it will be six instructions
1639 // see aarch64_enc_java_to_runtime
1640 // adr(rscratch2, retaddr)
1641 // lea(rscratch1, RuntimeAddress(addr)
1642 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1643 // blr(rscratch1)
1644 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1645 if (cb) {
1646 return 1 * NativeInstruction::instruction_size;
1647 } else if (_entry_point == nullptr) {
1648 // See CallLeafNoFPIndirect
1649 return 1 * NativeInstruction::instruction_size;
1650 } else {
1651 return 6 * NativeInstruction::instruction_size;
1652 }
1653 }
1654
1655 //=============================================================================
1656
1657 #ifndef PRODUCT
1658 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1659 st->print("BREAKPOINT");
1660 }
1661 #endif
1662
1663 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1664 C2_MacroAssembler _masm(&cbuf);
1665 __ brk(0);
1666 }
1667
1668 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1669 return MachNode::size(ra_);
1670 }
1671
1672 //=============================================================================
1673
1674 #ifndef PRODUCT
1675 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1676 st->print("nop \t# %d bytes pad for loops and calls", _count);
1677 }
1678 #endif
1679
1680 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1681 C2_MacroAssembler _masm(&cbuf);
1682 for (int i = 0; i < _count; i++) {
1683 __ nop();
1684 }
1685 }
1686
1687 uint MachNopNode::size(PhaseRegAlloc*) const {
1688 return _count * NativeInstruction::instruction_size;
1689 }
1690
1691 //=============================================================================
1692 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1693
1694 int ConstantTable::calculate_table_base_offset() const {
1695 return 0; // absolute addressing, no offset
1696 }
1697
1698 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1699 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1700 ShouldNotReachHere();
1701 }
1702
1703 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1704 // Empty encoding
1705 }
1706
1707 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1708 return 0;
1709 }
1710
1711 #ifndef PRODUCT
1712 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1713 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1714 }
1715 #endif
1716
1717 #ifndef PRODUCT
1718 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1719 Compile* C = ra_->C;
1720
1721 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1722
1723 if (C->output()->need_stack_bang(framesize))
1724 st->print("# stack bang size=%d\n\t", framesize);
1725
1726 if (VM_Version::use_rop_protection()) {
1727 st->print("ldr zr, [lr]\n\t");
1728 st->print("pacia lr, rfp\n\t");
1729 }
1730 if (framesize < ((1 << 9) + 2 * wordSize)) {
1731 st->print("sub sp, sp, #%d\n\t", framesize);
1732 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1733 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1734 } else {
1735 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1736 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1737 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1738 st->print("sub sp, sp, rscratch1");
1739 }
1740 if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1741 st->print("\n\t");
1742 st->print("ldr rscratch1, [guard]\n\t");
1743 st->print("dmb ishld\n\t");
1744 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1745 st->print("cmp rscratch1, rscratch2\n\t");
1746 st->print("b.eq skip");
1747 st->print("\n\t");
1748 st->print("blr #nmethod_entry_barrier_stub\n\t");
1749 st->print("b skip\n\t");
1750 st->print("guard: int\n\t");
1751 st->print("\n\t");
1752 st->print("skip:\n\t");
1753 }
1754 }
1755 #endif
1756
1757 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1758 Compile* C = ra_->C;
1759 C2_MacroAssembler _masm(&cbuf);
1760
1761 // insert a nop at the start of the prolog so we can patch in a
1762 // branch if we need to invalidate the method later
1763 __ nop();
1764
1765 __ verified_entry(C, 0);
1766
1767 if (C->stub_function() == nullptr) {
1768 __ entry_barrier();
1769 }
1770
1771 if (!Compile::current()->output()->in_scratch_emit_size()) {
1772 __ bind(*_verified_entry);
1773 }
1774
1775 if (VerifyStackAtCalls) {
1776 Unimplemented();
1777 }
1778
1779 C->output()->set_frame_complete(cbuf.insts_size());
1780
1781 if (C->has_mach_constant_base_node()) {
1782 // NOTE: We set the table base offset here because users might be
1783 // emitted before MachConstantBaseNode.
1784 ConstantTable& constant_table = C->output()->constant_table();
1785 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1786 }
1787 }
1788
1789 int MachPrologNode::reloc() const
1790 {
1791 return 0;
1792 }
1793
1794 //=============================================================================
1795
1796 #ifndef PRODUCT
1797 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1798 Compile* C = ra_->C;
1799 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1800
1801 st->print("# pop frame %d\n\t",framesize);
1802
1803 if (framesize == 0) {
1804 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1805 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1806 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1807 st->print("add sp, sp, #%d\n\t", framesize);
1808 } else {
1809 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1810 st->print("add sp, sp, rscratch1\n\t");
1811 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1812 }
1813 if (VM_Version::use_rop_protection()) {
1814 st->print("autia lr, rfp\n\t");
1815 st->print("ldr zr, [lr]\n\t");
1816 }
1817
1818 if (do_polling() && C->is_method_compilation()) {
1819 st->print("# test polling word\n\t");
1820 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1821 st->print("cmp sp, rscratch1\n\t");
1822 st->print("bhi #slow_path");
1823 }
1824 }
1825 #endif
1826
1827 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1828 Compile* C = ra_->C;
1829 C2_MacroAssembler _masm(&cbuf);
1830 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1831
1832 __ remove_frame(framesize, C->needs_stack_repair());
1833
1834 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1835 __ reserved_stack_check();
1836 }
1837
1838 if (do_polling() && C->is_method_compilation()) {
1839 Label dummy_label;
1840 Label* code_stub = &dummy_label;
1841 if (!C->output()->in_scratch_emit_size()) {
1842 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1843 C->output()->add_stub(stub);
1844 code_stub = &stub->entry();
1845 }
1846 __ relocate(relocInfo::poll_return_type);
1847 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1848 }
1849 }
1850
1851 int MachEpilogNode::reloc() const {
1852 // Return number of relocatable values contained in this instruction.
1853 return 1; // 1 for polling page.
1854 }
1855
1856 const Pipeline * MachEpilogNode::pipeline() const {
1857 return MachNode::pipeline_class();
1858 }
1859
1860 //=============================================================================
1861
1862 // Figure out which register class each belongs in: rc_int, rc_float or
1863 // rc_stack.
1864 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
1865
1866 static enum RC rc_class(OptoReg::Name reg) {
1867
1868 if (reg == OptoReg::Bad) {
1869 return rc_bad;
1870 }
1871
1872 // we have 32 int registers * 2 halves
1873 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1874
1875 if (reg < slots_of_int_registers) {
1876 return rc_int;
1877 }
1878
1879 // we have 32 float register * 8 halves
1880 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1881 if (reg < slots_of_int_registers + slots_of_float_registers) {
1882 return rc_float;
1883 }
1884
1885 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1886 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1887 return rc_predicate;
1888 }
1889
1890 // Between predicate regs & stack is the flags.
1891 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1892
1893 return rc_stack;
1894 }
1895
1896 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1897 Compile* C = ra_->C;
1898
1899 // Get registers to move.
1900 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1901 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1902 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1903 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1904
1905 enum RC src_hi_rc = rc_class(src_hi);
1906 enum RC src_lo_rc = rc_class(src_lo);
1907 enum RC dst_hi_rc = rc_class(dst_hi);
1908 enum RC dst_lo_rc = rc_class(dst_lo);
1909
1910 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1911
1912 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1913 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1914 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1915 "expected aligned-adjacent pairs");
1916 }
1917
1918 if (src_lo == dst_lo && src_hi == dst_hi) {
1919 return 0; // Self copy, no move.
1920 }
1921
1922 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1923 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1924 int src_offset = ra_->reg2offset(src_lo);
1925 int dst_offset = ra_->reg2offset(dst_lo);
1926
1927 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1928 uint ireg = ideal_reg();
1929 if (ireg == Op_VecA && cbuf) {
1930 C2_MacroAssembler _masm(cbuf);
1931 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1932 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1933 // stack->stack
1934 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1935 sve_vector_reg_size_in_bytes);
1936 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1937 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1938 sve_vector_reg_size_in_bytes);
1939 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1940 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1941 sve_vector_reg_size_in_bytes);
1942 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1943 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1944 as_FloatRegister(Matcher::_regEncode[src_lo]),
1945 as_FloatRegister(Matcher::_regEncode[src_lo]));
1946 } else {
1947 ShouldNotReachHere();
1948 }
1949 } else if (cbuf) {
1950 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1951 C2_MacroAssembler _masm(cbuf);
1952 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1953 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1954 // stack->stack
1955 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1956 if (ireg == Op_VecD) {
1957 __ unspill(rscratch1, true, src_offset);
1958 __ spill(rscratch1, true, dst_offset);
1959 } else {
1960 __ spill_copy128(src_offset, dst_offset);
1961 }
1962 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1963 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1964 ireg == Op_VecD ? __ T8B : __ T16B,
1965 as_FloatRegister(Matcher::_regEncode[src_lo]));
1966 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1967 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1968 ireg == Op_VecD ? __ D : __ Q,
1969 ra_->reg2offset(dst_lo));
1970 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1971 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1972 ireg == Op_VecD ? __ D : __ Q,
1973 ra_->reg2offset(src_lo));
1974 } else {
1975 ShouldNotReachHere();
1976 }
1977 }
1978 } else if (cbuf) {
1979 C2_MacroAssembler _masm(cbuf);
1980 switch (src_lo_rc) {
1981 case rc_int:
1982 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
1983 if (is64) {
1984 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1985 as_Register(Matcher::_regEncode[src_lo]));
1986 } else {
1987 C2_MacroAssembler _masm(cbuf);
1988 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1989 as_Register(Matcher::_regEncode[src_lo]));
1990 }
1991 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1992 if (is64) {
1993 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1994 as_Register(Matcher::_regEncode[src_lo]));
1995 } else {
1996 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1997 as_Register(Matcher::_regEncode[src_lo]));
1998 }
1999 } else { // gpr --> stack spill
2000 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2001 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2002 }
2003 break;
2004 case rc_float:
2005 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2006 if (is64) {
2007 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2008 as_FloatRegister(Matcher::_regEncode[src_lo]));
2009 } else {
2010 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2011 as_FloatRegister(Matcher::_regEncode[src_lo]));
2012 }
2013 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2014 if (is64) {
2015 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2016 as_FloatRegister(Matcher::_regEncode[src_lo]));
2017 } else {
2018 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2019 as_FloatRegister(Matcher::_regEncode[src_lo]));
2020 }
2021 } else { // fpr --> stack spill
2022 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2023 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2024 is64 ? __ D : __ S, dst_offset);
2025 }
2026 break;
2027 case rc_stack:
2028 if (dst_lo_rc == rc_int) { // stack --> gpr load
2029 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2030 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2031 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2032 is64 ? __ D : __ S, src_offset);
2033 } else if (dst_lo_rc == rc_predicate) {
2034 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2035 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2036 } else { // stack --> stack copy
2037 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2038 if (ideal_reg() == Op_RegVectMask) {
2039 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2040 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2041 } else {
2042 __ unspill(rscratch1, is64, src_offset);
2043 __ spill(rscratch1, is64, dst_offset);
2044 }
2045 }
2046 break;
2047 case rc_predicate:
2048 if (dst_lo_rc == rc_predicate) {
2049 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2050 } else if (dst_lo_rc == rc_stack) {
2051 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2052 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2053 } else {
2054 assert(false, "bad src and dst rc_class combination.");
2055 ShouldNotReachHere();
2056 }
2057 break;
2058 default:
2059 assert(false, "bad rc_class for spill");
2060 ShouldNotReachHere();
2061 }
2062 }
2063
2064 if (st) {
2065 st->print("spill ");
2066 if (src_lo_rc == rc_stack) {
2067 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2068 } else {
2069 st->print("%s -> ", Matcher::regName[src_lo]);
2070 }
2071 if (dst_lo_rc == rc_stack) {
2072 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2073 } else {
2074 st->print("%s", Matcher::regName[dst_lo]);
2075 }
2076 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2077 int vsize = 0;
2078 switch (ideal_reg()) {
2079 case Op_VecD:
2080 vsize = 64;
2081 break;
2082 case Op_VecX:
2083 vsize = 128;
2084 break;
2085 case Op_VecA:
2086 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2087 break;
2088 default:
2089 assert(false, "bad register type for spill");
2090 ShouldNotReachHere();
2091 }
2092 st->print("\t# vector spill size = %d", vsize);
2093 } else if (ideal_reg() == Op_RegVectMask) {
2094 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2095 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2096 st->print("\t# predicate spill size = %d", vsize);
2097 } else {
2098 st->print("\t# spill size = %d", is64 ? 64 : 32);
2099 }
2100 }
2101
2102 return 0;
2103
2104 }
2105
2106 #ifndef PRODUCT
2107 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2108 if (!ra_)
2109 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2110 else
2111 implementation(nullptr, ra_, false, st);
2112 }
2113 #endif
2114
2115 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2116 implementation(&cbuf, ra_, false, nullptr);
2117 }
2118
2119 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2120 return MachNode::size(ra_);
2121 }
2122
2123 //=============================================================================
2124
2125 #ifndef PRODUCT
2126 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2127 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2128 int reg = ra_->get_reg_first(this);
2129 st->print("add %s, rsp, #%d]\t# box lock",
2130 Matcher::regName[reg], offset);
2131 }
2132 #endif
2133
2134 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2135 C2_MacroAssembler _masm(&cbuf);
2136
2137 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2138 int reg = ra_->get_encode(this);
2139
2140 // This add will handle any 24-bit signed offset. 24 bits allows an
2141 // 8 megabyte stack frame.
2142 __ add(as_Register(reg), sp, offset);
2143 }
2144
2145 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2146 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2147 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2148
2149 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2150 return NativeInstruction::instruction_size;
2151 } else {
2152 return 2 * NativeInstruction::instruction_size;
2153 }
2154 }
2155
2156 ///=============================================================================
2157 #ifndef PRODUCT
2158 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2159 {
2160 st->print_cr("# MachVEPNode");
2161 if (!_verified) {
2162 st->print_cr("\t load_class");
2163 } else {
2164 st->print_cr("\t unpack_inline_arg");
2165 }
2166 }
2167 #endif
2168
2169 void MachVEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2170 {
2171 C2_MacroAssembler _masm(&cbuf);
2172
2173 if (!_verified) {
2174 Label skip;
2175 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2176 __ br(Assembler::EQ, skip);
2177 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2178 __ bind(skip);
2179
2180 } else {
2181 // insert a nop at the start of the prolog so we can patch in a
2182 // branch if we need to invalidate the method later
2183 __ nop();
2184
2185 // TODO 8284443 Avoid creation of temporary frame
2186 if (ra_->C->stub_function() == nullptr) {
2187 __ verified_entry(ra_->C, 0);
2188 __ entry_barrier();
2189 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2190 __ remove_frame(framesize, false);
2191 }
2192 // Unpack inline type args passed as oop and then jump to
2193 // the verified entry point (skipping the unverified entry).
2194 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2195 // Emit code for verified entry and save increment for stack repair on return
2196 __ verified_entry(ra_->C, sp_inc);
2197 if (Compile::current()->output()->in_scratch_emit_size()) {
2198 Label dummy_verified_entry;
2199 __ b(dummy_verified_entry);
2200 } else {
2201 __ b(*_verified_entry);
2202 }
2203 }
2204 }
2205
2206 //=============================================================================
2207 #ifndef PRODUCT
2208 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2209 {
2210 st->print_cr("# MachUEPNode");
2211 if (UseCompressedClassPointers) {
2212 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2213 if (CompressedKlassPointers::shift() != 0) {
2214 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2215 }
2216 } else {
2217 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2218 }
2219 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2220 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2221 }
2222 #endif
2223
2224 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2225 {
2226 // This is the unverified entry point.
2227 C2_MacroAssembler _masm(&cbuf);
2228 Label skip;
2229
2230 // UseCompressedClassPointers logic are inside cmp_klass
2231 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2232
2233 // TODO
2234 // can we avoid this skip and still use a reloc?
2235 __ br(Assembler::EQ, skip);
2236 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2237 __ bind(skip);
2238 }
2239
2240 // REQUIRED EMIT CODE
2241
2242 //=============================================================================
2243
2244 // Emit exception handler code.
2245 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2246 {
2247 // mov rscratch1 #exception_blob_entry_point
2248 // br rscratch1
2249 // Note that the code buffer's insts_mark is always relative to insts.
2250 // That's why we must use the macroassembler to generate a handler.
2251 C2_MacroAssembler _masm(&cbuf);
2252 address base = __ start_a_stub(size_exception_handler());
2253 if (base == nullptr) {
2254 ciEnv::current()->record_failure("CodeCache is full");
2255 return 0; // CodeBuffer::expand failed
2256 }
2257 int offset = __ offset();
2258 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2259 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2260 __ end_a_stub();
2261 return offset;
2262 }
2263
2264 // Emit deopt handler code.
2265 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2266 {
2267 // Note that the code buffer's insts_mark is always relative to insts.
2268 // That's why we must use the macroassembler to generate a handler.
2269 C2_MacroAssembler _masm(&cbuf);
2270 address base = __ start_a_stub(size_deopt_handler());
2271 if (base == nullptr) {
2272 ciEnv::current()->record_failure("CodeCache is full");
2273 return 0; // CodeBuffer::expand failed
2274 }
2275 int offset = __ offset();
2276
2277 __ adr(lr, __ pc());
2278 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2279
2280 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2281 __ end_a_stub();
2282 return offset;
2283 }
2284
2285 // REQUIRED MATCHER CODE
2286
2287 //=============================================================================
2288
2289 bool Matcher::match_rule_supported(int opcode) {
2290 if (!has_match_rule(opcode))
2291 return false;
2292
2293 bool ret_value = true;
2294 switch (opcode) {
2295 case Op_OnSpinWait:
2296 return VM_Version::supports_on_spin_wait();
2297 case Op_CacheWB:
2298 case Op_CacheWBPreSync:
2299 case Op_CacheWBPostSync:
2300 if (!VM_Version::supports_data_cache_line_flush()) {
2301 ret_value = false;
2302 }
2303 break;
2304 case Op_ExpandBits:
2305 case Op_CompressBits:
2306 if (!VM_Version::supports_svebitperm()) {
2307 ret_value = false;
2308 }
2309 break;
2310 }
2311
2312 return ret_value; // Per default match rules are supported.
2313 }
2314
2315 const RegMask* Matcher::predicate_reg_mask(void) {
2316 return &_PR_REG_mask;
2317 }
2318
2319 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2320 return new TypeVectMask(elemTy, length);
2321 }
2322
2323 // Vector calling convention not yet implemented.
2324 bool Matcher::supports_vector_calling_convention(void) {
2325 return false;
2326 }
2327
2328 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2329 Unimplemented();
2330 return OptoRegPair(0, 0);
2331 }
2332
2333 // Is this branch offset short enough that a short branch can be used?
2334 //
2335 // NOTE: If the platform does not provide any short branch variants, then
2336 // this method should return false for offset 0.
2337 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2338 // The passed offset is relative to address of the branch.
2339
2340 return (-32768 <= offset && offset < 32768);
2341 }
2342
2343 // Vector width in bytes.
2344 int Matcher::vector_width_in_bytes(BasicType bt) {
2345 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2346 int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
2347 // Minimum 2 values in vector
2348 if (size < 2*type2aelembytes(bt)) size = 0;
2349 // But never < 4
2350 if (size < 4) size = 0;
2351 return size;
2352 }
2353
2354 // Limits on vector size (number of elements) loaded into vector.
2355 int Matcher::max_vector_size(const BasicType bt) {
2356 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2357 }
2358
2359 int Matcher::min_vector_size(const BasicType bt) {
2360 int max_size = max_vector_size(bt);
2361 // Limit the min vector size to 8 bytes.
2362 int size = 8 / type2aelembytes(bt);
2363 if (bt == T_BYTE) {
2364 // To support vector api shuffle/rearrange.
2365 size = 4;
2366 } else if (bt == T_BOOLEAN) {
2367 // To support vector api load/store mask.
2368 size = 2;
2369 }
2370 if (size < 2) size = 2;
2371 return MIN2(size, max_size);
2372 }
2373
2374 int Matcher::superword_max_vector_size(const BasicType bt) {
2375 return Matcher::max_vector_size(bt);
2376 }
2377
2378 // Actual max scalable vector register length.
2379 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2380 return Matcher::max_vector_size(bt);
2381 }
2382
2383 // Vector ideal reg.
2384 uint Matcher::vector_ideal_reg(int len) {
2385 if (UseSVE > 0 && 16 < len && len <= 256) {
2386 return Op_VecA;
2387 }
2388 switch(len) {
2389 // For 16-bit/32-bit mask vector, reuse VecD.
2390 case 2:
2391 case 4:
2392 case 8: return Op_VecD;
2393 case 16: return Op_VecX;
2394 }
2395 ShouldNotReachHere();
2396 return 0;
2397 }
2398
2399 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2400 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2401 switch (ideal_reg) {
2402 case Op_VecA: return new vecAOper();
2403 case Op_VecD: return new vecDOper();
2404 case Op_VecX: return new vecXOper();
2405 }
2406 ShouldNotReachHere();
2407 return nullptr;
2408 }
2409
2410 bool Matcher::is_reg2reg_move(MachNode* m) {
2411 return false;
2412 }
2413
2414 bool Matcher::is_generic_vector(MachOper* opnd) {
2415 return opnd->opcode() == VREG;
2416 }
2417
2418 // Return whether or not this register is ever used as an argument.
2419 // This function is used on startup to build the trampoline stubs in
2420 // generateOptoStub. Registers not mentioned will be killed by the VM
2421 // call in the trampoline, and arguments in those registers not be
2422 // available to the callee.
2423 bool Matcher::can_be_java_arg(int reg)
2424 {
2425 return
2426 reg == R0_num || reg == R0_H_num ||
2427 reg == R1_num || reg == R1_H_num ||
2428 reg == R2_num || reg == R2_H_num ||
2429 reg == R3_num || reg == R3_H_num ||
2430 reg == R4_num || reg == R4_H_num ||
2431 reg == R5_num || reg == R5_H_num ||
2432 reg == R6_num || reg == R6_H_num ||
2433 reg == R7_num || reg == R7_H_num ||
2434 reg == V0_num || reg == V0_H_num ||
2435 reg == V1_num || reg == V1_H_num ||
2436 reg == V2_num || reg == V2_H_num ||
2437 reg == V3_num || reg == V3_H_num ||
2438 reg == V4_num || reg == V4_H_num ||
2439 reg == V5_num || reg == V5_H_num ||
2440 reg == V6_num || reg == V6_H_num ||
2441 reg == V7_num || reg == V7_H_num;
2442 }
2443
2444 bool Matcher::is_spillable_arg(int reg)
2445 {
2446 return can_be_java_arg(reg);
2447 }
2448
2449 uint Matcher::int_pressure_limit()
2450 {
2451 // JDK-8183543: When taking the number of available registers as int
2452 // register pressure threshold, the jtreg test:
2453 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2454 // failed due to C2 compilation failure with
2455 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2456 //
2457 // A derived pointer is live at CallNode and then is flagged by RA
2458 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2459 // derived pointers and lastly fail to spill after reaching maximum
2460 // number of iterations. Lowering the default pressure threshold to
2461 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2462 // a high register pressure area of the code so that split_DEF can
2463 // generate DefinitionSpillCopy for the derived pointer.
2464 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2465 if (!PreserveFramePointer) {
2466 // When PreserveFramePointer is off, frame pointer is allocatable,
2467 // but different from other SOC registers, it is excluded from
2468 // fatproj's mask because its save type is No-Save. Decrease 1 to
2469 // ensure high pressure at fatproj when PreserveFramePointer is off.
2470 // See check_pressure_at_fatproj().
2471 default_int_pressure_threshold--;
2472 }
2473 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2474 }
2475
2476 uint Matcher::float_pressure_limit()
2477 {
2478 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2479 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2480 }
2481
2482 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2483 return false;
2484 }
2485
2486 RegMask Matcher::divI_proj_mask() {
2487 ShouldNotReachHere();
2488 return RegMask();
2489 }
2490
2491 // Register for MODI projection of divmodI.
2492 RegMask Matcher::modI_proj_mask() {
2493 ShouldNotReachHere();
2494 return RegMask();
2495 }
2496
2497 // Register for DIVL projection of divmodL.
2498 RegMask Matcher::divL_proj_mask() {
2499 ShouldNotReachHere();
2500 return RegMask();
2501 }
2502
2503 // Register for MODL projection of divmodL.
2504 RegMask Matcher::modL_proj_mask() {
2505 ShouldNotReachHere();
2506 return RegMask();
2507 }
2508
2509 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2510 return FP_REG_mask();
2511 }
2512
2513 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2514 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2515 Node* u = addp->fast_out(i);
2516 if (u->is_LoadStore()) {
2517 // On AArch64, LoadStoreNodes (i.e. compare and swap
2518 // instructions) only take register indirect as an operand, so
2519 // any attempt to use an AddPNode as an input to a LoadStoreNode
2520 // must fail.
2521 return false;
2522 }
2523 if (u->is_Mem()) {
2524 int opsize = u->as_Mem()->memory_size();
2525 assert(opsize > 0, "unexpected memory operand size");
2526 if (u->as_Mem()->memory_size() != (1<<shift)) {
2527 return false;
2528 }
2529 }
2530 }
2531 return true;
2532 }
2533
2534 // Convert BootTest condition to Assembler condition.
2535 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2536 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2537 Assembler::Condition result;
2538 switch(cond) {
2539 case BoolTest::eq:
2540 result = Assembler::EQ; break;
2541 case BoolTest::ne:
2542 result = Assembler::NE; break;
2543 case BoolTest::le:
2544 result = Assembler::LE; break;
2545 case BoolTest::ge:
2546 result = Assembler::GE; break;
2547 case BoolTest::lt:
2548 result = Assembler::LT; break;
2549 case BoolTest::gt:
2550 result = Assembler::GT; break;
2551 case BoolTest::ule:
2552 result = Assembler::LS; break;
2553 case BoolTest::uge:
2554 result = Assembler::HS; break;
2555 case BoolTest::ult:
2556 result = Assembler::LO; break;
2557 case BoolTest::ugt:
2558 result = Assembler::HI; break;
2559 case BoolTest::overflow:
2560 result = Assembler::VS; break;
2561 case BoolTest::no_overflow:
2562 result = Assembler::VC; break;
2563 default:
2564 ShouldNotReachHere();
2565 return Assembler::Condition(-1);
2566 }
2567
2568 // Check conversion
2569 if (cond & BoolTest::unsigned_compare) {
2570 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2571 } else {
2572 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2573 }
2574
2575 return result;
2576 }
2577
2578 // Binary src (Replicate con)
2579 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2580 if (n == nullptr || m == nullptr) {
2581 return false;
2582 }
2583
2584 if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
2585 return false;
2586 }
2587
2588 Node* imm_node = m->in(1);
2589 if (!imm_node->is_Con()) {
2590 return false;
2591 }
2592
2593 const Type* t = imm_node->bottom_type();
2594 if (!(t->isa_int() || t->isa_long())) {
2595 return false;
2596 }
2597
2598 switch (n->Opcode()) {
2599 case Op_AndV:
2600 case Op_OrV:
2601 case Op_XorV: {
2602 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2603 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2604 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2605 }
2606 case Op_AddVB:
2607 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2608 case Op_AddVS:
2609 case Op_AddVI:
2610 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2611 case Op_AddVL:
2612 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2613 default:
2614 return false;
2615 }
2616 }
2617
2618 // (XorV src (Replicate m1))
2619 // (XorVMask src (MaskAll m1))
2620 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2621 if (n != nullptr && m != nullptr) {
2622 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2623 VectorNode::is_all_ones_vector(m);
2624 }
2625 return false;
2626 }
2627
2628 // Should the matcher clone input 'm' of node 'n'?
2629 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2630 if (is_vshift_con_pattern(n, m) ||
2631 is_vector_bitwise_not_pattern(n, m) ||
2632 is_valid_sve_arith_imm_pattern(n, m)) {
2633 mstack.push(m, Visit);
2634 return true;
2635 }
2636 return false;
2637 }
2638
2639 // Should the Matcher clone shifts on addressing modes, expecting them
2640 // to be subsumed into complex addressing expressions or compute them
2641 // into registers?
2642 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2643 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2644 return true;
2645 }
2646
2647 Node *off = m->in(AddPNode::Offset);
2648 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2649 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2650 // Are there other uses besides address expressions?
2651 !is_visited(off)) {
2652 address_visited.set(off->_idx); // Flag as address_visited
2653 mstack.push(off->in(2), Visit);
2654 Node *conv = off->in(1);
2655 if (conv->Opcode() == Op_ConvI2L &&
2656 // Are there other uses besides address expressions?
2657 !is_visited(conv)) {
2658 address_visited.set(conv->_idx); // Flag as address_visited
2659 mstack.push(conv->in(1), Pre_Visit);
2660 } else {
2661 mstack.push(conv, Pre_Visit);
2662 }
2663 address_visited.test_set(m->_idx); // Flag as address_visited
2664 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2665 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2666 return true;
2667 } else if (off->Opcode() == Op_ConvI2L &&
2668 // Are there other uses besides address expressions?
2669 !is_visited(off)) {
2670 address_visited.test_set(m->_idx); // Flag as address_visited
2671 address_visited.set(off->_idx); // Flag as address_visited
2672 mstack.push(off->in(1), Pre_Visit);
2673 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2674 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2675 return true;
2676 }
2677 return false;
2678 }
2679
2680 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2681 C2_MacroAssembler _masm(&cbuf); \
2682 { \
2683 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2684 guarantee(DISP == 0, "mode not permitted for volatile"); \
2685 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2686 __ INSN(REG, as_Register(BASE)); \
2687 }
2688
2689
2690 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2691 {
2692 Address::extend scale;
2693
2694 // Hooboy, this is fugly. We need a way to communicate to the
2695 // encoder that the index needs to be sign extended, so we have to
2696 // enumerate all the cases.
2697 switch (opcode) {
2698 case INDINDEXSCALEDI2L:
2699 case INDINDEXSCALEDI2LN:
2700 case INDINDEXI2L:
2701 case INDINDEXI2LN:
2702 scale = Address::sxtw(size);
2703 break;
2704 default:
2705 scale = Address::lsl(size);
2706 }
2707
2708 if (index == -1) {
2709 return Address(base, disp);
2710 } else {
2711 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2712 return Address(base, as_Register(index), scale);
2713 }
2714 }
2715
2716
2717 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2718 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2719 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2720 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2721 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2722
2723 // Used for all non-volatile memory accesses. The use of
2724 // $mem->opcode() to discover whether this pattern uses sign-extended
2725 // offsets is something of a kludge.
2726 static void loadStore(C2_MacroAssembler masm, mem_insn insn,
2727 Register reg, int opcode,
2728 Register base, int index, int scale, int disp,
2729 int size_in_memory)
2730 {
2731 Address addr = mem2address(opcode, base, index, scale, disp);
2732 if (addr.getMode() == Address::base_plus_offset) {
2733 /* If we get an out-of-range offset it is a bug in the compiler,
2734 so we assert here. */
2735 assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
2736 "c2 compiler bug");
2737 /* Fix up any out-of-range offsets. */
2738 assert_different_registers(rscratch1, base);
2739 assert_different_registers(rscratch1, reg);
2740 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2741 }
2742 (masm.*insn)(reg, addr);
2743 }
2744
2745 static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
2746 FloatRegister reg, int opcode,
2747 Register base, int index, int size, int disp,
2748 int size_in_memory)
2749 {
2750 Address::extend scale;
2751
2752 switch (opcode) {
2753 case INDINDEXSCALEDI2L:
2754 case INDINDEXSCALEDI2LN:
2755 scale = Address::sxtw(size);
2756 break;
2757 default:
2758 scale = Address::lsl(size);
2759 }
2760
2761 if (index == -1) {
2762 /* If we get an out-of-range offset it is a bug in the compiler,
2763 so we assert here. */
2764 assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2765 /* Fix up any out-of-range offsets. */
2766 assert_different_registers(rscratch1, base);
2767 Address addr = Address(base, disp);
2768 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2769 (masm.*insn)(reg, addr);
2770 } else {
2771 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2772 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2773 }
2774 }
2775
2776 static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
2777 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2778 int opcode, Register base, int index, int size, int disp)
2779 {
2780 if (index == -1) {
2781 (masm.*insn)(reg, T, Address(base, disp));
2782 } else {
2783 assert(disp == 0, "unsupported address mode");
2784 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2785 }
2786 }
2787
2788 %}
2789
2790
2791
2792 //----------ENCODING BLOCK-----------------------------------------------------
2793 // This block specifies the encoding classes used by the compiler to
2794 // output byte streams. Encoding classes are parameterized macros
2795 // used by Machine Instruction Nodes in order to generate the bit
2796 // encoding of the instruction. Operands specify their base encoding
2797 // interface with the interface keyword. There are currently
2798 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2799 // COND_INTER. REG_INTER causes an operand to generate a function
2800 // which returns its register number when queried. CONST_INTER causes
2801 // an operand to generate a function which returns the value of the
2802 // constant when queried. MEMORY_INTER causes an operand to generate
2803 // four functions which return the Base Register, the Index Register,
2804 // the Scale Value, and the Offset Value of the operand when queried.
2805 // COND_INTER causes an operand to generate six functions which return
2806 // the encoding code (ie - encoding bits for the instruction)
2807 // associated with each basic boolean condition for a conditional
2808 // instruction.
2809 //
2810 // Instructions specify two basic values for encoding. Again, a
2811 // function is available to check if the constant displacement is an
2812 // oop. They use the ins_encode keyword to specify their encoding
2813 // classes (which must be a sequence of enc_class names, and their
2814 // parameters, specified in the encoding block), and they use the
2815 // opcode keyword to specify, in order, their primary, secondary, and
2816 // tertiary opcode. Only the opcode sections which a particular
2817 // instruction needs for encoding need to be specified.
2818 encode %{
2819 // Build emit functions for each basic byte or larger field in the
2820 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2821 // from C++ code in the enc_class source block. Emit functions will
2822 // live in the main source block for now. In future, we can
2823 // generalize this by adding a syntax that specifies the sizes of
2824 // fields in an order, so that the adlc can build the emit functions
2825 // automagically
2826
2827 // catch all for unimplemented encodings
2828 enc_class enc_unimplemented %{
2829 C2_MacroAssembler _masm(&cbuf);
2830 __ unimplemented("C2 catch all");
2831 %}
2832
2833 // BEGIN Non-volatile memory access
2834
2835 // This encoding class is generated automatically from ad_encode.m4.
2836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2837 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2838 Register dst_reg = as_Register($dst$$reg);
2839 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2840 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2841 %}
2842
2843 // This encoding class is generated automatically from ad_encode.m4.
2844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2845 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2846 Register dst_reg = as_Register($dst$$reg);
2847 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2848 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2849 %}
2850
2851 // This encoding class is generated automatically from ad_encode.m4.
2852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2853 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2854 Register dst_reg = as_Register($dst$$reg);
2855 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2856 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2857 %}
2858
2859 // This encoding class is generated automatically from ad_encode.m4.
2860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2861 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2862 Register dst_reg = as_Register($dst$$reg);
2863 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2864 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2865 %}
2866
2867 // This encoding class is generated automatically from ad_encode.m4.
2868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2869 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2870 Register dst_reg = as_Register($dst$$reg);
2871 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2872 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2873 %}
2874
2875 // This encoding class is generated automatically from ad_encode.m4.
2876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2877 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2878 Register dst_reg = as_Register($dst$$reg);
2879 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2880 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2881 %}
2882
2883 // This encoding class is generated automatically from ad_encode.m4.
2884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2885 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2886 Register dst_reg = as_Register($dst$$reg);
2887 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2888 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2889 %}
2890
2891 // This encoding class is generated automatically from ad_encode.m4.
2892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2893 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2894 Register dst_reg = as_Register($dst$$reg);
2895 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2897 %}
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2926 Register dst_reg = as_Register($dst$$reg);
2927 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2934 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2935 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2942 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2943 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2950 Register src_reg = as_Register($src$$reg);
2951 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2952 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2953 %}
2954
2955 // This encoding class is generated automatically from ad_encode.m4.
2956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2957 enc_class aarch64_enc_strb0(memory1 mem) %{
2958 C2_MacroAssembler _masm(&cbuf);
2959 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2960 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2961 %}
2962
2963 // This encoding class is generated automatically from ad_encode.m4.
2964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2965 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2966 Register src_reg = as_Register($src$$reg);
2967 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2968 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2969 %}
2970
2971 // This encoding class is generated automatically from ad_encode.m4.
2972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2973 enc_class aarch64_enc_strh0(memory2 mem) %{
2974 C2_MacroAssembler _masm(&cbuf);
2975 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2976 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2977 %}
2978
2979 // This encoding class is generated automatically from ad_encode.m4.
2980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2981 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2982 Register src_reg = as_Register($src$$reg);
2983 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2984 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2985 %}
2986
2987 // This encoding class is generated automatically from ad_encode.m4.
2988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2989 enc_class aarch64_enc_strw0(memory4 mem) %{
2990 C2_MacroAssembler _masm(&cbuf);
2991 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2992 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2993 %}
2994
2995 // This encoding class is generated automatically from ad_encode.m4.
2996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2997 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2998 Register src_reg = as_Register($src$$reg);
2999 // we sometimes get asked to store the stack pointer into the
3000 // current thread -- we cannot do that directly on AArch64
3001 if (src_reg == r31_sp) {
3002 C2_MacroAssembler _masm(&cbuf);
3003 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3004 __ mov(rscratch2, sp);
3005 src_reg = rscratch2;
3006 }
3007 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
3008 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3009 %}
3010
3011 // This encoding class is generated automatically from ad_encode.m4.
3012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3013 enc_class aarch64_enc_str0(memory8 mem) %{
3014 C2_MacroAssembler _masm(&cbuf);
3015 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
3016 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3017 %}
3018
3019 // This encoding class is generated automatically from ad_encode.m4.
3020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3021 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3022 FloatRegister src_reg = as_FloatRegister($src$$reg);
3023 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
3024 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3025 %}
3026
3027 // This encoding class is generated automatically from ad_encode.m4.
3028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3029 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3030 FloatRegister src_reg = as_FloatRegister($src$$reg);
3031 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
3032 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3033 %}
3034
3035 // This encoding class is generated automatically from ad_encode.m4.
3036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3037 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3038 C2_MacroAssembler _masm(&cbuf);
3039 __ membar(Assembler::StoreStore);
3040 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
3041 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3042 %}
3043
3044 // END Non-volatile memory access
3045
3046 // Vector loads and stores
3047 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3048 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3049 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3050 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3051 %}
3052
3053 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3054 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3055 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3056 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3057 %}
3058
3059 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3060 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3061 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3062 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3063 %}
3064
3065 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3066 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3067 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3068 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3069 %}
3070
3071 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3072 FloatRegister src_reg = as_FloatRegister($src$$reg);
3073 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
3074 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3075 %}
3076
3077 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3078 FloatRegister src_reg = as_FloatRegister($src$$reg);
3079 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
3080 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3081 %}
3082
3083 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3084 FloatRegister src_reg = as_FloatRegister($src$$reg);
3085 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
3086 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3087 %}
3088
3089 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3090 FloatRegister src_reg = as_FloatRegister($src$$reg);
3091 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
3092 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3093 %}
3094
3095 // volatile loads and stores
3096
3097 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3098 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3099 rscratch1, stlrb);
3100 %}
3101
3102 enc_class aarch64_enc_stlrb0(memory mem) %{
3103 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3104 rscratch1, stlrb);
3105 %}
3106
3107 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3108 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3109 rscratch1, stlrh);
3110 %}
3111
3112 enc_class aarch64_enc_stlrh0(memory mem) %{
3113 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3114 rscratch1, stlrh);
3115 %}
3116
3117 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3118 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3119 rscratch1, stlrw);
3120 %}
3121
3122 enc_class aarch64_enc_stlrw0(memory mem) %{
3123 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3124 rscratch1, stlrw);
3125 %}
3126
3127 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3128 Register dst_reg = as_Register($dst$$reg);
3129 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3130 rscratch1, ldarb);
3131 __ sxtbw(dst_reg, dst_reg);
3132 %}
3133
3134 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3135 Register dst_reg = as_Register($dst$$reg);
3136 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3137 rscratch1, ldarb);
3138 __ sxtb(dst_reg, dst_reg);
3139 %}
3140
3141 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3142 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3143 rscratch1, ldarb);
3144 %}
3145
3146 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3147 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3148 rscratch1, ldarb);
3149 %}
3150
3151 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3152 Register dst_reg = as_Register($dst$$reg);
3153 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3154 rscratch1, ldarh);
3155 __ sxthw(dst_reg, dst_reg);
3156 %}
3157
3158 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3159 Register dst_reg = as_Register($dst$$reg);
3160 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3161 rscratch1, ldarh);
3162 __ sxth(dst_reg, dst_reg);
3163 %}
3164
3165 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3166 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3167 rscratch1, ldarh);
3168 %}
3169
3170 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3171 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3172 rscratch1, ldarh);
3173 %}
3174
3175 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3176 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3177 rscratch1, ldarw);
3178 %}
3179
3180 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3181 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3182 rscratch1, ldarw);
3183 %}
3184
3185 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3186 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3187 rscratch1, ldar);
3188 %}
3189
3190 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3191 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3192 rscratch1, ldarw);
3193 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3194 %}
3195
3196 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3197 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3198 rscratch1, ldar);
3199 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3200 %}
3201
3202 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3203 Register src_reg = as_Register($src$$reg);
3204 // we sometimes get asked to store the stack pointer into the
3205 // current thread -- we cannot do that directly on AArch64
3206 if (src_reg == r31_sp) {
3207 C2_MacroAssembler _masm(&cbuf);
3208 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3209 __ mov(rscratch2, sp);
3210 src_reg = rscratch2;
3211 }
3212 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3213 rscratch1, stlr);
3214 %}
3215
3216 enc_class aarch64_enc_stlr0(memory mem) %{
3217 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3218 rscratch1, stlr);
3219 %}
3220
3221 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3222 {
3223 C2_MacroAssembler _masm(&cbuf);
3224 FloatRegister src_reg = as_FloatRegister($src$$reg);
3225 __ fmovs(rscratch2, src_reg);
3226 }
3227 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3228 rscratch1, stlrw);
3229 %}
3230
3231 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3232 {
3233 C2_MacroAssembler _masm(&cbuf);
3234 FloatRegister src_reg = as_FloatRegister($src$$reg);
3235 __ fmovd(rscratch2, src_reg);
3236 }
3237 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3238 rscratch1, stlr);
3239 %}
3240
3241 // synchronized read/update encodings
3242
3243 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3244 C2_MacroAssembler _masm(&cbuf);
3245 Register dst_reg = as_Register($dst$$reg);
3246 Register base = as_Register($mem$$base);
3247 int index = $mem$$index;
3248 int scale = $mem$$scale;
3249 int disp = $mem$$disp;
3250 if (index == -1) {
3251 if (disp != 0) {
3252 __ lea(rscratch1, Address(base, disp));
3253 __ ldaxr(dst_reg, rscratch1);
3254 } else {
3255 // TODO
3256 // should we ever get anything other than this case?
3257 __ ldaxr(dst_reg, base);
3258 }
3259 } else {
3260 Register index_reg = as_Register(index);
3261 if (disp == 0) {
3262 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3263 __ ldaxr(dst_reg, rscratch1);
3264 } else {
3265 __ lea(rscratch1, Address(base, disp));
3266 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3267 __ ldaxr(dst_reg, rscratch1);
3268 }
3269 }
3270 %}
3271
3272 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3273 C2_MacroAssembler _masm(&cbuf);
3274 Register src_reg = as_Register($src$$reg);
3275 Register base = as_Register($mem$$base);
3276 int index = $mem$$index;
3277 int scale = $mem$$scale;
3278 int disp = $mem$$disp;
3279 if (index == -1) {
3280 if (disp != 0) {
3281 __ lea(rscratch2, Address(base, disp));
3282 __ stlxr(rscratch1, src_reg, rscratch2);
3283 } else {
3284 // TODO
3285 // should we ever get anything other than this case?
3286 __ stlxr(rscratch1, src_reg, base);
3287 }
3288 } else {
3289 Register index_reg = as_Register(index);
3290 if (disp == 0) {
3291 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3292 __ stlxr(rscratch1, src_reg, rscratch2);
3293 } else {
3294 __ lea(rscratch2, Address(base, disp));
3295 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3296 __ stlxr(rscratch1, src_reg, rscratch2);
3297 }
3298 }
3299 __ cmpw(rscratch1, zr);
3300 %}
3301
3302 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3303 C2_MacroAssembler _masm(&cbuf);
3304 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3305 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3306 Assembler::xword, /*acquire*/ false, /*release*/ true,
3307 /*weak*/ false, noreg);
3308 %}
3309
3310 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3311 C2_MacroAssembler _masm(&cbuf);
3312 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3313 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3314 Assembler::word, /*acquire*/ false, /*release*/ true,
3315 /*weak*/ false, noreg);
3316 %}
3317
3318 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3319 C2_MacroAssembler _masm(&cbuf);
3320 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3321 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3322 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3323 /*weak*/ false, noreg);
3324 %}
3325
3326 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3327 C2_MacroAssembler _masm(&cbuf);
3328 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3329 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3330 Assembler::byte, /*acquire*/ false, /*release*/ true,
3331 /*weak*/ false, noreg);
3332 %}
3333
3334
3335 // The only difference between aarch64_enc_cmpxchg and
3336 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3337 // CompareAndSwap sequence to serve as a barrier on acquiring a
3338 // lock.
3339 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3340 C2_MacroAssembler _masm(&cbuf);
3341 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3342 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3343 Assembler::xword, /*acquire*/ true, /*release*/ true,
3344 /*weak*/ false, noreg);
3345 %}
3346
3347 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3348 C2_MacroAssembler _masm(&cbuf);
3349 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3350 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3351 Assembler::word, /*acquire*/ true, /*release*/ true,
3352 /*weak*/ false, noreg);
3353 %}
3354
3355 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3356 C2_MacroAssembler _masm(&cbuf);
3357 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3358 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3359 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3360 /*weak*/ false, noreg);
3361 %}
3362
3363 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3364 C2_MacroAssembler _masm(&cbuf);
3365 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3366 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3367 Assembler::byte, /*acquire*/ true, /*release*/ true,
3368 /*weak*/ false, noreg);
3369 %}
3370
3371 // auxiliary used for CompareAndSwapX to set result register
3372 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3373 C2_MacroAssembler _masm(&cbuf);
3374 Register res_reg = as_Register($res$$reg);
3375 __ cset(res_reg, Assembler::EQ);
3376 %}
3377
3378 // prefetch encodings
3379
3380 enc_class aarch64_enc_prefetchw(memory mem) %{
3381 C2_MacroAssembler _masm(&cbuf);
3382 Register base = as_Register($mem$$base);
3383 int index = $mem$$index;
3384 int scale = $mem$$scale;
3385 int disp = $mem$$disp;
3386 if (index == -1) {
3387 __ prfm(Address(base, disp), PSTL1KEEP);
3388 } else {
3389 Register index_reg = as_Register(index);
3390 if (disp == 0) {
3391 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3392 } else {
3393 __ lea(rscratch1, Address(base, disp));
3394 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3395 }
3396 }
3397 %}
3398
3399 /// mov envcodings
3400
3401 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3402 C2_MacroAssembler _masm(&cbuf);
3403 uint32_t con = (uint32_t)$src$$constant;
3404 Register dst_reg = as_Register($dst$$reg);
3405 if (con == 0) {
3406 __ movw(dst_reg, zr);
3407 } else {
3408 __ movw(dst_reg, con);
3409 }
3410 %}
3411
3412 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3413 C2_MacroAssembler _masm(&cbuf);
3414 Register dst_reg = as_Register($dst$$reg);
3415 uint64_t con = (uint64_t)$src$$constant;
3416 if (con == 0) {
3417 __ mov(dst_reg, zr);
3418 } else {
3419 __ mov(dst_reg, con);
3420 }
3421 %}
3422
3423 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3424 C2_MacroAssembler _masm(&cbuf);
3425 Register dst_reg = as_Register($dst$$reg);
3426 address con = (address)$src$$constant;
3427 if (con == nullptr || con == (address)1) {
3428 ShouldNotReachHere();
3429 } else {
3430 relocInfo::relocType rtype = $src->constant_reloc();
3431 if (rtype == relocInfo::oop_type) {
3432 __ movoop(dst_reg, (jobject)con);
3433 } else if (rtype == relocInfo::metadata_type) {
3434 __ mov_metadata(dst_reg, (Metadata*)con);
3435 } else {
3436 assert(rtype == relocInfo::none, "unexpected reloc type");
3437 if (! __ is_valid_AArch64_address(con) ||
3438 con < (address)(uintptr_t)os::vm_page_size()) {
3439 __ mov(dst_reg, con);
3440 } else {
3441 uint64_t offset;
3442 __ adrp(dst_reg, con, offset);
3443 __ add(dst_reg, dst_reg, offset);
3444 }
3445 }
3446 }
3447 %}
3448
3449 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3450 C2_MacroAssembler _masm(&cbuf);
3451 Register dst_reg = as_Register($dst$$reg);
3452 __ mov(dst_reg, zr);
3453 %}
3454
3455 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3456 C2_MacroAssembler _masm(&cbuf);
3457 Register dst_reg = as_Register($dst$$reg);
3458 __ mov(dst_reg, (uint64_t)1);
3459 %}
3460
3461 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3462 C2_MacroAssembler _masm(&cbuf);
3463 __ load_byte_map_base($dst$$Register);
3464 %}
3465
3466 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3467 C2_MacroAssembler _masm(&cbuf);
3468 Register dst_reg = as_Register($dst$$reg);
3469 address con = (address)$src$$constant;
3470 if (con == nullptr) {
3471 ShouldNotReachHere();
3472 } else {
3473 relocInfo::relocType rtype = $src->constant_reloc();
3474 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3475 __ set_narrow_oop(dst_reg, (jobject)con);
3476 }
3477 %}
3478
3479 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3480 C2_MacroAssembler _masm(&cbuf);
3481 Register dst_reg = as_Register($dst$$reg);
3482 __ mov(dst_reg, zr);
3483 %}
3484
3485 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3486 C2_MacroAssembler _masm(&cbuf);
3487 Register dst_reg = as_Register($dst$$reg);
3488 address con = (address)$src$$constant;
3489 if (con == nullptr) {
3490 ShouldNotReachHere();
3491 } else {
3492 relocInfo::relocType rtype = $src->constant_reloc();
3493 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3494 __ set_narrow_klass(dst_reg, (Klass *)con);
3495 }
3496 %}
3497
3498 // arithmetic encodings
3499
3500 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3501 C2_MacroAssembler _masm(&cbuf);
3502 Register dst_reg = as_Register($dst$$reg);
3503 Register src_reg = as_Register($src1$$reg);
3504 int32_t con = (int32_t)$src2$$constant;
3505 // add has primary == 0, subtract has primary == 1
3506 if ($primary) { con = -con; }
3507 if (con < 0) {
3508 __ subw(dst_reg, src_reg, -con);
3509 } else {
3510 __ addw(dst_reg, src_reg, con);
3511 }
3512 %}
3513
3514 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3515 C2_MacroAssembler _masm(&cbuf);
3516 Register dst_reg = as_Register($dst$$reg);
3517 Register src_reg = as_Register($src1$$reg);
3518 int32_t con = (int32_t)$src2$$constant;
3519 // add has primary == 0, subtract has primary == 1
3520 if ($primary) { con = -con; }
3521 if (con < 0) {
3522 __ sub(dst_reg, src_reg, -con);
3523 } else {
3524 __ add(dst_reg, src_reg, con);
3525 }
3526 %}
3527
3528 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3529 C2_MacroAssembler _masm(&cbuf);
3530 Register dst_reg = as_Register($dst$$reg);
3531 Register src1_reg = as_Register($src1$$reg);
3532 Register src2_reg = as_Register($src2$$reg);
3533 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3534 %}
3535
3536 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3537 C2_MacroAssembler _masm(&cbuf);
3538 Register dst_reg = as_Register($dst$$reg);
3539 Register src1_reg = as_Register($src1$$reg);
3540 Register src2_reg = as_Register($src2$$reg);
3541 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3542 %}
3543
3544 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3545 C2_MacroAssembler _masm(&cbuf);
3546 Register dst_reg = as_Register($dst$$reg);
3547 Register src1_reg = as_Register($src1$$reg);
3548 Register src2_reg = as_Register($src2$$reg);
3549 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3550 %}
3551
3552 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3553 C2_MacroAssembler _masm(&cbuf);
3554 Register dst_reg = as_Register($dst$$reg);
3555 Register src1_reg = as_Register($src1$$reg);
3556 Register src2_reg = as_Register($src2$$reg);
3557 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3558 %}
3559
3560 // compare instruction encodings
3561
3562 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3563 C2_MacroAssembler _masm(&cbuf);
3564 Register reg1 = as_Register($src1$$reg);
3565 Register reg2 = as_Register($src2$$reg);
3566 __ cmpw(reg1, reg2);
3567 %}
3568
3569 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3570 C2_MacroAssembler _masm(&cbuf);
3571 Register reg = as_Register($src1$$reg);
3572 int32_t val = $src2$$constant;
3573 if (val >= 0) {
3574 __ subsw(zr, reg, val);
3575 } else {
3576 __ addsw(zr, reg, -val);
3577 }
3578 %}
3579
3580 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3581 C2_MacroAssembler _masm(&cbuf);
3582 Register reg1 = as_Register($src1$$reg);
3583 uint32_t val = (uint32_t)$src2$$constant;
3584 __ movw(rscratch1, val);
3585 __ cmpw(reg1, rscratch1);
3586 %}
3587
3588 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3589 C2_MacroAssembler _masm(&cbuf);
3590 Register reg1 = as_Register($src1$$reg);
3591 Register reg2 = as_Register($src2$$reg);
3592 __ cmp(reg1, reg2);
3593 %}
3594
3595 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3596 C2_MacroAssembler _masm(&cbuf);
3597 Register reg = as_Register($src1$$reg);
3598 int64_t val = $src2$$constant;
3599 if (val >= 0) {
3600 __ subs(zr, reg, val);
3601 } else if (val != -val) {
3602 __ adds(zr, reg, -val);
3603 } else {
3604 // aargh, Long.MIN_VALUE is a special case
3605 __ orr(rscratch1, zr, (uint64_t)val);
3606 __ subs(zr, reg, rscratch1);
3607 }
3608 %}
3609
3610 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3611 C2_MacroAssembler _masm(&cbuf);
3612 Register reg1 = as_Register($src1$$reg);
3613 uint64_t val = (uint64_t)$src2$$constant;
3614 __ mov(rscratch1, val);
3615 __ cmp(reg1, rscratch1);
3616 %}
3617
3618 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3619 C2_MacroAssembler _masm(&cbuf);
3620 Register reg1 = as_Register($src1$$reg);
3621 Register reg2 = as_Register($src2$$reg);
3622 __ cmp(reg1, reg2);
3623 %}
3624
3625 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3626 C2_MacroAssembler _masm(&cbuf);
3627 Register reg1 = as_Register($src1$$reg);
3628 Register reg2 = as_Register($src2$$reg);
3629 __ cmpw(reg1, reg2);
3630 %}
3631
3632 enc_class aarch64_enc_testp(iRegP src) %{
3633 C2_MacroAssembler _masm(&cbuf);
3634 Register reg = as_Register($src$$reg);
3635 __ cmp(reg, zr);
3636 %}
3637
3638 enc_class aarch64_enc_testn(iRegN src) %{
3639 C2_MacroAssembler _masm(&cbuf);
3640 Register reg = as_Register($src$$reg);
3641 __ cmpw(reg, zr);
3642 %}
3643
3644 enc_class aarch64_enc_b(label lbl) %{
3645 C2_MacroAssembler _masm(&cbuf);
3646 Label *L = $lbl$$label;
3647 __ b(*L);
3648 %}
3649
3650 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3651 C2_MacroAssembler _masm(&cbuf);
3652 Label *L = $lbl$$label;
3653 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3654 %}
3655
3656 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3657 C2_MacroAssembler _masm(&cbuf);
3658 Label *L = $lbl$$label;
3659 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3660 %}
3661
3662 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3663 %{
3664 Register sub_reg = as_Register($sub$$reg);
3665 Register super_reg = as_Register($super$$reg);
3666 Register temp_reg = as_Register($temp$$reg);
3667 Register result_reg = as_Register($result$$reg);
3668
3669 Label miss;
3670 C2_MacroAssembler _masm(&cbuf);
3671 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3672 nullptr, &miss,
3673 /*set_cond_codes:*/ true);
3674 if ($primary) {
3675 __ mov(result_reg, zr);
3676 }
3677 __ bind(miss);
3678 %}
3679
3680 enc_class aarch64_enc_java_static_call(method meth) %{
3681 C2_MacroAssembler _masm(&cbuf);
3682
3683 address addr = (address)$meth$$method;
3684 address call;
3685 if (!_method) {
3686 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3687 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3688 if (call == nullptr) {
3689 ciEnv::current()->record_failure("CodeCache is full");
3690 return;
3691 }
3692 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3693 // The NOP here is purely to ensure that eliding a call to
3694 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3695 __ nop();
3696 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3697 } else {
3698 int method_index = resolved_method_index(cbuf);
3699 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3700 : static_call_Relocation::spec(method_index);
3701 call = __ trampoline_call(Address(addr, rspec));
3702 if (call == nullptr) {
3703 ciEnv::current()->record_failure("CodeCache is full");
3704 return;
3705 }
3706 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3707 // Calls of the same statically bound method can share
3708 // a stub to the interpreter.
3709 cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
3710 } else {
3711 // Emit stub for static call
3712 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
3713 if (stub == nullptr) {
3714 ciEnv::current()->record_failure("CodeCache is full");
3715 return;
3716 }
3717 }
3718 }
3719
3720 __ post_call_nop();
3721
3722 // Only non uncommon_trap calls need to reinitialize ptrue.
3723 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3724 __ reinitialize_ptrue();
3725 }
3726 %}
3727
3728 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3729 C2_MacroAssembler _masm(&cbuf);
3730 int method_index = resolved_method_index(cbuf);
3731 address call = __ ic_call((address)$meth$$method, method_index);
3732 if (call == nullptr) {
3733 ciEnv::current()->record_failure("CodeCache is full");
3734 return;
3735 }
3736 __ post_call_nop();
3737 if (Compile::current()->max_vector_size() > 0) {
3738 __ reinitialize_ptrue();
3739 }
3740 %}
3741
3742 enc_class aarch64_enc_call_epilog() %{
3743 C2_MacroAssembler _masm(&cbuf);
3744 if (VerifyStackAtCalls) {
3745 // Check that stack depth is unchanged: find majik cookie on stack
3746 __ call_Unimplemented();
3747 }
3748 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
3749 if (!_method->signature()->returns_null_free_inline_type()) {
3750 // The last return value is not set by the callee but used to pass IsInit information to compiled code.
3751 // Search for the corresponding projection, get the register and emit code that initialized it.
3752 uint con = (tf()->range_cc()->cnt() - 1);
3753 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3754 ProjNode* proj = fast_out(i)->as_Proj();
3755 if (proj->_con == con) {
3756 // Set IsInit if r0 is non-null (a non-null value is returned buffered or scalarized)
3757 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3758 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3759 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3760 __ cmp(r0, zr);
3761 __ cset(toReg, Assembler::NE);
3762 if (reg->is_stack()) {
3763 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3764 __ str(toReg, Address(sp, st_off));
3765 }
3766 break;
3767 }
3768 }
3769 }
3770 if (return_value_is_used()) {
3771 // An inline type is returned as fields in multiple registers.
3772 // R0 either contains an oop if the inline type is buffered or a pointer
3773 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3774 // if the lowest bit is set to allow C2 to use the oop after null checking.
3775 // r0 &= (r0 & 1) - 1
3776 __ andr(rscratch1, r0, 0x1);
3777 __ sub(rscratch1, rscratch1, 0x1);
3778 __ andr(r0, r0, rscratch1);
3779 }
3780 }
3781 %}
3782
3783 enc_class aarch64_enc_java_to_runtime(method meth) %{
3784 C2_MacroAssembler _masm(&cbuf);
3785
3786 // some calls to generated routines (arraycopy code) are scheduled
3787 // by C2 as runtime calls. if so we can call them using a br (they
3788 // will be in a reachable segment) otherwise we have to use a blr
3789 // which loads the absolute address into a register.
3790 address entry = (address)$meth$$method;
3791 CodeBlob *cb = CodeCache::find_blob(entry);
3792 if (cb) {
3793 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3794 if (call == nullptr) {
3795 ciEnv::current()->record_failure("CodeCache is full");
3796 return;
3797 }
3798 __ post_call_nop();
3799 } else {
3800 Label retaddr;
3801 __ adr(rscratch2, retaddr);
3802 __ lea(rscratch1, RuntimeAddress(entry));
3803 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3804 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3805 __ blr(rscratch1);
3806 __ bind(retaddr);
3807 __ post_call_nop();
3808 __ add(sp, sp, 2 * wordSize);
3809 }
3810 if (Compile::current()->max_vector_size() > 0) {
3811 __ reinitialize_ptrue();
3812 }
3813 %}
3814
3815 enc_class aarch64_enc_rethrow() %{
3816 C2_MacroAssembler _masm(&cbuf);
3817 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3818 %}
3819
3820 enc_class aarch64_enc_ret() %{
3821 C2_MacroAssembler _masm(&cbuf);
3822 #ifdef ASSERT
3823 if (Compile::current()->max_vector_size() > 0) {
3824 __ verify_ptrue();
3825 }
3826 #endif
3827 __ ret(lr);
3828 %}
3829
3830 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3831 C2_MacroAssembler _masm(&cbuf);
3832 Register target_reg = as_Register($jump_target$$reg);
3833 __ br(target_reg);
3834 %}
3835
3836 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3837 C2_MacroAssembler _masm(&cbuf);
3838 Register target_reg = as_Register($jump_target$$reg);
3839 // exception oop should be in r0
3840 // ret addr has been popped into lr
3841 // callee expects it in r3
3842 __ mov(r3, lr);
3843 __ br(target_reg);
3844 %}
3845
3846 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3847 C2_MacroAssembler _masm(&cbuf);
3848 Register oop = as_Register($object$$reg);
3849 Register box = as_Register($box$$reg);
3850 Register disp_hdr = as_Register($tmp$$reg);
3851 Register tmp = as_Register($tmp2$$reg);
3852 Label cont;
3853 Label object_has_monitor;
3854 Label count, no_count;
3855
3856 assert_different_registers(oop, box, tmp, disp_hdr);
3857
3858 // Load markWord from object into displaced_header.
3859 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3860
3861 if (DiagnoseSyncOnValueBasedClasses != 0) {
3862 __ load_klass(tmp, oop);
3863 __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
3864 __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
3865 __ br(Assembler::NE, cont);
3866 }
3867
3868 // Check for existing monitor
3869 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3870
3871 if (LockingMode == LM_MONITOR) {
3872 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3873 __ b(cont);
3874 } else if (LockingMode == LM_LEGACY) {
3875 // Set tmp to be (markWord of object | UNLOCK_VALUE).
3876 __ orr(tmp, disp_hdr, markWord::unlocked_value);
3877
3878 if (EnableValhalla) {
3879 // Mask inline_type bit such that we go to the slow path if object is an inline type
3880 __ andr(tmp, tmp, ~((int) markWord::inline_type_bit_in_place));
3881 }
3882
3883 // Initialize the box. (Must happen before we update the object mark!)
3884 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3885
3886 // Compare object markWord with an unlocked value (tmp) and if
3887 // equal exchange the stack address of our box with object markWord.
3888 // On failure disp_hdr contains the possibly locked markWord.
3889 __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3890 /*release*/ true, /*weak*/ false, disp_hdr);
3891 __ br(Assembler::EQ, cont);
3892
3893 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3894
3895 // If the compare-and-exchange succeeded, then we found an unlocked
3896 // object, will have now locked it will continue at label cont
3897
3898 // Check if the owner is self by comparing the value in the
3899 // markWord of object (disp_hdr) with the stack pointer.
3900 __ mov(rscratch1, sp);
3901 __ sub(disp_hdr, disp_hdr, rscratch1);
3902 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
3903 // If condition is true we are cont and hence we can store 0 as the
3904 // displaced header in the box, which indicates that it is a recursive lock.
3905 __ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
3906 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3907 __ b(cont);
3908 } else {
3909 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
3910 __ fast_lock(oop, disp_hdr, tmp, rscratch1, no_count);
3911 __ b(count);
3912 }
3913
3914 // Handle existing monitor.
3915 __ bind(object_has_monitor);
3916
3917 // The object's monitor m is unlocked iff m->owner == nullptr,
3918 // otherwise m->owner may contain a thread or a stack address.
3919 //
3920 // Try to CAS m->owner from nullptr to current thread.
3921 __ add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value));
3922 __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3923 /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
3924
3925 if (LockingMode != LM_LIGHTWEIGHT) {
3926 // Store a non-null value into the box to avoid looking like a re-entrant
3927 // lock. The fast-path monitor unlock code checks for
3928 // markWord::monitor_value so use markWord::unused_mark which has the
3929 // relevant bit set, and also matches ObjectSynchronizer::enter.
3930 __ mov(tmp, (address)markWord::unused_mark().value());
3931 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3932 }
3933 __ br(Assembler::EQ, cont); // CAS success means locking succeeded
3934
3935 __ cmp(rscratch1, rthread);
3936 __ br(Assembler::NE, cont); // Check for recursive locking
3937
3938 // Recursive lock case
3939 __ increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1);
3940 // flag == EQ still from the cmp above, checking if this is a reentrant lock
3941
3942 __ bind(cont);
3943 // flag == EQ indicates success
3944 // flag == NE indicates failure
3945 __ br(Assembler::NE, no_count);
3946
3947 __ bind(count);
3948 __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
3949
3950 __ bind(no_count);
3951 %}
3952
3953 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3954 C2_MacroAssembler _masm(&cbuf);
3955 Register oop = as_Register($object$$reg);
3956 Register box = as_Register($box$$reg);
3957 Register disp_hdr = as_Register($tmp$$reg);
3958 Register tmp = as_Register($tmp2$$reg);
3959 Label cont;
3960 Label object_has_monitor;
3961 Label count, no_count;
3962
3963 assert_different_registers(oop, box, tmp, disp_hdr);
3964
3965 if (LockingMode == LM_LEGACY) {
3966 // Find the lock address and load the displaced header from the stack.
3967 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3968
3969 // If the displaced header is 0, we have a recursive unlock.
3970 __ cmp(disp_hdr, zr);
3971 __ br(Assembler::EQ, cont);
3972 }
3973
3974 // Handle existing monitor.
3975 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3976 __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
3977
3978 if (LockingMode == LM_MONITOR) {
3979 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3980 __ b(cont);
3981 } else if (LockingMode == LM_LEGACY) {
3982 // Check if it is still a light weight lock, this is is true if we
3983 // see the stack address of the basicLock in the markWord of the
3984 // object.
3985
3986 __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
3987 /*release*/ true, /*weak*/ false, tmp);
3988 __ b(cont);
3989 } else {
3990 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
3991 __ fast_unlock(oop, tmp, box, disp_hdr, no_count);
3992 __ b(count);
3993 }
3994
3995 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3996
3997 // Handle existing monitor.
3998 __ bind(object_has_monitor);
3999 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
4000 __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
4001
4002 if (LockingMode == LM_LIGHTWEIGHT) {
4003 // If the owner is anonymous, we need to fix it -- in an outline stub.
4004 Register tmp2 = disp_hdr;
4005 __ ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset()));
4006 // We cannot use tbnz here, the target might be too far away and cannot
4007 // be encoded.
4008 __ tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER);
4009 C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
4010 Compile::current()->output()->add_stub(stub);
4011 __ br(Assembler::NE, stub->entry());
4012 __ bind(stub->continuation());
4013 }
4014
4015 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
4016
4017 Label notRecursive;
4018 __ cbz(disp_hdr, notRecursive);
4019
4020 // Recursive lock
4021 __ sub(disp_hdr, disp_hdr, 1u);
4022 __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
4023 __ cmp(disp_hdr, disp_hdr); // Sets flags for result
4024 __ b(cont);
4025
4026 __ bind(notRecursive);
4027 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset()));
4028 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
4029 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
4030 __ cmp(rscratch1, zr); // Sets flags for result
4031 __ cbnz(rscratch1, cont);
4032 // need a release store here
4033 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset()));
4034 __ stlr(zr, tmp); // set unowned
4035
4036 __ bind(cont);
4037 // flag == EQ indicates success
4038 // flag == NE indicates failure
4039 __ br(Assembler::NE, no_count);
4040
4041 __ bind(count);
4042 __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
4043
4044 __ bind(no_count);
4045 %}
4046
4047 %}
4048
4049 //----------FRAME--------------------------------------------------------------
4050 // Definition of frame structure and management information.
4051 //
4052 // S T A C K L A Y O U T Allocators stack-slot number
4053 // | (to get allocators register number
4054 // G Owned by | | v add OptoReg::stack0())
4055 // r CALLER | |
4056 // o | +--------+ pad to even-align allocators stack-slot
4057 // w V | pad0 | numbers; owned by CALLER
4058 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
4059 // h ^ | in | 5
4060 // | | args | 4 Holes in incoming args owned by SELF
4061 // | | | | 3
4062 // | | +--------+
4063 // V | | old out| Empty on Intel, window on Sparc
4064 // | old |preserve| Must be even aligned.
4065 // | SP-+--------+----> Matcher::_old_SP, even aligned
4066 // | | in | 3 area for Intel ret address
4067 // Owned by |preserve| Empty on Sparc.
4068 // SELF +--------+
4069 // | | pad2 | 2 pad to align old SP
4070 // | +--------+ 1
4071 // | | locks | 0
4072 // | +--------+----> OptoReg::stack0(), even aligned
4073 // | | pad1 | 11 pad to align new SP
4074 // | +--------+
4075 // | | | 10
4076 // | | spills | 9 spills
4077 // V | | 8 (pad0 slot for callee)
4078 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
4079 // ^ | out | 7
4080 // | | args | 6 Holes in outgoing args owned by CALLEE
4081 // Owned by +--------+
4082 // CALLEE | new out| 6 Empty on Intel, window on Sparc
4083 // | new |preserve| Must be even-aligned.
4084 // | SP-+--------+----> Matcher::_new_SP, even aligned
4085 // | | |
4086 //
4087 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
4088 // known from SELF's arguments and the Java calling convention.
4089 // Region 6-7 is determined per call site.
4090 // Note 2: If the calling convention leaves holes in the incoming argument
4091 // area, those holes are owned by SELF. Holes in the outgoing area
4092 // are owned by the CALLEE. Holes should not be necessary in the
4093 // incoming area, as the Java calling convention is completely under
4094 // the control of the AD file. Doubles can be sorted and packed to
4095 // avoid holes. Holes in the outgoing arguments may be necessary for
4096 // varargs C calling conventions.
4097 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
4098 // even aligned with pad0 as needed.
4099 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
4100 // (the latter is true on Intel but is it false on AArch64?)
4101 // region 6-11 is even aligned; it may be padded out more so that
4102 // the region from SP to FP meets the minimum stack alignment.
4103 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
4104 // alignment. Region 11, pad1, may be dynamically extended so that
4105 // SP meets the minimum alignment.
4106
4107 frame %{
4108 // These three registers define part of the calling convention
4109 // between compiled code and the interpreter.
4110
4111 // Inline Cache Register or Method for I2C.
4112 inline_cache_reg(R12);
4113
4114 // Number of stack slots consumed by locking an object
4115 sync_stack_slots(2);
4116
4117 // Compiled code's Frame Pointer
4118 frame_pointer(R31);
4119
4120 // Interpreter stores its frame pointer in a register which is
4121 // stored to the stack by I2CAdaptors.
4122 // I2CAdaptors convert from interpreted java to compiled java.
4123 interpreter_frame_pointer(R29);
4124
4125 // Stack alignment requirement
4126 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4127
4128 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4129 // for calls to C. Supports the var-args backing area for register parms.
4130 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4131
4132 // The after-PROLOG location of the return address. Location of
4133 // return address specifies a type (REG or STACK) and a number
4134 // representing the register number (i.e. - use a register name) or
4135 // stack slot.
4136 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4137 // Otherwise, it is above the locks and verification slot and alignment word
4138 // TODO this may well be correct but need to check why that - 2 is there
4139 // ppc port uses 0 but we definitely need to allow for fixed_slots
4140 // which folds in the space used for monitors
4141 return_addr(STACK - 2 +
4142 align_up((Compile::current()->in_preserve_stack_slots() +
4143 Compile::current()->fixed_slots()),
4144 stack_alignment_in_slots()));
4145
4146 // Location of compiled Java return values. Same as C for now.
4147 return_value
4148 %{
4149 // TODO do we allow ideal_reg == Op_RegN???
4150 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4151 "only return normal values");
4152
4153 static const int lo[Op_RegL + 1] = { // enum name
4154 0, // Op_Node
4155 0, // Op_Set
4156 R0_num, // Op_RegN
4157 R0_num, // Op_RegI
4158 R0_num, // Op_RegP
4159 V0_num, // Op_RegF
4160 V0_num, // Op_RegD
4161 R0_num // Op_RegL
4162 };
4163
4164 static const int hi[Op_RegL + 1] = { // enum name
4165 0, // Op_Node
4166 0, // Op_Set
4167 OptoReg::Bad, // Op_RegN
4168 OptoReg::Bad, // Op_RegI
4169 R0_H_num, // Op_RegP
4170 OptoReg::Bad, // Op_RegF
4171 V0_H_num, // Op_RegD
4172 R0_H_num // Op_RegL
4173 };
4174
4175 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4176 %}
4177 %}
4178
4179 //----------ATTRIBUTES---------------------------------------------------------
4180 //----------Operand Attributes-------------------------------------------------
4181 op_attrib op_cost(1); // Required cost attribute
4182
4183 //----------Instruction Attributes---------------------------------------------
4184 ins_attrib ins_cost(INSN_COST); // Required cost attribute
4185 ins_attrib ins_size(32); // Required size attribute (in bits)
4186 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4187 // a non-matching short branch variant
4188 // of some long branch?
4189 ins_attrib ins_alignment(4); // Required alignment attribute (must
4190 // be a power of 2) specifies the
4191 // alignment that some part of the
4192 // instruction (not necessarily the
4193 // start) requires. If > 1, a
4194 // compute_padding() function must be
4195 // provided for the instruction
4196
4197 //----------OPERANDS-----------------------------------------------------------
4198 // Operand definitions must precede instruction definitions for correct parsing
4199 // in the ADLC because operands constitute user defined types which are used in
4200 // instruction definitions.
4201
4202 //----------Simple Operands----------------------------------------------------
4203
4204 // Integer operands 32 bit
4205 // 32 bit immediate
4206 operand immI()
4207 %{
4208 match(ConI);
4209
4210 op_cost(0);
4211 format %{ %}
4212 interface(CONST_INTER);
4213 %}
4214
4215 // 32 bit zero
4216 operand immI0()
4217 %{
4218 predicate(n->get_int() == 0);
4219 match(ConI);
4220
4221 op_cost(0);
4222 format %{ %}
4223 interface(CONST_INTER);
4224 %}
4225
4226 // 32 bit unit increment
4227 operand immI_1()
4228 %{
4229 predicate(n->get_int() == 1);
4230 match(ConI);
4231
4232 op_cost(0);
4233 format %{ %}
4234 interface(CONST_INTER);
4235 %}
4236
4237 // 32 bit unit decrement
4238 operand immI_M1()
4239 %{
4240 predicate(n->get_int() == -1);
4241 match(ConI);
4242
4243 op_cost(0);
4244 format %{ %}
4245 interface(CONST_INTER);
4246 %}
4247
4248 // Shift values for add/sub extension shift
4249 operand immIExt()
4250 %{
4251 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4252 match(ConI);
4253
4254 op_cost(0);
4255 format %{ %}
4256 interface(CONST_INTER);
4257 %}
4258
4259 operand immI_gt_1()
4260 %{
4261 predicate(n->get_int() > 1);
4262 match(ConI);
4263
4264 op_cost(0);
4265 format %{ %}
4266 interface(CONST_INTER);
4267 %}
4268
4269 operand immI_le_4()
4270 %{
4271 predicate(n->get_int() <= 4);
4272 match(ConI);
4273
4274 op_cost(0);
4275 format %{ %}
4276 interface(CONST_INTER);
4277 %}
4278
4279 operand immI_16()
4280 %{
4281 predicate(n->get_int() == 16);
4282 match(ConI);
4283
4284 op_cost(0);
4285 format %{ %}
4286 interface(CONST_INTER);
4287 %}
4288
4289 operand immI_24()
4290 %{
4291 predicate(n->get_int() == 24);
4292 match(ConI);
4293
4294 op_cost(0);
4295 format %{ %}
4296 interface(CONST_INTER);
4297 %}
4298
4299 operand immI_32()
4300 %{
4301 predicate(n->get_int() == 32);
4302 match(ConI);
4303
4304 op_cost(0);
4305 format %{ %}
4306 interface(CONST_INTER);
4307 %}
4308
4309 operand immI_48()
4310 %{
4311 predicate(n->get_int() == 48);
4312 match(ConI);
4313
4314 op_cost(0);
4315 format %{ %}
4316 interface(CONST_INTER);
4317 %}
4318
4319 operand immI_56()
4320 %{
4321 predicate(n->get_int() == 56);
4322 match(ConI);
4323
4324 op_cost(0);
4325 format %{ %}
4326 interface(CONST_INTER);
4327 %}
4328
4329 operand immI_63()
4330 %{
4331 predicate(n->get_int() == 63);
4332 match(ConI);
4333
4334 op_cost(0);
4335 format %{ %}
4336 interface(CONST_INTER);
4337 %}
4338
4339 operand immI_64()
4340 %{
4341 predicate(n->get_int() == 64);
4342 match(ConI);
4343
4344 op_cost(0);
4345 format %{ %}
4346 interface(CONST_INTER);
4347 %}
4348
4349 operand immI_255()
4350 %{
4351 predicate(n->get_int() == 255);
4352 match(ConI);
4353
4354 op_cost(0);
4355 format %{ %}
4356 interface(CONST_INTER);
4357 %}
4358
4359 operand immI_65535()
4360 %{
4361 predicate(n->get_int() == 65535);
4362 match(ConI);
4363
4364 op_cost(0);
4365 format %{ %}
4366 interface(CONST_INTER);
4367 %}
4368
4369 operand immI_positive()
4370 %{
4371 predicate(n->get_int() > 0);
4372 match(ConI);
4373
4374 op_cost(0);
4375 format %{ %}
4376 interface(CONST_INTER);
4377 %}
4378
4379 // BoolTest condition for signed compare
4380 operand immI_cmp_cond()
4381 %{
4382 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4383 match(ConI);
4384
4385 op_cost(0);
4386 format %{ %}
4387 interface(CONST_INTER);
4388 %}
4389
4390 // BoolTest condition for unsigned compare
4391 operand immI_cmpU_cond()
4392 %{
4393 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4394 match(ConI);
4395
4396 op_cost(0);
4397 format %{ %}
4398 interface(CONST_INTER);
4399 %}
4400
4401 operand immL_255()
4402 %{
4403 predicate(n->get_long() == 255L);
4404 match(ConL);
4405
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 operand immL_65535()
4412 %{
4413 predicate(n->get_long() == 65535L);
4414 match(ConL);
4415
4416 op_cost(0);
4417 format %{ %}
4418 interface(CONST_INTER);
4419 %}
4420
4421 operand immL_4294967295()
4422 %{
4423 predicate(n->get_long() == 4294967295L);
4424 match(ConL);
4425
4426 op_cost(0);
4427 format %{ %}
4428 interface(CONST_INTER);
4429 %}
4430
4431 operand immL_bitmask()
4432 %{
4433 predicate((n->get_long() != 0)
4434 && ((n->get_long() & 0xc000000000000000l) == 0)
4435 && is_power_of_2(n->get_long() + 1));
4436 match(ConL);
4437
4438 op_cost(0);
4439 format %{ %}
4440 interface(CONST_INTER);
4441 %}
4442
4443 operand immI_bitmask()
4444 %{
4445 predicate((n->get_int() != 0)
4446 && ((n->get_int() & 0xc0000000) == 0)
4447 && is_power_of_2(n->get_int() + 1));
4448 match(ConI);
4449
4450 op_cost(0);
4451 format %{ %}
4452 interface(CONST_INTER);
4453 %}
4454
4455 operand immL_positive_bitmaskI()
4456 %{
4457 predicate((n->get_long() != 0)
4458 && ((julong)n->get_long() < 0x80000000ULL)
4459 && is_power_of_2(n->get_long() + 1));
4460 match(ConL);
4461
4462 op_cost(0);
4463 format %{ %}
4464 interface(CONST_INTER);
4465 %}
4466
4467 // Scale values for scaled offset addressing modes (up to long but not quad)
4468 operand immIScale()
4469 %{
4470 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4471 match(ConI);
4472
4473 op_cost(0);
4474 format %{ %}
4475 interface(CONST_INTER);
4476 %}
4477
4478 // 26 bit signed offset -- for pc-relative branches
4479 operand immI26()
4480 %{
4481 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4482 match(ConI);
4483
4484 op_cost(0);
4485 format %{ %}
4486 interface(CONST_INTER);
4487 %}
4488
4489 // 19 bit signed offset -- for pc-relative loads
4490 operand immI19()
4491 %{
4492 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4493 match(ConI);
4494
4495 op_cost(0);
4496 format %{ %}
4497 interface(CONST_INTER);
4498 %}
4499
4500 // 5 bit signed integer
4501 operand immI5()
4502 %{
4503 predicate(Assembler::is_simm(n->get_int(), 5));
4504 match(ConI);
4505
4506 op_cost(0);
4507 format %{ %}
4508 interface(CONST_INTER);
4509 %}
4510
4511 // 7 bit unsigned integer
4512 operand immIU7()
4513 %{
4514 predicate(Assembler::is_uimm(n->get_int(), 7));
4515 match(ConI);
4516
4517 op_cost(0);
4518 format %{ %}
4519 interface(CONST_INTER);
4520 %}
4521
4522 // 12 bit unsigned offset -- for base plus immediate loads
4523 operand immIU12()
4524 %{
4525 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4526 match(ConI);
4527
4528 op_cost(0);
4529 format %{ %}
4530 interface(CONST_INTER);
4531 %}
4532
4533 operand immLU12()
4534 %{
4535 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4536 match(ConL);
4537
4538 op_cost(0);
4539 format %{ %}
4540 interface(CONST_INTER);
4541 %}
4542
4543 // Offset for scaled or unscaled immediate loads and stores
4544 operand immIOffset()
4545 %{
4546 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4547 match(ConI);
4548
4549 op_cost(0);
4550 format %{ %}
4551 interface(CONST_INTER);
4552 %}
4553
4554 operand immIOffset1()
4555 %{
4556 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4557 match(ConI);
4558
4559 op_cost(0);
4560 format %{ %}
4561 interface(CONST_INTER);
4562 %}
4563
4564 operand immIOffset2()
4565 %{
4566 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4567 match(ConI);
4568
4569 op_cost(0);
4570 format %{ %}
4571 interface(CONST_INTER);
4572 %}
4573
4574 operand immIOffset4()
4575 %{
4576 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4577 match(ConI);
4578
4579 op_cost(0);
4580 format %{ %}
4581 interface(CONST_INTER);
4582 %}
4583
4584 operand immIOffset8()
4585 %{
4586 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4587 match(ConI);
4588
4589 op_cost(0);
4590 format %{ %}
4591 interface(CONST_INTER);
4592 %}
4593
4594 operand immIOffset16()
4595 %{
4596 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4597 match(ConI);
4598
4599 op_cost(0);
4600 format %{ %}
4601 interface(CONST_INTER);
4602 %}
4603
4604 operand immLoffset()
4605 %{
4606 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4607 match(ConL);
4608
4609 op_cost(0);
4610 format %{ %}
4611 interface(CONST_INTER);
4612 %}
4613
4614 operand immLoffset1()
4615 %{
4616 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4617 match(ConL);
4618
4619 op_cost(0);
4620 format %{ %}
4621 interface(CONST_INTER);
4622 %}
4623
4624 operand immLoffset2()
4625 %{
4626 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4627 match(ConL);
4628
4629 op_cost(0);
4630 format %{ %}
4631 interface(CONST_INTER);
4632 %}
4633
4634 operand immLoffset4()
4635 %{
4636 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4637 match(ConL);
4638
4639 op_cost(0);
4640 format %{ %}
4641 interface(CONST_INTER);
4642 %}
4643
4644 operand immLoffset8()
4645 %{
4646 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4647 match(ConL);
4648
4649 op_cost(0);
4650 format %{ %}
4651 interface(CONST_INTER);
4652 %}
4653
4654 operand immLoffset16()
4655 %{
4656 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4657 match(ConL);
4658
4659 op_cost(0);
4660 format %{ %}
4661 interface(CONST_INTER);
4662 %}
4663
4664 // 5 bit signed long integer
4665 operand immL5()
4666 %{
4667 predicate(Assembler::is_simm(n->get_long(), 5));
4668 match(ConL);
4669
4670 op_cost(0);
4671 format %{ %}
4672 interface(CONST_INTER);
4673 %}
4674
4675 // 7 bit unsigned long integer
4676 operand immLU7()
4677 %{
4678 predicate(Assembler::is_uimm(n->get_long(), 7));
4679 match(ConL);
4680
4681 op_cost(0);
4682 format %{ %}
4683 interface(CONST_INTER);
4684 %}
4685
4686 // 8 bit signed value.
4687 operand immI8()
4688 %{
4689 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4690 match(ConI);
4691
4692 op_cost(0);
4693 format %{ %}
4694 interface(CONST_INTER);
4695 %}
4696
4697 // 8 bit signed value (simm8), or #simm8 LSL 8.
4698 operand immI8_shift8()
4699 %{
4700 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4701 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4702 match(ConI);
4703
4704 op_cost(0);
4705 format %{ %}
4706 interface(CONST_INTER);
4707 %}
4708
4709 // 8 bit signed value (simm8), or #simm8 LSL 8.
4710 operand immL8_shift8()
4711 %{
4712 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4713 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4714 match(ConL);
4715
4716 op_cost(0);
4717 format %{ %}
4718 interface(CONST_INTER);
4719 %}
4720
4721 // 8 bit integer valid for vector add sub immediate
4722 operand immBAddSubV()
4723 %{
4724 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4725 match(ConI);
4726
4727 op_cost(0);
4728 format %{ %}
4729 interface(CONST_INTER);
4730 %}
4731
4732 // 32 bit integer valid for add sub immediate
4733 operand immIAddSub()
4734 %{
4735 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4736 match(ConI);
4737 op_cost(0);
4738 format %{ %}
4739 interface(CONST_INTER);
4740 %}
4741
4742 // 32 bit integer valid for vector add sub immediate
4743 operand immIAddSubV()
4744 %{
4745 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4746 match(ConI);
4747
4748 op_cost(0);
4749 format %{ %}
4750 interface(CONST_INTER);
4751 %}
4752
4753 // 32 bit unsigned integer valid for logical immediate
4754
4755 operand immBLog()
4756 %{
4757 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4758 match(ConI);
4759
4760 op_cost(0);
4761 format %{ %}
4762 interface(CONST_INTER);
4763 %}
4764
4765 operand immSLog()
4766 %{
4767 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4768 match(ConI);
4769
4770 op_cost(0);
4771 format %{ %}
4772 interface(CONST_INTER);
4773 %}
4774
4775 operand immILog()
4776 %{
4777 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4778 match(ConI);
4779
4780 op_cost(0);
4781 format %{ %}
4782 interface(CONST_INTER);
4783 %}
4784
4785 // Integer operands 64 bit
4786 // 64 bit immediate
4787 operand immL()
4788 %{
4789 match(ConL);
4790
4791 op_cost(0);
4792 format %{ %}
4793 interface(CONST_INTER);
4794 %}
4795
4796 // 64 bit zero
4797 operand immL0()
4798 %{
4799 predicate(n->get_long() == 0);
4800 match(ConL);
4801
4802 op_cost(0);
4803 format %{ %}
4804 interface(CONST_INTER);
4805 %}
4806
4807 // 64 bit unit increment
4808 operand immL_1()
4809 %{
4810 predicate(n->get_long() == 1);
4811 match(ConL);
4812
4813 op_cost(0);
4814 format %{ %}
4815 interface(CONST_INTER);
4816 %}
4817
4818 // 64 bit unit decrement
4819 operand immL_M1()
4820 %{
4821 predicate(n->get_long() == -1);
4822 match(ConL);
4823
4824 op_cost(0);
4825 format %{ %}
4826 interface(CONST_INTER);
4827 %}
4828
4829 // 32 bit offset of pc in thread anchor
4830
4831 operand immL_pc_off()
4832 %{
4833 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4834 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4835 match(ConL);
4836
4837 op_cost(0);
4838 format %{ %}
4839 interface(CONST_INTER);
4840 %}
4841
4842 // 64 bit integer valid for add sub immediate
4843 operand immLAddSub()
4844 %{
4845 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4846 match(ConL);
4847 op_cost(0);
4848 format %{ %}
4849 interface(CONST_INTER);
4850 %}
4851
4852 // 64 bit integer valid for addv subv immediate
4853 operand immLAddSubV()
4854 %{
4855 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4856 match(ConL);
4857
4858 op_cost(0);
4859 format %{ %}
4860 interface(CONST_INTER);
4861 %}
4862
4863 // 64 bit integer valid for logical immediate
4864 operand immLLog()
4865 %{
4866 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4867 match(ConL);
4868 op_cost(0);
4869 format %{ %}
4870 interface(CONST_INTER);
4871 %}
4872
4873 // Long Immediate: low 32-bit mask
4874 operand immL_32bits()
4875 %{
4876 predicate(n->get_long() == 0xFFFFFFFFL);
4877 match(ConL);
4878 op_cost(0);
4879 format %{ %}
4880 interface(CONST_INTER);
4881 %}
4882
4883 // Pointer operands
4884 // Pointer Immediate
4885 operand immP()
4886 %{
4887 match(ConP);
4888
4889 op_cost(0);
4890 format %{ %}
4891 interface(CONST_INTER);
4892 %}
4893
4894 // nullptr Pointer Immediate
4895 operand immP0()
4896 %{
4897 predicate(n->get_ptr() == 0);
4898 match(ConP);
4899
4900 op_cost(0);
4901 format %{ %}
4902 interface(CONST_INTER);
4903 %}
4904
4905 // Pointer Immediate One
4906 // this is used in object initialization (initial object header)
4907 operand immP_1()
4908 %{
4909 predicate(n->get_ptr() == 1);
4910 match(ConP);
4911
4912 op_cost(0);
4913 format %{ %}
4914 interface(CONST_INTER);
4915 %}
4916
4917 // Card Table Byte Map Base
4918 operand immByteMapBase()
4919 %{
4920 // Get base of card map
4921 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4922 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4923 match(ConP);
4924
4925 op_cost(0);
4926 format %{ %}
4927 interface(CONST_INTER);
4928 %}
4929
4930 // Pointer Immediate Minus One
4931 // this is used when we want to write the current PC to the thread anchor
4932 operand immP_M1()
4933 %{
4934 predicate(n->get_ptr() == -1);
4935 match(ConP);
4936
4937 op_cost(0);
4938 format %{ %}
4939 interface(CONST_INTER);
4940 %}
4941
4942 // Pointer Immediate Minus Two
4943 // this is used when we want to write the current PC to the thread anchor
4944 operand immP_M2()
4945 %{
4946 predicate(n->get_ptr() == -2);
4947 match(ConP);
4948
4949 op_cost(0);
4950 format %{ %}
4951 interface(CONST_INTER);
4952 %}
4953
4954 // Float and Double operands
4955 // Double Immediate
4956 operand immD()
4957 %{
4958 match(ConD);
4959 op_cost(0);
4960 format %{ %}
4961 interface(CONST_INTER);
4962 %}
4963
4964 // Double Immediate: +0.0d
4965 operand immD0()
4966 %{
4967 predicate(jlong_cast(n->getd()) == 0);
4968 match(ConD);
4969
4970 op_cost(0);
4971 format %{ %}
4972 interface(CONST_INTER);
4973 %}
4974
4975 // constant 'double +0.0'.
4976 operand immDPacked()
4977 %{
4978 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4979 match(ConD);
4980 op_cost(0);
4981 format %{ %}
4982 interface(CONST_INTER);
4983 %}
4984
4985 // Float Immediate
4986 operand immF()
4987 %{
4988 match(ConF);
4989 op_cost(0);
4990 format %{ %}
4991 interface(CONST_INTER);
4992 %}
4993
4994 // Float Immediate: +0.0f.
4995 operand immF0()
4996 %{
4997 predicate(jint_cast(n->getf()) == 0);
4998 match(ConF);
4999
5000 op_cost(0);
5001 format %{ %}
5002 interface(CONST_INTER);
5003 %}
5004
5005 //
5006 operand immFPacked()
5007 %{
5008 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
5009 match(ConF);
5010 op_cost(0);
5011 format %{ %}
5012 interface(CONST_INTER);
5013 %}
5014
5015 // Narrow pointer operands
5016 // Narrow Pointer Immediate
5017 operand immN()
5018 %{
5019 match(ConN);
5020
5021 op_cost(0);
5022 format %{ %}
5023 interface(CONST_INTER);
5024 %}
5025
5026 // Narrow nullptr Pointer Immediate
5027 operand immN0()
5028 %{
5029 predicate(n->get_narrowcon() == 0);
5030 match(ConN);
5031
5032 op_cost(0);
5033 format %{ %}
5034 interface(CONST_INTER);
5035 %}
5036
5037 operand immNKlass()
5038 %{
5039 match(ConNKlass);
5040
5041 op_cost(0);
5042 format %{ %}
5043 interface(CONST_INTER);
5044 %}
5045
5046 // Integer 32 bit Register Operands
5047 // Integer 32 bitRegister (excludes SP)
5048 operand iRegI()
5049 %{
5050 constraint(ALLOC_IN_RC(any_reg32));
5051 match(RegI);
5052 match(iRegINoSp);
5053 op_cost(0);
5054 format %{ %}
5055 interface(REG_INTER);
5056 %}
5057
5058 // Integer 32 bit Register not Special
5059 operand iRegINoSp()
5060 %{
5061 constraint(ALLOC_IN_RC(no_special_reg32));
5062 match(RegI);
5063 op_cost(0);
5064 format %{ %}
5065 interface(REG_INTER);
5066 %}
5067
5068 // Integer 64 bit Register Operands
5069 // Integer 64 bit Register (includes SP)
5070 operand iRegL()
5071 %{
5072 constraint(ALLOC_IN_RC(any_reg));
5073 match(RegL);
5074 match(iRegLNoSp);
5075 op_cost(0);
5076 format %{ %}
5077 interface(REG_INTER);
5078 %}
5079
5080 // Integer 64 bit Register not Special
5081 operand iRegLNoSp()
5082 %{
5083 constraint(ALLOC_IN_RC(no_special_reg));
5084 match(RegL);
5085 match(iRegL_R0);
5086 format %{ %}
5087 interface(REG_INTER);
5088 %}
5089
5090 // Pointer Register Operands
5091 // Pointer Register
5092 operand iRegP()
5093 %{
5094 constraint(ALLOC_IN_RC(ptr_reg));
5095 match(RegP);
5096 match(iRegPNoSp);
5097 match(iRegP_R0);
5098 //match(iRegP_R2);
5099 //match(iRegP_R4);
5100 match(iRegP_R5);
5101 match(thread_RegP);
5102 op_cost(0);
5103 format %{ %}
5104 interface(REG_INTER);
5105 %}
5106
5107 // Pointer 64 bit Register not Special
5108 operand iRegPNoSp()
5109 %{
5110 constraint(ALLOC_IN_RC(no_special_ptr_reg));
5111 match(RegP);
5112 // match(iRegP);
5113 // match(iRegP_R0);
5114 // match(iRegP_R2);
5115 // match(iRegP_R4);
5116 // match(iRegP_R5);
5117 // match(thread_RegP);
5118 op_cost(0);
5119 format %{ %}
5120 interface(REG_INTER);
5121 %}
5122
5123 // Pointer 64 bit Register R0 only
5124 operand iRegP_R0()
5125 %{
5126 constraint(ALLOC_IN_RC(r0_reg));
5127 match(RegP);
5128 // match(iRegP);
5129 match(iRegPNoSp);
5130 op_cost(0);
5131 format %{ %}
5132 interface(REG_INTER);
5133 %}
5134
5135 // Pointer 64 bit Register R1 only
5136 operand iRegP_R1()
5137 %{
5138 constraint(ALLOC_IN_RC(r1_reg));
5139 match(RegP);
5140 // match(iRegP);
5141 match(iRegPNoSp);
5142 op_cost(0);
5143 format %{ %}
5144 interface(REG_INTER);
5145 %}
5146
5147 // Pointer 64 bit Register R2 only
5148 operand iRegP_R2()
5149 %{
5150 constraint(ALLOC_IN_RC(r2_reg));
5151 match(RegP);
5152 // match(iRegP);
5153 match(iRegPNoSp);
5154 op_cost(0);
5155 format %{ %}
5156 interface(REG_INTER);
5157 %}
5158
5159 // Pointer 64 bit Register R3 only
5160 operand iRegP_R3()
5161 %{
5162 constraint(ALLOC_IN_RC(r3_reg));
5163 match(RegP);
5164 // match(iRegP);
5165 match(iRegPNoSp);
5166 op_cost(0);
5167 format %{ %}
5168 interface(REG_INTER);
5169 %}
5170
5171 // Pointer 64 bit Register R4 only
5172 operand iRegP_R4()
5173 %{
5174 constraint(ALLOC_IN_RC(r4_reg));
5175 match(RegP);
5176 // match(iRegP);
5177 match(iRegPNoSp);
5178 op_cost(0);
5179 format %{ %}
5180 interface(REG_INTER);
5181 %}
5182
5183 // Pointer 64 bit Register R5 only
5184 operand iRegP_R5()
5185 %{
5186 constraint(ALLOC_IN_RC(r5_reg));
5187 match(RegP);
5188 // match(iRegP);
5189 match(iRegPNoSp);
5190 op_cost(0);
5191 format %{ %}
5192 interface(REG_INTER);
5193 %}
5194
5195 // Pointer 64 bit Register R10 only
5196 operand iRegP_R10()
5197 %{
5198 constraint(ALLOC_IN_RC(r10_reg));
5199 match(RegP);
5200 // match(iRegP);
5201 match(iRegPNoSp);
5202 op_cost(0);
5203 format %{ %}
5204 interface(REG_INTER);
5205 %}
5206
5207 // Long 64 bit Register R0 only
5208 operand iRegL_R0()
5209 %{
5210 constraint(ALLOC_IN_RC(r0_reg));
5211 match(RegL);
5212 match(iRegLNoSp);
5213 op_cost(0);
5214 format %{ %}
5215 interface(REG_INTER);
5216 %}
5217
5218 // Long 64 bit Register R2 only
5219 operand iRegL_R2()
5220 %{
5221 constraint(ALLOC_IN_RC(r2_reg));
5222 match(RegL);
5223 match(iRegLNoSp);
5224 op_cost(0);
5225 format %{ %}
5226 interface(REG_INTER);
5227 %}
5228
5229 // Long 64 bit Register R3 only
5230 operand iRegL_R3()
5231 %{
5232 constraint(ALLOC_IN_RC(r3_reg));
5233 match(RegL);
5234 match(iRegLNoSp);
5235 op_cost(0);
5236 format %{ %}
5237 interface(REG_INTER);
5238 %}
5239
5240 // Long 64 bit Register R11 only
5241 operand iRegL_R11()
5242 %{
5243 constraint(ALLOC_IN_RC(r11_reg));
5244 match(RegL);
5245 match(iRegLNoSp);
5246 op_cost(0);
5247 format %{ %}
5248 interface(REG_INTER);
5249 %}
5250
5251 // Pointer 64 bit Register FP only
5252 operand iRegP_FP()
5253 %{
5254 constraint(ALLOC_IN_RC(fp_reg));
5255 match(RegP);
5256 // match(iRegP);
5257 op_cost(0);
5258 format %{ %}
5259 interface(REG_INTER);
5260 %}
5261
5262 // Register R0 only
5263 operand iRegI_R0()
5264 %{
5265 constraint(ALLOC_IN_RC(int_r0_reg));
5266 match(RegI);
5267 match(iRegINoSp);
5268 op_cost(0);
5269 format %{ %}
5270 interface(REG_INTER);
5271 %}
5272
5273 // Register R2 only
5274 operand iRegI_R2()
5275 %{
5276 constraint(ALLOC_IN_RC(int_r2_reg));
5277 match(RegI);
5278 match(iRegINoSp);
5279 op_cost(0);
5280 format %{ %}
5281 interface(REG_INTER);
5282 %}
5283
5284 // Register R3 only
5285 operand iRegI_R3()
5286 %{
5287 constraint(ALLOC_IN_RC(int_r3_reg));
5288 match(RegI);
5289 match(iRegINoSp);
5290 op_cost(0);
5291 format %{ %}
5292 interface(REG_INTER);
5293 %}
5294
5295
5296 // Register R4 only
5297 operand iRegI_R4()
5298 %{
5299 constraint(ALLOC_IN_RC(int_r4_reg));
5300 match(RegI);
5301 match(iRegINoSp);
5302 op_cost(0);
5303 format %{ %}
5304 interface(REG_INTER);
5305 %}
5306
5307
5308 // Pointer Register Operands
5309 // Narrow Pointer Register
5310 operand iRegN()
5311 %{
5312 constraint(ALLOC_IN_RC(any_reg32));
5313 match(RegN);
5314 match(iRegNNoSp);
5315 op_cost(0);
5316 format %{ %}
5317 interface(REG_INTER);
5318 %}
5319
5320 operand iRegN_R0()
5321 %{
5322 constraint(ALLOC_IN_RC(r0_reg));
5323 match(iRegN);
5324 op_cost(0);
5325 format %{ %}
5326 interface(REG_INTER);
5327 %}
5328
5329 operand iRegN_R2()
5330 %{
5331 constraint(ALLOC_IN_RC(r2_reg));
5332 match(iRegN);
5333 op_cost(0);
5334 format %{ %}
5335 interface(REG_INTER);
5336 %}
5337
5338 operand iRegN_R3()
5339 %{
5340 constraint(ALLOC_IN_RC(r3_reg));
5341 match(iRegN);
5342 op_cost(0);
5343 format %{ %}
5344 interface(REG_INTER);
5345 %}
5346
5347 // Integer 64 bit Register not Special
5348 operand iRegNNoSp()
5349 %{
5350 constraint(ALLOC_IN_RC(no_special_reg32));
5351 match(RegN);
5352 op_cost(0);
5353 format %{ %}
5354 interface(REG_INTER);
5355 %}
5356
5357 // Float Register
5358 // Float register operands
5359 operand vRegF()
5360 %{
5361 constraint(ALLOC_IN_RC(float_reg));
5362 match(RegF);
5363
5364 op_cost(0);
5365 format %{ %}
5366 interface(REG_INTER);
5367 %}
5368
5369 // Double Register
5370 // Double register operands
5371 operand vRegD()
5372 %{
5373 constraint(ALLOC_IN_RC(double_reg));
5374 match(RegD);
5375
5376 op_cost(0);
5377 format %{ %}
5378 interface(REG_INTER);
5379 %}
5380
5381 // Generic vector class. This will be used for
5382 // all vector operands, including NEON and SVE.
5383 operand vReg()
5384 %{
5385 constraint(ALLOC_IN_RC(dynamic));
5386 match(VecA);
5387 match(VecD);
5388 match(VecX);
5389
5390 op_cost(0);
5391 format %{ %}
5392 interface(REG_INTER);
5393 %}
5394
5395 operand vecA()
5396 %{
5397 constraint(ALLOC_IN_RC(vectora_reg));
5398 match(VecA);
5399
5400 op_cost(0);
5401 format %{ %}
5402 interface(REG_INTER);
5403 %}
5404
5405 operand vecD()
5406 %{
5407 constraint(ALLOC_IN_RC(vectord_reg));
5408 match(VecD);
5409
5410 op_cost(0);
5411 format %{ %}
5412 interface(REG_INTER);
5413 %}
5414
5415 operand vecX()
5416 %{
5417 constraint(ALLOC_IN_RC(vectorx_reg));
5418 match(VecX);
5419
5420 op_cost(0);
5421 format %{ %}
5422 interface(REG_INTER);
5423 %}
5424
5425 operand vRegD_V0()
5426 %{
5427 constraint(ALLOC_IN_RC(v0_reg));
5428 match(RegD);
5429 op_cost(0);
5430 format %{ %}
5431 interface(REG_INTER);
5432 %}
5433
5434 operand vRegD_V1()
5435 %{
5436 constraint(ALLOC_IN_RC(v1_reg));
5437 match(RegD);
5438 op_cost(0);
5439 format %{ %}
5440 interface(REG_INTER);
5441 %}
5442
5443 operand vRegD_V2()
5444 %{
5445 constraint(ALLOC_IN_RC(v2_reg));
5446 match(RegD);
5447 op_cost(0);
5448 format %{ %}
5449 interface(REG_INTER);
5450 %}
5451
5452 operand vRegD_V3()
5453 %{
5454 constraint(ALLOC_IN_RC(v3_reg));
5455 match(RegD);
5456 op_cost(0);
5457 format %{ %}
5458 interface(REG_INTER);
5459 %}
5460
5461 operand vRegD_V4()
5462 %{
5463 constraint(ALLOC_IN_RC(v4_reg));
5464 match(RegD);
5465 op_cost(0);
5466 format %{ %}
5467 interface(REG_INTER);
5468 %}
5469
5470 operand vRegD_V5()
5471 %{
5472 constraint(ALLOC_IN_RC(v5_reg));
5473 match(RegD);
5474 op_cost(0);
5475 format %{ %}
5476 interface(REG_INTER);
5477 %}
5478
5479 operand vRegD_V6()
5480 %{
5481 constraint(ALLOC_IN_RC(v6_reg));
5482 match(RegD);
5483 op_cost(0);
5484 format %{ %}
5485 interface(REG_INTER);
5486 %}
5487
5488 operand vRegD_V7()
5489 %{
5490 constraint(ALLOC_IN_RC(v7_reg));
5491 match(RegD);
5492 op_cost(0);
5493 format %{ %}
5494 interface(REG_INTER);
5495 %}
5496
5497 operand vRegD_V8()
5498 %{
5499 constraint(ALLOC_IN_RC(v8_reg));
5500 match(RegD);
5501 op_cost(0);
5502 format %{ %}
5503 interface(REG_INTER);
5504 %}
5505
5506 operand vRegD_V9()
5507 %{
5508 constraint(ALLOC_IN_RC(v9_reg));
5509 match(RegD);
5510 op_cost(0);
5511 format %{ %}
5512 interface(REG_INTER);
5513 %}
5514
5515 operand vRegD_V10()
5516 %{
5517 constraint(ALLOC_IN_RC(v10_reg));
5518 match(RegD);
5519 op_cost(0);
5520 format %{ %}
5521 interface(REG_INTER);
5522 %}
5523
5524 operand vRegD_V11()
5525 %{
5526 constraint(ALLOC_IN_RC(v11_reg));
5527 match(RegD);
5528 op_cost(0);
5529 format %{ %}
5530 interface(REG_INTER);
5531 %}
5532
5533 operand vRegD_V12()
5534 %{
5535 constraint(ALLOC_IN_RC(v12_reg));
5536 match(RegD);
5537 op_cost(0);
5538 format %{ %}
5539 interface(REG_INTER);
5540 %}
5541
5542 operand vRegD_V13()
5543 %{
5544 constraint(ALLOC_IN_RC(v13_reg));
5545 match(RegD);
5546 op_cost(0);
5547 format %{ %}
5548 interface(REG_INTER);
5549 %}
5550
5551 operand vRegD_V14()
5552 %{
5553 constraint(ALLOC_IN_RC(v14_reg));
5554 match(RegD);
5555 op_cost(0);
5556 format %{ %}
5557 interface(REG_INTER);
5558 %}
5559
5560 operand vRegD_V15()
5561 %{
5562 constraint(ALLOC_IN_RC(v15_reg));
5563 match(RegD);
5564 op_cost(0);
5565 format %{ %}
5566 interface(REG_INTER);
5567 %}
5568
5569 operand vRegD_V16()
5570 %{
5571 constraint(ALLOC_IN_RC(v16_reg));
5572 match(RegD);
5573 op_cost(0);
5574 format %{ %}
5575 interface(REG_INTER);
5576 %}
5577
5578 operand vRegD_V17()
5579 %{
5580 constraint(ALLOC_IN_RC(v17_reg));
5581 match(RegD);
5582 op_cost(0);
5583 format %{ %}
5584 interface(REG_INTER);
5585 %}
5586
5587 operand vRegD_V18()
5588 %{
5589 constraint(ALLOC_IN_RC(v18_reg));
5590 match(RegD);
5591 op_cost(0);
5592 format %{ %}
5593 interface(REG_INTER);
5594 %}
5595
5596 operand vRegD_V19()
5597 %{
5598 constraint(ALLOC_IN_RC(v19_reg));
5599 match(RegD);
5600 op_cost(0);
5601 format %{ %}
5602 interface(REG_INTER);
5603 %}
5604
5605 operand vRegD_V20()
5606 %{
5607 constraint(ALLOC_IN_RC(v20_reg));
5608 match(RegD);
5609 op_cost(0);
5610 format %{ %}
5611 interface(REG_INTER);
5612 %}
5613
5614 operand vRegD_V21()
5615 %{
5616 constraint(ALLOC_IN_RC(v21_reg));
5617 match(RegD);
5618 op_cost(0);
5619 format %{ %}
5620 interface(REG_INTER);
5621 %}
5622
5623 operand vRegD_V22()
5624 %{
5625 constraint(ALLOC_IN_RC(v22_reg));
5626 match(RegD);
5627 op_cost(0);
5628 format %{ %}
5629 interface(REG_INTER);
5630 %}
5631
5632 operand vRegD_V23()
5633 %{
5634 constraint(ALLOC_IN_RC(v23_reg));
5635 match(RegD);
5636 op_cost(0);
5637 format %{ %}
5638 interface(REG_INTER);
5639 %}
5640
5641 operand vRegD_V24()
5642 %{
5643 constraint(ALLOC_IN_RC(v24_reg));
5644 match(RegD);
5645 op_cost(0);
5646 format %{ %}
5647 interface(REG_INTER);
5648 %}
5649
5650 operand vRegD_V25()
5651 %{
5652 constraint(ALLOC_IN_RC(v25_reg));
5653 match(RegD);
5654 op_cost(0);
5655 format %{ %}
5656 interface(REG_INTER);
5657 %}
5658
5659 operand vRegD_V26()
5660 %{
5661 constraint(ALLOC_IN_RC(v26_reg));
5662 match(RegD);
5663 op_cost(0);
5664 format %{ %}
5665 interface(REG_INTER);
5666 %}
5667
5668 operand vRegD_V27()
5669 %{
5670 constraint(ALLOC_IN_RC(v27_reg));
5671 match(RegD);
5672 op_cost(0);
5673 format %{ %}
5674 interface(REG_INTER);
5675 %}
5676
5677 operand vRegD_V28()
5678 %{
5679 constraint(ALLOC_IN_RC(v28_reg));
5680 match(RegD);
5681 op_cost(0);
5682 format %{ %}
5683 interface(REG_INTER);
5684 %}
5685
5686 operand vRegD_V29()
5687 %{
5688 constraint(ALLOC_IN_RC(v29_reg));
5689 match(RegD);
5690 op_cost(0);
5691 format %{ %}
5692 interface(REG_INTER);
5693 %}
5694
5695 operand vRegD_V30()
5696 %{
5697 constraint(ALLOC_IN_RC(v30_reg));
5698 match(RegD);
5699 op_cost(0);
5700 format %{ %}
5701 interface(REG_INTER);
5702 %}
5703
5704 operand vRegD_V31()
5705 %{
5706 constraint(ALLOC_IN_RC(v31_reg));
5707 match(RegD);
5708 op_cost(0);
5709 format %{ %}
5710 interface(REG_INTER);
5711 %}
5712
5713 operand pReg()
5714 %{
5715 constraint(ALLOC_IN_RC(pr_reg));
5716 match(RegVectMask);
5717 match(pRegGov);
5718 op_cost(0);
5719 format %{ %}
5720 interface(REG_INTER);
5721 %}
5722
5723 operand pRegGov()
5724 %{
5725 constraint(ALLOC_IN_RC(gov_pr));
5726 match(RegVectMask);
5727 match(pReg);
5728 op_cost(0);
5729 format %{ %}
5730 interface(REG_INTER);
5731 %}
5732
5733 operand pRegGov_P0()
5734 %{
5735 constraint(ALLOC_IN_RC(p0_reg));
5736 match(RegVectMask);
5737 op_cost(0);
5738 format %{ %}
5739 interface(REG_INTER);
5740 %}
5741
5742 operand pRegGov_P1()
5743 %{
5744 constraint(ALLOC_IN_RC(p1_reg));
5745 match(RegVectMask);
5746 op_cost(0);
5747 format %{ %}
5748 interface(REG_INTER);
5749 %}
5750
5751 // Flags register, used as output of signed compare instructions
5752
5753 // note that on AArch64 we also use this register as the output for
5754 // for floating point compare instructions (CmpF CmpD). this ensures
5755 // that ordered inequality tests use GT, GE, LT or LE none of which
5756 // pass through cases where the result is unordered i.e. one or both
5757 // inputs to the compare is a NaN. this means that the ideal code can
5758 // replace e.g. a GT with an LE and not end up capturing the NaN case
5759 // (where the comparison should always fail). EQ and NE tests are
5760 // always generated in ideal code so that unordered folds into the NE
5761 // case, matching the behaviour of AArch64 NE.
5762 //
5763 // This differs from x86 where the outputs of FP compares use a
5764 // special FP flags registers and where compares based on this
5765 // register are distinguished into ordered inequalities (cmpOpUCF) and
5766 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5767 // to explicitly handle the unordered case in branches. x86 also has
5768 // to include extra CMoveX rules to accept a cmpOpUCF input.
5769
5770 operand rFlagsReg()
5771 %{
5772 constraint(ALLOC_IN_RC(int_flags));
5773 match(RegFlags);
5774
5775 op_cost(0);
5776 format %{ "RFLAGS" %}
5777 interface(REG_INTER);
5778 %}
5779
5780 // Flags register, used as output of unsigned compare instructions
5781 operand rFlagsRegU()
5782 %{
5783 constraint(ALLOC_IN_RC(int_flags));
5784 match(RegFlags);
5785
5786 op_cost(0);
5787 format %{ "RFLAGSU" %}
5788 interface(REG_INTER);
5789 %}
5790
5791 // Special Registers
5792
5793 // Method Register
5794 operand inline_cache_RegP(iRegP reg)
5795 %{
5796 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5797 match(reg);
5798 match(iRegPNoSp);
5799 op_cost(0);
5800 format %{ %}
5801 interface(REG_INTER);
5802 %}
5803
5804 // Thread Register
5805 operand thread_RegP(iRegP reg)
5806 %{
5807 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5808 match(reg);
5809 op_cost(0);
5810 format %{ %}
5811 interface(REG_INTER);
5812 %}
5813
5814 operand lr_RegP(iRegP reg)
5815 %{
5816 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5817 match(reg);
5818 op_cost(0);
5819 format %{ %}
5820 interface(REG_INTER);
5821 %}
5822
5823 //----------Memory Operands----------------------------------------------------
5824
5825 operand indirect(iRegP reg)
5826 %{
5827 constraint(ALLOC_IN_RC(ptr_reg));
5828 match(reg);
5829 op_cost(0);
5830 format %{ "[$reg]" %}
5831 interface(MEMORY_INTER) %{
5832 base($reg);
5833 index(0xffffffff);
5834 scale(0x0);
5835 disp(0x0);
5836 %}
5837 %}
5838
5839 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5840 %{
5841 constraint(ALLOC_IN_RC(ptr_reg));
5842 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5843 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5844 op_cost(0);
5845 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5846 interface(MEMORY_INTER) %{
5847 base($reg);
5848 index($ireg);
5849 scale($scale);
5850 disp(0x0);
5851 %}
5852 %}
5853
5854 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5855 %{
5856 constraint(ALLOC_IN_RC(ptr_reg));
5857 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5858 match(AddP reg (LShiftL lreg scale));
5859 op_cost(0);
5860 format %{ "$reg, $lreg lsl($scale)" %}
5861 interface(MEMORY_INTER) %{
5862 base($reg);
5863 index($lreg);
5864 scale($scale);
5865 disp(0x0);
5866 %}
5867 %}
5868
5869 operand indIndexI2L(iRegP reg, iRegI ireg)
5870 %{
5871 constraint(ALLOC_IN_RC(ptr_reg));
5872 match(AddP reg (ConvI2L ireg));
5873 op_cost(0);
5874 format %{ "$reg, $ireg, 0, I2L" %}
5875 interface(MEMORY_INTER) %{
5876 base($reg);
5877 index($ireg);
5878 scale(0x0);
5879 disp(0x0);
5880 %}
5881 %}
5882
5883 operand indIndex(iRegP reg, iRegL lreg)
5884 %{
5885 constraint(ALLOC_IN_RC(ptr_reg));
5886 match(AddP reg lreg);
5887 op_cost(0);
5888 format %{ "$reg, $lreg" %}
5889 interface(MEMORY_INTER) %{
5890 base($reg);
5891 index($lreg);
5892 scale(0x0);
5893 disp(0x0);
5894 %}
5895 %}
5896
5897 operand indOffI(iRegP reg, immIOffset off)
5898 %{
5899 constraint(ALLOC_IN_RC(ptr_reg));
5900 match(AddP reg off);
5901 op_cost(0);
5902 format %{ "[$reg, $off]" %}
5903 interface(MEMORY_INTER) %{
5904 base($reg);
5905 index(0xffffffff);
5906 scale(0x0);
5907 disp($off);
5908 %}
5909 %}
5910
5911 operand indOffI1(iRegP reg, immIOffset1 off)
5912 %{
5913 constraint(ALLOC_IN_RC(ptr_reg));
5914 match(AddP reg off);
5915 op_cost(0);
5916 format %{ "[$reg, $off]" %}
5917 interface(MEMORY_INTER) %{
5918 base($reg);
5919 index(0xffffffff);
5920 scale(0x0);
5921 disp($off);
5922 %}
5923 %}
5924
5925 operand indOffI2(iRegP reg, immIOffset2 off)
5926 %{
5927 constraint(ALLOC_IN_RC(ptr_reg));
5928 match(AddP reg off);
5929 op_cost(0);
5930 format %{ "[$reg, $off]" %}
5931 interface(MEMORY_INTER) %{
5932 base($reg);
5933 index(0xffffffff);
5934 scale(0x0);
5935 disp($off);
5936 %}
5937 %}
5938
5939 operand indOffI4(iRegP reg, immIOffset4 off)
5940 %{
5941 constraint(ALLOC_IN_RC(ptr_reg));
5942 match(AddP reg off);
5943 op_cost(0);
5944 format %{ "[$reg, $off]" %}
5945 interface(MEMORY_INTER) %{
5946 base($reg);
5947 index(0xffffffff);
5948 scale(0x0);
5949 disp($off);
5950 %}
5951 %}
5952
5953 operand indOffI8(iRegP reg, immIOffset8 off)
5954 %{
5955 constraint(ALLOC_IN_RC(ptr_reg));
5956 match(AddP reg off);
5957 op_cost(0);
5958 format %{ "[$reg, $off]" %}
5959 interface(MEMORY_INTER) %{
5960 base($reg);
5961 index(0xffffffff);
5962 scale(0x0);
5963 disp($off);
5964 %}
5965 %}
5966
5967 operand indOffI16(iRegP reg, immIOffset16 off)
5968 %{
5969 constraint(ALLOC_IN_RC(ptr_reg));
5970 match(AddP reg off);
5971 op_cost(0);
5972 format %{ "[$reg, $off]" %}
5973 interface(MEMORY_INTER) %{
5974 base($reg);
5975 index(0xffffffff);
5976 scale(0x0);
5977 disp($off);
5978 %}
5979 %}
5980
5981 operand indOffL(iRegP reg, immLoffset off)
5982 %{
5983 constraint(ALLOC_IN_RC(ptr_reg));
5984 match(AddP reg off);
5985 op_cost(0);
5986 format %{ "[$reg, $off]" %}
5987 interface(MEMORY_INTER) %{
5988 base($reg);
5989 index(0xffffffff);
5990 scale(0x0);
5991 disp($off);
5992 %}
5993 %}
5994
5995 operand indOffL1(iRegP reg, immLoffset1 off)
5996 %{
5997 constraint(ALLOC_IN_RC(ptr_reg));
5998 match(AddP reg off);
5999 op_cost(0);
6000 format %{ "[$reg, $off]" %}
6001 interface(MEMORY_INTER) %{
6002 base($reg);
6003 index(0xffffffff);
6004 scale(0x0);
6005 disp($off);
6006 %}
6007 %}
6008
6009 operand indOffL2(iRegP reg, immLoffset2 off)
6010 %{
6011 constraint(ALLOC_IN_RC(ptr_reg));
6012 match(AddP reg off);
6013 op_cost(0);
6014 format %{ "[$reg, $off]" %}
6015 interface(MEMORY_INTER) %{
6016 base($reg);
6017 index(0xffffffff);
6018 scale(0x0);
6019 disp($off);
6020 %}
6021 %}
6022
6023 operand indOffL4(iRegP reg, immLoffset4 off)
6024 %{
6025 constraint(ALLOC_IN_RC(ptr_reg));
6026 match(AddP reg off);
6027 op_cost(0);
6028 format %{ "[$reg, $off]" %}
6029 interface(MEMORY_INTER) %{
6030 base($reg);
6031 index(0xffffffff);
6032 scale(0x0);
6033 disp($off);
6034 %}
6035 %}
6036
6037 operand indOffL8(iRegP reg, immLoffset8 off)
6038 %{
6039 constraint(ALLOC_IN_RC(ptr_reg));
6040 match(AddP reg off);
6041 op_cost(0);
6042 format %{ "[$reg, $off]" %}
6043 interface(MEMORY_INTER) %{
6044 base($reg);
6045 index(0xffffffff);
6046 scale(0x0);
6047 disp($off);
6048 %}
6049 %}
6050
6051 operand indOffL16(iRegP reg, immLoffset16 off)
6052 %{
6053 constraint(ALLOC_IN_RC(ptr_reg));
6054 match(AddP reg off);
6055 op_cost(0);
6056 format %{ "[$reg, $off]" %}
6057 interface(MEMORY_INTER) %{
6058 base($reg);
6059 index(0xffffffff);
6060 scale(0x0);
6061 disp($off);
6062 %}
6063 %}
6064
6065 operand indirectN(iRegN reg)
6066 %{
6067 predicate(CompressedOops::shift() == 0);
6068 constraint(ALLOC_IN_RC(ptr_reg));
6069 match(DecodeN reg);
6070 op_cost(0);
6071 format %{ "[$reg]\t# narrow" %}
6072 interface(MEMORY_INTER) %{
6073 base($reg);
6074 index(0xffffffff);
6075 scale(0x0);
6076 disp(0x0);
6077 %}
6078 %}
6079
6080 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
6081 %{
6082 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
6083 constraint(ALLOC_IN_RC(ptr_reg));
6084 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
6085 op_cost(0);
6086 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
6087 interface(MEMORY_INTER) %{
6088 base($reg);
6089 index($ireg);
6090 scale($scale);
6091 disp(0x0);
6092 %}
6093 %}
6094
6095 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
6096 %{
6097 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
6098 constraint(ALLOC_IN_RC(ptr_reg));
6099 match(AddP (DecodeN reg) (LShiftL lreg scale));
6100 op_cost(0);
6101 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
6102 interface(MEMORY_INTER) %{
6103 base($reg);
6104 index($lreg);
6105 scale($scale);
6106 disp(0x0);
6107 %}
6108 %}
6109
6110 operand indIndexI2LN(iRegN reg, iRegI ireg)
6111 %{
6112 predicate(CompressedOops::shift() == 0);
6113 constraint(ALLOC_IN_RC(ptr_reg));
6114 match(AddP (DecodeN reg) (ConvI2L ireg));
6115 op_cost(0);
6116 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
6117 interface(MEMORY_INTER) %{
6118 base($reg);
6119 index($ireg);
6120 scale(0x0);
6121 disp(0x0);
6122 %}
6123 %}
6124
6125 operand indIndexN(iRegN reg, iRegL lreg)
6126 %{
6127 predicate(CompressedOops::shift() == 0);
6128 constraint(ALLOC_IN_RC(ptr_reg));
6129 match(AddP (DecodeN reg) lreg);
6130 op_cost(0);
6131 format %{ "$reg, $lreg\t# narrow" %}
6132 interface(MEMORY_INTER) %{
6133 base($reg);
6134 index($lreg);
6135 scale(0x0);
6136 disp(0x0);
6137 %}
6138 %}
6139
6140 operand indOffIN(iRegN reg, immIOffset off)
6141 %{
6142 predicate(CompressedOops::shift() == 0);
6143 constraint(ALLOC_IN_RC(ptr_reg));
6144 match(AddP (DecodeN reg) off);
6145 op_cost(0);
6146 format %{ "[$reg, $off]\t# narrow" %}
6147 interface(MEMORY_INTER) %{
6148 base($reg);
6149 index(0xffffffff);
6150 scale(0x0);
6151 disp($off);
6152 %}
6153 %}
6154
6155 operand indOffLN(iRegN reg, immLoffset off)
6156 %{
6157 predicate(CompressedOops::shift() == 0);
6158 constraint(ALLOC_IN_RC(ptr_reg));
6159 match(AddP (DecodeN reg) off);
6160 op_cost(0);
6161 format %{ "[$reg, $off]\t# narrow" %}
6162 interface(MEMORY_INTER) %{
6163 base($reg);
6164 index(0xffffffff);
6165 scale(0x0);
6166 disp($off);
6167 %}
6168 %}
6169
6170
6171
6172 // AArch64 opto stubs need to write to the pc slot in the thread anchor
6173 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
6174 %{
6175 constraint(ALLOC_IN_RC(ptr_reg));
6176 match(AddP reg off);
6177 op_cost(0);
6178 format %{ "[$reg, $off]" %}
6179 interface(MEMORY_INTER) %{
6180 base($reg);
6181 index(0xffffffff);
6182 scale(0x0);
6183 disp($off);
6184 %}
6185 %}
6186
6187 //----------Special Memory Operands--------------------------------------------
6188 // Stack Slot Operand - This operand is used for loading and storing temporary
6189 // values on the stack where a match requires a value to
6190 // flow through memory.
6191 operand stackSlotP(sRegP reg)
6192 %{
6193 constraint(ALLOC_IN_RC(stack_slots));
6194 op_cost(100);
6195 // No match rule because this operand is only generated in matching
6196 // match(RegP);
6197 format %{ "[$reg]" %}
6198 interface(MEMORY_INTER) %{
6199 base(0x1e); // RSP
6200 index(0x0); // No Index
6201 scale(0x0); // No Scale
6202 disp($reg); // Stack Offset
6203 %}
6204 %}
6205
6206 operand stackSlotI(sRegI reg)
6207 %{
6208 constraint(ALLOC_IN_RC(stack_slots));
6209 // No match rule because this operand is only generated in matching
6210 // match(RegI);
6211 format %{ "[$reg]" %}
6212 interface(MEMORY_INTER) %{
6213 base(0x1e); // RSP
6214 index(0x0); // No Index
6215 scale(0x0); // No Scale
6216 disp($reg); // Stack Offset
6217 %}
6218 %}
6219
6220 operand stackSlotF(sRegF reg)
6221 %{
6222 constraint(ALLOC_IN_RC(stack_slots));
6223 // No match rule because this operand is only generated in matching
6224 // match(RegF);
6225 format %{ "[$reg]" %}
6226 interface(MEMORY_INTER) %{
6227 base(0x1e); // RSP
6228 index(0x0); // No Index
6229 scale(0x0); // No Scale
6230 disp($reg); // Stack Offset
6231 %}
6232 %}
6233
6234 operand stackSlotD(sRegD reg)
6235 %{
6236 constraint(ALLOC_IN_RC(stack_slots));
6237 // No match rule because this operand is only generated in matching
6238 // match(RegD);
6239 format %{ "[$reg]" %}
6240 interface(MEMORY_INTER) %{
6241 base(0x1e); // RSP
6242 index(0x0); // No Index
6243 scale(0x0); // No Scale
6244 disp($reg); // Stack Offset
6245 %}
6246 %}
6247
6248 operand stackSlotL(sRegL reg)
6249 %{
6250 constraint(ALLOC_IN_RC(stack_slots));
6251 // No match rule because this operand is only generated in matching
6252 // match(RegL);
6253 format %{ "[$reg]" %}
6254 interface(MEMORY_INTER) %{
6255 base(0x1e); // RSP
6256 index(0x0); // No Index
6257 scale(0x0); // No Scale
6258 disp($reg); // Stack Offset
6259 %}
6260 %}
6261
6262 // Operands for expressing Control Flow
6263 // NOTE: Label is a predefined operand which should not be redefined in
6264 // the AD file. It is generically handled within the ADLC.
6265
6266 //----------Conditional Branch Operands----------------------------------------
6267 // Comparison Op - This is the operation of the comparison, and is limited to
6268 // the following set of codes:
6269 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6270 //
6271 // Other attributes of the comparison, such as unsignedness, are specified
6272 // by the comparison instruction that sets a condition code flags register.
6273 // That result is represented by a flags operand whose subtype is appropriate
6274 // to the unsignedness (etc.) of the comparison.
6275 //
6276 // Later, the instruction which matches both the Comparison Op (a Bool) and
6277 // the flags (produced by the Cmp) specifies the coding of the comparison op
6278 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6279
6280 // used for signed integral comparisons and fp comparisons
6281
6282 operand cmpOp()
6283 %{
6284 match(Bool);
6285
6286 format %{ "" %}
6287 interface(COND_INTER) %{
6288 equal(0x0, "eq");
6289 not_equal(0x1, "ne");
6290 less(0xb, "lt");
6291 greater_equal(0xa, "ge");
6292 less_equal(0xd, "le");
6293 greater(0xc, "gt");
6294 overflow(0x6, "vs");
6295 no_overflow(0x7, "vc");
6296 %}
6297 %}
6298
6299 // used for unsigned integral comparisons
6300
6301 operand cmpOpU()
6302 %{
6303 match(Bool);
6304
6305 format %{ "" %}
6306 interface(COND_INTER) %{
6307 equal(0x0, "eq");
6308 not_equal(0x1, "ne");
6309 less(0x3, "lo");
6310 greater_equal(0x2, "hs");
6311 less_equal(0x9, "ls");
6312 greater(0x8, "hi");
6313 overflow(0x6, "vs");
6314 no_overflow(0x7, "vc");
6315 %}
6316 %}
6317
6318 // used for certain integral comparisons which can be
6319 // converted to cbxx or tbxx instructions
6320
6321 operand cmpOpEqNe()
6322 %{
6323 match(Bool);
6324 op_cost(0);
6325 predicate(n->as_Bool()->_test._test == BoolTest::ne
6326 || n->as_Bool()->_test._test == BoolTest::eq);
6327
6328 format %{ "" %}
6329 interface(COND_INTER) %{
6330 equal(0x0, "eq");
6331 not_equal(0x1, "ne");
6332 less(0xb, "lt");
6333 greater_equal(0xa, "ge");
6334 less_equal(0xd, "le");
6335 greater(0xc, "gt");
6336 overflow(0x6, "vs");
6337 no_overflow(0x7, "vc");
6338 %}
6339 %}
6340
6341 // used for certain integral comparisons which can be
6342 // converted to cbxx or tbxx instructions
6343
6344 operand cmpOpLtGe()
6345 %{
6346 match(Bool);
6347 op_cost(0);
6348
6349 predicate(n->as_Bool()->_test._test == BoolTest::lt
6350 || n->as_Bool()->_test._test == BoolTest::ge);
6351
6352 format %{ "" %}
6353 interface(COND_INTER) %{
6354 equal(0x0, "eq");
6355 not_equal(0x1, "ne");
6356 less(0xb, "lt");
6357 greater_equal(0xa, "ge");
6358 less_equal(0xd, "le");
6359 greater(0xc, "gt");
6360 overflow(0x6, "vs");
6361 no_overflow(0x7, "vc");
6362 %}
6363 %}
6364
6365 // used for certain unsigned integral comparisons which can be
6366 // converted to cbxx or tbxx instructions
6367
6368 operand cmpOpUEqNeLtGe()
6369 %{
6370 match(Bool);
6371 op_cost(0);
6372
6373 predicate(n->as_Bool()->_test._test == BoolTest::eq
6374 || n->as_Bool()->_test._test == BoolTest::ne
6375 || n->as_Bool()->_test._test == BoolTest::lt
6376 || n->as_Bool()->_test._test == BoolTest::ge);
6377
6378 format %{ "" %}
6379 interface(COND_INTER) %{
6380 equal(0x0, "eq");
6381 not_equal(0x1, "ne");
6382 less(0xb, "lt");
6383 greater_equal(0xa, "ge");
6384 less_equal(0xd, "le");
6385 greater(0xc, "gt");
6386 overflow(0x6, "vs");
6387 no_overflow(0x7, "vc");
6388 %}
6389 %}
6390
6391 // Special operand allowing long args to int ops to be truncated for free
6392
6393 operand iRegL2I(iRegL reg) %{
6394
6395 op_cost(0);
6396
6397 match(ConvL2I reg);
6398
6399 format %{ "l2i($reg)" %}
6400
6401 interface(REG_INTER)
6402 %}
6403
6404 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
6405 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
6406 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
6407 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
6408
6409 //----------OPERAND CLASSES----------------------------------------------------
6410 // Operand Classes are groups of operands that are used as to simplify
6411 // instruction definitions by not requiring the AD writer to specify
6412 // separate instructions for every form of operand when the
6413 // instruction accepts multiple operand types with the same basic
6414 // encoding and format. The classic case of this is memory operands.
6415
6416 // memory is used to define read/write location for load/store
6417 // instruction defs. we can turn a memory op into an Address
6418
6419 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
6420 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6421
6422 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
6423 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6424
6425 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
6426 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6427
6428 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
6429 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6430
6431 // All of the memory operands. For the pipeline description.
6432 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
6433 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
6434 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6435
6436
6437 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
6438 // operations. it allows the src to be either an iRegI or a (ConvL2I
6439 // iRegL). in the latter case the l2i normally planted for a ConvL2I
6440 // can be elided because the 32-bit instruction will just employ the
6441 // lower 32 bits anyway.
6442 //
6443 // n.b. this does not elide all L2I conversions. if the truncated
6444 // value is consumed by more than one operation then the ConvL2I
6445 // cannot be bundled into the consuming nodes so an l2i gets planted
6446 // (actually a movw $dst $src) and the downstream instructions consume
6447 // the result of the l2i as an iRegI input. That's a shame since the
6448 // movw is actually redundant but its not too costly.
6449
6450 opclass iRegIorL2I(iRegI, iRegL2I);
6451
6452 //----------PIPELINE-----------------------------------------------------------
6453 // Rules which define the behavior of the target architectures pipeline.
6454
6455 // For specific pipelines, eg A53, define the stages of that pipeline
6456 //pipe_desc(ISS, EX1, EX2, WR);
6457 #define ISS S0
6458 #define EX1 S1
6459 #define EX2 S2
6460 #define WR S3
6461
6462 // Integer ALU reg operation
6463 pipeline %{
6464
6465 attributes %{
6466 // ARM instructions are of fixed length
6467 fixed_size_instructions; // Fixed size instructions TODO does
6468 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
6469 // ARM instructions come in 32-bit word units
6470 instruction_unit_size = 4; // An instruction is 4 bytes long
6471 instruction_fetch_unit_size = 64; // The processor fetches one line
6472 instruction_fetch_units = 1; // of 64 bytes
6473
6474 // List of nop instructions
6475 nops( MachNop );
6476 %}
6477
6478 // We don't use an actual pipeline model so don't care about resources
6479 // or description. we do use pipeline classes to introduce fixed
6480 // latencies
6481
6482 //----------RESOURCES----------------------------------------------------------
6483 // Resources are the functional units available to the machine
6484
6485 resources( INS0, INS1, INS01 = INS0 | INS1,
6486 ALU0, ALU1, ALU = ALU0 | ALU1,
6487 MAC,
6488 DIV,
6489 BRANCH,
6490 LDST,
6491 NEON_FP);
6492
6493 //----------PIPELINE DESCRIPTION-----------------------------------------------
6494 // Pipeline Description specifies the stages in the machine's pipeline
6495
6496 // Define the pipeline as a generic 6 stage pipeline
6497 pipe_desc(S0, S1, S2, S3, S4, S5);
6498
6499 //----------PIPELINE CLASSES---------------------------------------------------
6500 // Pipeline Classes describe the stages in which input and output are
6501 // referenced by the hardware pipeline.
6502
6503 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
6504 %{
6505 single_instruction;
6506 src1 : S1(read);
6507 src2 : S2(read);
6508 dst : S5(write);
6509 INS01 : ISS;
6510 NEON_FP : S5;
6511 %}
6512
6513 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6514 %{
6515 single_instruction;
6516 src1 : S1(read);
6517 src2 : S2(read);
6518 dst : S5(write);
6519 INS01 : ISS;
6520 NEON_FP : S5;
6521 %}
6522
6523 pipe_class fp_uop_s(vRegF dst, vRegF src)
6524 %{
6525 single_instruction;
6526 src : S1(read);
6527 dst : S5(write);
6528 INS01 : ISS;
6529 NEON_FP : S5;
6530 %}
6531
6532 pipe_class fp_uop_d(vRegD dst, vRegD src)
6533 %{
6534 single_instruction;
6535 src : S1(read);
6536 dst : S5(write);
6537 INS01 : ISS;
6538 NEON_FP : S5;
6539 %}
6540
6541 pipe_class fp_d2f(vRegF dst, vRegD src)
6542 %{
6543 single_instruction;
6544 src : S1(read);
6545 dst : S5(write);
6546 INS01 : ISS;
6547 NEON_FP : S5;
6548 %}
6549
6550 pipe_class fp_f2d(vRegD dst, vRegF src)
6551 %{
6552 single_instruction;
6553 src : S1(read);
6554 dst : S5(write);
6555 INS01 : ISS;
6556 NEON_FP : S5;
6557 %}
6558
6559 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6560 %{
6561 single_instruction;
6562 src : S1(read);
6563 dst : S5(write);
6564 INS01 : ISS;
6565 NEON_FP : S5;
6566 %}
6567
6568 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6569 %{
6570 single_instruction;
6571 src : S1(read);
6572 dst : S5(write);
6573 INS01 : ISS;
6574 NEON_FP : S5;
6575 %}
6576
6577 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6578 %{
6579 single_instruction;
6580 src : S1(read);
6581 dst : S5(write);
6582 INS01 : ISS;
6583 NEON_FP : S5;
6584 %}
6585
6586 pipe_class fp_l2f(vRegF dst, iRegL src)
6587 %{
6588 single_instruction;
6589 src : S1(read);
6590 dst : S5(write);
6591 INS01 : ISS;
6592 NEON_FP : S5;
6593 %}
6594
6595 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6596 %{
6597 single_instruction;
6598 src : S1(read);
6599 dst : S5(write);
6600 INS01 : ISS;
6601 NEON_FP : S5;
6602 %}
6603
6604 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6605 %{
6606 single_instruction;
6607 src : S1(read);
6608 dst : S5(write);
6609 INS01 : ISS;
6610 NEON_FP : S5;
6611 %}
6612
6613 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6614 %{
6615 single_instruction;
6616 src : S1(read);
6617 dst : S5(write);
6618 INS01 : ISS;
6619 NEON_FP : S5;
6620 %}
6621
6622 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6623 %{
6624 single_instruction;
6625 src : S1(read);
6626 dst : S5(write);
6627 INS01 : ISS;
6628 NEON_FP : S5;
6629 %}
6630
6631 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6632 %{
6633 single_instruction;
6634 src1 : S1(read);
6635 src2 : S2(read);
6636 dst : S5(write);
6637 INS0 : ISS;
6638 NEON_FP : S5;
6639 %}
6640
6641 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6642 %{
6643 single_instruction;
6644 src1 : S1(read);
6645 src2 : S2(read);
6646 dst : S5(write);
6647 INS0 : ISS;
6648 NEON_FP : S5;
6649 %}
6650
6651 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6652 %{
6653 single_instruction;
6654 cr : S1(read);
6655 src1 : S1(read);
6656 src2 : S1(read);
6657 dst : S3(write);
6658 INS01 : ISS;
6659 NEON_FP : S3;
6660 %}
6661
6662 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6663 %{
6664 single_instruction;
6665 cr : S1(read);
6666 src1 : S1(read);
6667 src2 : S1(read);
6668 dst : S3(write);
6669 INS01 : ISS;
6670 NEON_FP : S3;
6671 %}
6672
6673 pipe_class fp_imm_s(vRegF dst)
6674 %{
6675 single_instruction;
6676 dst : S3(write);
6677 INS01 : ISS;
6678 NEON_FP : S3;
6679 %}
6680
6681 pipe_class fp_imm_d(vRegD dst)
6682 %{
6683 single_instruction;
6684 dst : S3(write);
6685 INS01 : ISS;
6686 NEON_FP : S3;
6687 %}
6688
6689 pipe_class fp_load_constant_s(vRegF dst)
6690 %{
6691 single_instruction;
6692 dst : S4(write);
6693 INS01 : ISS;
6694 NEON_FP : S4;
6695 %}
6696
6697 pipe_class fp_load_constant_d(vRegD dst)
6698 %{
6699 single_instruction;
6700 dst : S4(write);
6701 INS01 : ISS;
6702 NEON_FP : S4;
6703 %}
6704
6705 //------- Integer ALU operations --------------------------
6706
6707 // Integer ALU reg-reg operation
6708 // Operands needed in EX1, result generated in EX2
6709 // Eg. ADD x0, x1, x2
6710 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6711 %{
6712 single_instruction;
6713 dst : EX2(write);
6714 src1 : EX1(read);
6715 src2 : EX1(read);
6716 INS01 : ISS; // Dual issue as instruction 0 or 1
6717 ALU : EX2;
6718 %}
6719
6720 // Integer ALU reg-reg operation with constant shift
6721 // Shifted register must be available in LATE_ISS instead of EX1
6722 // Eg. ADD x0, x1, x2, LSL #2
6723 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6724 %{
6725 single_instruction;
6726 dst : EX2(write);
6727 src1 : EX1(read);
6728 src2 : ISS(read);
6729 INS01 : ISS;
6730 ALU : EX2;
6731 %}
6732
6733 // Integer ALU reg operation with constant shift
6734 // Eg. LSL x0, x1, #shift
6735 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6736 %{
6737 single_instruction;
6738 dst : EX2(write);
6739 src1 : ISS(read);
6740 INS01 : ISS;
6741 ALU : EX2;
6742 %}
6743
6744 // Integer ALU reg-reg operation with variable shift
6745 // Both operands must be available in LATE_ISS instead of EX1
6746 // Result is available in EX1 instead of EX2
6747 // Eg. LSLV x0, x1, x2
6748 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6749 %{
6750 single_instruction;
6751 dst : EX1(write);
6752 src1 : ISS(read);
6753 src2 : ISS(read);
6754 INS01 : ISS;
6755 ALU : EX1;
6756 %}
6757
6758 // Integer ALU reg-reg operation with extract
6759 // As for _vshift above, but result generated in EX2
6760 // Eg. EXTR x0, x1, x2, #N
6761 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6762 %{
6763 single_instruction;
6764 dst : EX2(write);
6765 src1 : ISS(read);
6766 src2 : ISS(read);
6767 INS1 : ISS; // Can only dual issue as Instruction 1
6768 ALU : EX1;
6769 %}
6770
6771 // Integer ALU reg operation
6772 // Eg. NEG x0, x1
6773 pipe_class ialu_reg(iRegI dst, iRegI src)
6774 %{
6775 single_instruction;
6776 dst : EX2(write);
6777 src : EX1(read);
6778 INS01 : ISS;
6779 ALU : EX2;
6780 %}
6781
6782 // Integer ALU reg mmediate operation
6783 // Eg. ADD x0, x1, #N
6784 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6785 %{
6786 single_instruction;
6787 dst : EX2(write);
6788 src1 : EX1(read);
6789 INS01 : ISS;
6790 ALU : EX2;
6791 %}
6792
6793 // Integer ALU immediate operation (no source operands)
6794 // Eg. MOV x0, #N
6795 pipe_class ialu_imm(iRegI dst)
6796 %{
6797 single_instruction;
6798 dst : EX1(write);
6799 INS01 : ISS;
6800 ALU : EX1;
6801 %}
6802
6803 //------- Compare operation -------------------------------
6804
6805 // Compare reg-reg
6806 // Eg. CMP x0, x1
6807 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6808 %{
6809 single_instruction;
6810 // fixed_latency(16);
6811 cr : EX2(write);
6812 op1 : EX1(read);
6813 op2 : EX1(read);
6814 INS01 : ISS;
6815 ALU : EX2;
6816 %}
6817
6818 // Compare reg-reg
6819 // Eg. CMP x0, #N
6820 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6821 %{
6822 single_instruction;
6823 // fixed_latency(16);
6824 cr : EX2(write);
6825 op1 : EX1(read);
6826 INS01 : ISS;
6827 ALU : EX2;
6828 %}
6829
6830 //------- Conditional instructions ------------------------
6831
6832 // Conditional no operands
6833 // Eg. CSINC x0, zr, zr, <cond>
6834 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6835 %{
6836 single_instruction;
6837 cr : EX1(read);
6838 dst : EX2(write);
6839 INS01 : ISS;
6840 ALU : EX2;
6841 %}
6842
6843 // Conditional 2 operand
6844 // EG. CSEL X0, X1, X2, <cond>
6845 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6846 %{
6847 single_instruction;
6848 cr : EX1(read);
6849 src1 : EX1(read);
6850 src2 : EX1(read);
6851 dst : EX2(write);
6852 INS01 : ISS;
6853 ALU : EX2;
6854 %}
6855
6856 // Conditional 2 operand
6857 // EG. CSEL X0, X1, X2, <cond>
6858 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6859 %{
6860 single_instruction;
6861 cr : EX1(read);
6862 src : EX1(read);
6863 dst : EX2(write);
6864 INS01 : ISS;
6865 ALU : EX2;
6866 %}
6867
6868 //------- Multiply pipeline operations --------------------
6869
6870 // Multiply reg-reg
6871 // Eg. MUL w0, w1, w2
6872 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6873 %{
6874 single_instruction;
6875 dst : WR(write);
6876 src1 : ISS(read);
6877 src2 : ISS(read);
6878 INS01 : ISS;
6879 MAC : WR;
6880 %}
6881
6882 // Multiply accumulate
6883 // Eg. MADD w0, w1, w2, w3
6884 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6885 %{
6886 single_instruction;
6887 dst : WR(write);
6888 src1 : ISS(read);
6889 src2 : ISS(read);
6890 src3 : ISS(read);
6891 INS01 : ISS;
6892 MAC : WR;
6893 %}
6894
6895 // Eg. MUL w0, w1, w2
6896 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6897 %{
6898 single_instruction;
6899 fixed_latency(3); // Maximum latency for 64 bit mul
6900 dst : WR(write);
6901 src1 : ISS(read);
6902 src2 : ISS(read);
6903 INS01 : ISS;
6904 MAC : WR;
6905 %}
6906
6907 // Multiply accumulate
6908 // Eg. MADD w0, w1, w2, w3
6909 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6910 %{
6911 single_instruction;
6912 fixed_latency(3); // Maximum latency for 64 bit mul
6913 dst : WR(write);
6914 src1 : ISS(read);
6915 src2 : ISS(read);
6916 src3 : ISS(read);
6917 INS01 : ISS;
6918 MAC : WR;
6919 %}
6920
6921 //------- Divide pipeline operations --------------------
6922
6923 // Eg. SDIV w0, w1, w2
6924 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6925 %{
6926 single_instruction;
6927 fixed_latency(8); // Maximum latency for 32 bit divide
6928 dst : WR(write);
6929 src1 : ISS(read);
6930 src2 : ISS(read);
6931 INS0 : ISS; // Can only dual issue as instruction 0
6932 DIV : WR;
6933 %}
6934
6935 // Eg. SDIV x0, x1, x2
6936 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6937 %{
6938 single_instruction;
6939 fixed_latency(16); // Maximum latency for 64 bit divide
6940 dst : WR(write);
6941 src1 : ISS(read);
6942 src2 : ISS(read);
6943 INS0 : ISS; // Can only dual issue as instruction 0
6944 DIV : WR;
6945 %}
6946
6947 //------- Load pipeline operations ------------------------
6948
6949 // Load - prefetch
6950 // Eg. PFRM <mem>
6951 pipe_class iload_prefetch(memory mem)
6952 %{
6953 single_instruction;
6954 mem : ISS(read);
6955 INS01 : ISS;
6956 LDST : WR;
6957 %}
6958
6959 // Load - reg, mem
6960 // Eg. LDR x0, <mem>
6961 pipe_class iload_reg_mem(iRegI dst, memory mem)
6962 %{
6963 single_instruction;
6964 dst : WR(write);
6965 mem : ISS(read);
6966 INS01 : ISS;
6967 LDST : WR;
6968 %}
6969
6970 // Load - reg, reg
6971 // Eg. LDR x0, [sp, x1]
6972 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6973 %{
6974 single_instruction;
6975 dst : WR(write);
6976 src : ISS(read);
6977 INS01 : ISS;
6978 LDST : WR;
6979 %}
6980
6981 //------- Store pipeline operations -----------------------
6982
6983 // Store - zr, mem
6984 // Eg. STR zr, <mem>
6985 pipe_class istore_mem(memory mem)
6986 %{
6987 single_instruction;
6988 mem : ISS(read);
6989 INS01 : ISS;
6990 LDST : WR;
6991 %}
6992
6993 // Store - reg, mem
6994 // Eg. STR x0, <mem>
6995 pipe_class istore_reg_mem(iRegI src, memory mem)
6996 %{
6997 single_instruction;
6998 mem : ISS(read);
6999 src : EX2(read);
7000 INS01 : ISS;
7001 LDST : WR;
7002 %}
7003
7004 // Store - reg, reg
7005 // Eg. STR x0, [sp, x1]
7006 pipe_class istore_reg_reg(iRegI dst, iRegI src)
7007 %{
7008 single_instruction;
7009 dst : ISS(read);
7010 src : EX2(read);
7011 INS01 : ISS;
7012 LDST : WR;
7013 %}
7014
7015 //------- Store pipeline operations -----------------------
7016
7017 // Branch
7018 pipe_class pipe_branch()
7019 %{
7020 single_instruction;
7021 INS01 : ISS;
7022 BRANCH : EX1;
7023 %}
7024
7025 // Conditional branch
7026 pipe_class pipe_branch_cond(rFlagsReg cr)
7027 %{
7028 single_instruction;
7029 cr : EX1(read);
7030 INS01 : ISS;
7031 BRANCH : EX1;
7032 %}
7033
7034 // Compare & Branch
7035 // EG. CBZ/CBNZ
7036 pipe_class pipe_cmp_branch(iRegI op1)
7037 %{
7038 single_instruction;
7039 op1 : EX1(read);
7040 INS01 : ISS;
7041 BRANCH : EX1;
7042 %}
7043
7044 //------- Synchronisation operations ----------------------
7045
7046 // Any operation requiring serialization.
7047 // EG. DMB/Atomic Ops/Load Acquire/Str Release
7048 pipe_class pipe_serial()
7049 %{
7050 single_instruction;
7051 force_serialization;
7052 fixed_latency(16);
7053 INS01 : ISS(2); // Cannot dual issue with any other instruction
7054 LDST : WR;
7055 %}
7056
7057 // Generic big/slow expanded idiom - also serialized
7058 pipe_class pipe_slow()
7059 %{
7060 instruction_count(10);
7061 multiple_bundles;
7062 force_serialization;
7063 fixed_latency(16);
7064 INS01 : ISS(2); // Cannot dual issue with any other instruction
7065 LDST : WR;
7066 %}
7067
7068 // Empty pipeline class
7069 pipe_class pipe_class_empty()
7070 %{
7071 single_instruction;
7072 fixed_latency(0);
7073 %}
7074
7075 // Default pipeline class.
7076 pipe_class pipe_class_default()
7077 %{
7078 single_instruction;
7079 fixed_latency(2);
7080 %}
7081
7082 // Pipeline class for compares.
7083 pipe_class pipe_class_compare()
7084 %{
7085 single_instruction;
7086 fixed_latency(16);
7087 %}
7088
7089 // Pipeline class for memory operations.
7090 pipe_class pipe_class_memory()
7091 %{
7092 single_instruction;
7093 fixed_latency(16);
7094 %}
7095
7096 // Pipeline class for call.
7097 pipe_class pipe_class_call()
7098 %{
7099 single_instruction;
7100 fixed_latency(100);
7101 %}
7102
7103 // Define the class for the Nop node.
7104 define %{
7105 MachNop = pipe_class_empty;
7106 %}
7107
7108 %}
7109 //----------INSTRUCTIONS-------------------------------------------------------
7110 //
7111 // match -- States which machine-independent subtree may be replaced
7112 // by this instruction.
7113 // ins_cost -- The estimated cost of this instruction is used by instruction
7114 // selection to identify a minimum cost tree of machine
7115 // instructions that matches a tree of machine-independent
7116 // instructions.
7117 // format -- A string providing the disassembly for this instruction.
7118 // The value of an instruction's operand may be inserted
7119 // by referring to it with a '$' prefix.
7120 // opcode -- Three instruction opcodes may be provided. These are referred
7121 // to within an encode class as $primary, $secondary, and $tertiary
7122 // rrspectively. The primary opcode is commonly used to
7123 // indicate the type of machine instruction, while secondary
7124 // and tertiary are often used for prefix options or addressing
7125 // modes.
7126 // ins_encode -- A list of encode classes with parameters. The encode class
7127 // name must have been defined in an 'enc_class' specification
7128 // in the encode section of the architecture description.
7129
7130 // ============================================================================
7131 // Memory (Load/Store) Instructions
7132
7133 // Load Instructions
7134
7135 // Load Byte (8 bit signed)
7136 instruct loadB(iRegINoSp dst, memory1 mem)
7137 %{
7138 match(Set dst (LoadB mem));
7139 predicate(!needs_acquiring_load(n));
7140
7141 ins_cost(4 * INSN_COST);
7142 format %{ "ldrsbw $dst, $mem\t# byte" %}
7143
7144 ins_encode(aarch64_enc_ldrsbw(dst, mem));
7145
7146 ins_pipe(iload_reg_mem);
7147 %}
7148
7149 // Load Byte (8 bit signed) into long
7150 instruct loadB2L(iRegLNoSp dst, memory1 mem)
7151 %{
7152 match(Set dst (ConvI2L (LoadB mem)));
7153 predicate(!needs_acquiring_load(n->in(1)));
7154
7155 ins_cost(4 * INSN_COST);
7156 format %{ "ldrsb $dst, $mem\t# byte" %}
7157
7158 ins_encode(aarch64_enc_ldrsb(dst, mem));
7159
7160 ins_pipe(iload_reg_mem);
7161 %}
7162
7163 // Load Byte (8 bit unsigned)
7164 instruct loadUB(iRegINoSp dst, memory1 mem)
7165 %{
7166 match(Set dst (LoadUB mem));
7167 predicate(!needs_acquiring_load(n));
7168
7169 ins_cost(4 * INSN_COST);
7170 format %{ "ldrbw $dst, $mem\t# byte" %}
7171
7172 ins_encode(aarch64_enc_ldrb(dst, mem));
7173
7174 ins_pipe(iload_reg_mem);
7175 %}
7176
7177 // Load Byte (8 bit unsigned) into long
7178 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
7179 %{
7180 match(Set dst (ConvI2L (LoadUB mem)));
7181 predicate(!needs_acquiring_load(n->in(1)));
7182
7183 ins_cost(4 * INSN_COST);
7184 format %{ "ldrb $dst, $mem\t# byte" %}
7185
7186 ins_encode(aarch64_enc_ldrb(dst, mem));
7187
7188 ins_pipe(iload_reg_mem);
7189 %}
7190
7191 // Load Short (16 bit signed)
7192 instruct loadS(iRegINoSp dst, memory2 mem)
7193 %{
7194 match(Set dst (LoadS mem));
7195 predicate(!needs_acquiring_load(n));
7196
7197 ins_cost(4 * INSN_COST);
7198 format %{ "ldrshw $dst, $mem\t# short" %}
7199
7200 ins_encode(aarch64_enc_ldrshw(dst, mem));
7201
7202 ins_pipe(iload_reg_mem);
7203 %}
7204
7205 // Load Short (16 bit signed) into long
7206 instruct loadS2L(iRegLNoSp dst, memory2 mem)
7207 %{
7208 match(Set dst (ConvI2L (LoadS mem)));
7209 predicate(!needs_acquiring_load(n->in(1)));
7210
7211 ins_cost(4 * INSN_COST);
7212 format %{ "ldrsh $dst, $mem\t# short" %}
7213
7214 ins_encode(aarch64_enc_ldrsh(dst, mem));
7215
7216 ins_pipe(iload_reg_mem);
7217 %}
7218
7219 // Load Char (16 bit unsigned)
7220 instruct loadUS(iRegINoSp dst, memory2 mem)
7221 %{
7222 match(Set dst (LoadUS mem));
7223 predicate(!needs_acquiring_load(n));
7224
7225 ins_cost(4 * INSN_COST);
7226 format %{ "ldrh $dst, $mem\t# short" %}
7227
7228 ins_encode(aarch64_enc_ldrh(dst, mem));
7229
7230 ins_pipe(iload_reg_mem);
7231 %}
7232
7233 // Load Short/Char (16 bit unsigned) into long
7234 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
7235 %{
7236 match(Set dst (ConvI2L (LoadUS mem)));
7237 predicate(!needs_acquiring_load(n->in(1)));
7238
7239 ins_cost(4 * INSN_COST);
7240 format %{ "ldrh $dst, $mem\t# short" %}
7241
7242 ins_encode(aarch64_enc_ldrh(dst, mem));
7243
7244 ins_pipe(iload_reg_mem);
7245 %}
7246
7247 // Load Integer (32 bit signed)
7248 instruct loadI(iRegINoSp dst, memory4 mem)
7249 %{
7250 match(Set dst (LoadI mem));
7251 predicate(!needs_acquiring_load(n));
7252
7253 ins_cost(4 * INSN_COST);
7254 format %{ "ldrw $dst, $mem\t# int" %}
7255
7256 ins_encode(aarch64_enc_ldrw(dst, mem));
7257
7258 ins_pipe(iload_reg_mem);
7259 %}
7260
7261 // Load Integer (32 bit signed) into long
7262 instruct loadI2L(iRegLNoSp dst, memory4 mem)
7263 %{
7264 match(Set dst (ConvI2L (LoadI mem)));
7265 predicate(!needs_acquiring_load(n->in(1)));
7266
7267 ins_cost(4 * INSN_COST);
7268 format %{ "ldrsw $dst, $mem\t# int" %}
7269
7270 ins_encode(aarch64_enc_ldrsw(dst, mem));
7271
7272 ins_pipe(iload_reg_mem);
7273 %}
7274
7275 // Load Integer (32 bit unsigned) into long
7276 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
7277 %{
7278 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7279 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
7280
7281 ins_cost(4 * INSN_COST);
7282 format %{ "ldrw $dst, $mem\t# int" %}
7283
7284 ins_encode(aarch64_enc_ldrw(dst, mem));
7285
7286 ins_pipe(iload_reg_mem);
7287 %}
7288
7289 // Load Long (64 bit signed)
7290 instruct loadL(iRegLNoSp dst, memory8 mem)
7291 %{
7292 match(Set dst (LoadL mem));
7293 predicate(!needs_acquiring_load(n));
7294
7295 ins_cost(4 * INSN_COST);
7296 format %{ "ldr $dst, $mem\t# int" %}
7297
7298 ins_encode(aarch64_enc_ldr(dst, mem));
7299
7300 ins_pipe(iload_reg_mem);
7301 %}
7302
7303 // Load Range
7304 instruct loadRange(iRegINoSp dst, memory4 mem)
7305 %{
7306 match(Set dst (LoadRange mem));
7307
7308 ins_cost(4 * INSN_COST);
7309 format %{ "ldrw $dst, $mem\t# range" %}
7310
7311 ins_encode(aarch64_enc_ldrw(dst, mem));
7312
7313 ins_pipe(iload_reg_mem);
7314 %}
7315
7316 // Load Pointer
7317 instruct loadP(iRegPNoSp dst, memory8 mem)
7318 %{
7319 match(Set dst (LoadP mem));
7320 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
7321
7322 ins_cost(4 * INSN_COST);
7323 format %{ "ldr $dst, $mem\t# ptr" %}
7324
7325 ins_encode(aarch64_enc_ldr(dst, mem));
7326
7327 ins_pipe(iload_reg_mem);
7328 %}
7329
7330 // Load Compressed Pointer
7331 instruct loadN(iRegNNoSp dst, memory4 mem)
7332 %{
7333 match(Set dst (LoadN mem));
7334 predicate(!needs_acquiring_load(n));
7335
7336 ins_cost(4 * INSN_COST);
7337 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
7338
7339 ins_encode(aarch64_enc_ldrw(dst, mem));
7340
7341 ins_pipe(iload_reg_mem);
7342 %}
7343
7344 // Load Klass Pointer
7345 instruct loadKlass(iRegPNoSp dst, memory8 mem)
7346 %{
7347 match(Set dst (LoadKlass mem));
7348 predicate(!needs_acquiring_load(n));
7349
7350 ins_cost(4 * INSN_COST);
7351 format %{ "ldr $dst, $mem\t# class" %}
7352
7353 ins_encode(aarch64_enc_ldr(dst, mem));
7354
7355 ins_pipe(iload_reg_mem);
7356 %}
7357
7358 // Load Narrow Klass Pointer
7359 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
7360 %{
7361 match(Set dst (LoadNKlass mem));
7362 predicate(!needs_acquiring_load(n));
7363
7364 ins_cost(4 * INSN_COST);
7365 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7366
7367 ins_encode(aarch64_enc_ldrw(dst, mem));
7368
7369 ins_pipe(iload_reg_mem);
7370 %}
7371
7372 // Load Float
7373 instruct loadF(vRegF dst, memory4 mem)
7374 %{
7375 match(Set dst (LoadF mem));
7376 predicate(!needs_acquiring_load(n));
7377
7378 ins_cost(4 * INSN_COST);
7379 format %{ "ldrs $dst, $mem\t# float" %}
7380
7381 ins_encode( aarch64_enc_ldrs(dst, mem) );
7382
7383 ins_pipe(pipe_class_memory);
7384 %}
7385
7386 // Load Double
7387 instruct loadD(vRegD dst, memory8 mem)
7388 %{
7389 match(Set dst (LoadD mem));
7390 predicate(!needs_acquiring_load(n));
7391
7392 ins_cost(4 * INSN_COST);
7393 format %{ "ldrd $dst, $mem\t# double" %}
7394
7395 ins_encode( aarch64_enc_ldrd(dst, mem) );
7396
7397 ins_pipe(pipe_class_memory);
7398 %}
7399
7400
7401 // Load Int Constant
7402 instruct loadConI(iRegINoSp dst, immI src)
7403 %{
7404 match(Set dst src);
7405
7406 ins_cost(INSN_COST);
7407 format %{ "mov $dst, $src\t# int" %}
7408
7409 ins_encode( aarch64_enc_movw_imm(dst, src) );
7410
7411 ins_pipe(ialu_imm);
7412 %}
7413
7414 // Load Long Constant
7415 instruct loadConL(iRegLNoSp dst, immL src)
7416 %{
7417 match(Set dst src);
7418
7419 ins_cost(INSN_COST);
7420 format %{ "mov $dst, $src\t# long" %}
7421
7422 ins_encode( aarch64_enc_mov_imm(dst, src) );
7423
7424 ins_pipe(ialu_imm);
7425 %}
7426
7427 // Load Pointer Constant
7428
7429 instruct loadConP(iRegPNoSp dst, immP con)
7430 %{
7431 match(Set dst con);
7432
7433 ins_cost(INSN_COST * 4);
7434 format %{
7435 "mov $dst, $con\t# ptr"
7436 %}
7437
7438 ins_encode(aarch64_enc_mov_p(dst, con));
7439
7440 ins_pipe(ialu_imm);
7441 %}
7442
7443 // Load Null Pointer Constant
7444
7445 instruct loadConP0(iRegPNoSp dst, immP0 con)
7446 %{
7447 match(Set dst con);
7448
7449 ins_cost(INSN_COST);
7450 format %{ "mov $dst, $con\t# nullptr ptr" %}
7451
7452 ins_encode(aarch64_enc_mov_p0(dst, con));
7453
7454 ins_pipe(ialu_imm);
7455 %}
7456
7457 // Load Pointer Constant One
7458
7459 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7460 %{
7461 match(Set dst con);
7462
7463 ins_cost(INSN_COST);
7464 format %{ "mov $dst, $con\t# nullptr ptr" %}
7465
7466 ins_encode(aarch64_enc_mov_p1(dst, con));
7467
7468 ins_pipe(ialu_imm);
7469 %}
7470
7471 // Load Byte Map Base Constant
7472
7473 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7474 %{
7475 match(Set dst con);
7476
7477 ins_cost(INSN_COST);
7478 format %{ "adr $dst, $con\t# Byte Map Base" %}
7479
7480 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7481
7482 ins_pipe(ialu_imm);
7483 %}
7484
7485 // Load Narrow Pointer Constant
7486
7487 instruct loadConN(iRegNNoSp dst, immN con)
7488 %{
7489 match(Set dst con);
7490
7491 ins_cost(INSN_COST * 4);
7492 format %{ "mov $dst, $con\t# compressed ptr" %}
7493
7494 ins_encode(aarch64_enc_mov_n(dst, con));
7495
7496 ins_pipe(ialu_imm);
7497 %}
7498
7499 // Load Narrow Null Pointer Constant
7500
7501 instruct loadConN0(iRegNNoSp dst, immN0 con)
7502 %{
7503 match(Set dst con);
7504
7505 ins_cost(INSN_COST);
7506 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
7507
7508 ins_encode(aarch64_enc_mov_n0(dst, con));
7509
7510 ins_pipe(ialu_imm);
7511 %}
7512
7513 // Load Narrow Klass Constant
7514
7515 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7516 %{
7517 match(Set dst con);
7518
7519 ins_cost(INSN_COST);
7520 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7521
7522 ins_encode(aarch64_enc_mov_nk(dst, con));
7523
7524 ins_pipe(ialu_imm);
7525 %}
7526
7527 // Load Packed Float Constant
7528
7529 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7530 match(Set dst con);
7531 ins_cost(INSN_COST * 4);
7532 format %{ "fmovs $dst, $con"%}
7533 ins_encode %{
7534 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7535 %}
7536
7537 ins_pipe(fp_imm_s);
7538 %}
7539
7540 // Load Float Constant
7541
7542 instruct loadConF(vRegF dst, immF con) %{
7543 match(Set dst con);
7544
7545 ins_cost(INSN_COST * 4);
7546
7547 format %{
7548 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7549 %}
7550
7551 ins_encode %{
7552 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7553 %}
7554
7555 ins_pipe(fp_load_constant_s);
7556 %}
7557
7558 // Load Packed Double Constant
7559
7560 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7561 match(Set dst con);
7562 ins_cost(INSN_COST);
7563 format %{ "fmovd $dst, $con"%}
7564 ins_encode %{
7565 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7566 %}
7567
7568 ins_pipe(fp_imm_d);
7569 %}
7570
7571 // Load Double Constant
7572
7573 instruct loadConD(vRegD dst, immD con) %{
7574 match(Set dst con);
7575
7576 ins_cost(INSN_COST * 5);
7577 format %{
7578 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7579 %}
7580
7581 ins_encode %{
7582 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7583 %}
7584
7585 ins_pipe(fp_load_constant_d);
7586 %}
7587
7588 // Store Instructions
7589
7590 // Store CMS card-mark Immediate
7591 instruct storeimmCM0(immI0 zero, memory1 mem)
7592 %{
7593 match(Set mem (StoreCM mem zero));
7594
7595 ins_cost(INSN_COST);
7596 format %{ "storestore (elided)\n\t"
7597 "strb zr, $mem\t# byte" %}
7598
7599 ins_encode(aarch64_enc_strb0(mem));
7600
7601 ins_pipe(istore_mem);
7602 %}
7603
7604 // Store CMS card-mark Immediate with intervening StoreStore
7605 // needed when using CMS with no conditional card marking
7606 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
7607 %{
7608 match(Set mem (StoreCM mem zero));
7609
7610 ins_cost(INSN_COST * 2);
7611 format %{ "storestore\n\t"
7612 "dmb ishst"
7613 "\n\tstrb zr, $mem\t# byte" %}
7614
7615 ins_encode(aarch64_enc_strb0_ordered(mem));
7616
7617 ins_pipe(istore_mem);
7618 %}
7619
7620 // Store Byte
7621 instruct storeB(iRegIorL2I src, memory1 mem)
7622 %{
7623 match(Set mem (StoreB mem src));
7624 predicate(!needs_releasing_store(n));
7625
7626 ins_cost(INSN_COST);
7627 format %{ "strb $src, $mem\t# byte" %}
7628
7629 ins_encode(aarch64_enc_strb(src, mem));
7630
7631 ins_pipe(istore_reg_mem);
7632 %}
7633
7634
7635 instruct storeimmB0(immI0 zero, memory1 mem)
7636 %{
7637 match(Set mem (StoreB mem zero));
7638 predicate(!needs_releasing_store(n));
7639
7640 ins_cost(INSN_COST);
7641 format %{ "strb rscractch2, $mem\t# byte" %}
7642
7643 ins_encode(aarch64_enc_strb0(mem));
7644
7645 ins_pipe(istore_mem);
7646 %}
7647
7648 // Store Char/Short
7649 instruct storeC(iRegIorL2I src, memory2 mem)
7650 %{
7651 match(Set mem (StoreC mem src));
7652 predicate(!needs_releasing_store(n));
7653
7654 ins_cost(INSN_COST);
7655 format %{ "strh $src, $mem\t# short" %}
7656
7657 ins_encode(aarch64_enc_strh(src, mem));
7658
7659 ins_pipe(istore_reg_mem);
7660 %}
7661
7662 instruct storeimmC0(immI0 zero, memory2 mem)
7663 %{
7664 match(Set mem (StoreC mem zero));
7665 predicate(!needs_releasing_store(n));
7666
7667 ins_cost(INSN_COST);
7668 format %{ "strh zr, $mem\t# short" %}
7669
7670 ins_encode(aarch64_enc_strh0(mem));
7671
7672 ins_pipe(istore_mem);
7673 %}
7674
7675 // Store Integer
7676
7677 instruct storeI(iRegIorL2I src, memory4 mem)
7678 %{
7679 match(Set mem(StoreI mem src));
7680 predicate(!needs_releasing_store(n));
7681
7682 ins_cost(INSN_COST);
7683 format %{ "strw $src, $mem\t# int" %}
7684
7685 ins_encode(aarch64_enc_strw(src, mem));
7686
7687 ins_pipe(istore_reg_mem);
7688 %}
7689
7690 instruct storeimmI0(immI0 zero, memory4 mem)
7691 %{
7692 match(Set mem(StoreI mem zero));
7693 predicate(!needs_releasing_store(n));
7694
7695 ins_cost(INSN_COST);
7696 format %{ "strw zr, $mem\t# int" %}
7697
7698 ins_encode(aarch64_enc_strw0(mem));
7699
7700 ins_pipe(istore_mem);
7701 %}
7702
7703 // Store Long (64 bit signed)
7704 instruct storeL(iRegL src, memory8 mem)
7705 %{
7706 match(Set mem (StoreL mem src));
7707 predicate(!needs_releasing_store(n));
7708
7709 ins_cost(INSN_COST);
7710 format %{ "str $src, $mem\t# int" %}
7711
7712 ins_encode(aarch64_enc_str(src, mem));
7713
7714 ins_pipe(istore_reg_mem);
7715 %}
7716
7717 // Store Long (64 bit signed)
7718 instruct storeimmL0(immL0 zero, memory8 mem)
7719 %{
7720 match(Set mem (StoreL mem zero));
7721 predicate(!needs_releasing_store(n));
7722
7723 ins_cost(INSN_COST);
7724 format %{ "str zr, $mem\t# int" %}
7725
7726 ins_encode(aarch64_enc_str0(mem));
7727
7728 ins_pipe(istore_mem);
7729 %}
7730
7731 // Store Pointer
7732 instruct storeP(iRegP src, memory8 mem)
7733 %{
7734 match(Set mem (StoreP mem src));
7735 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7736
7737 ins_cost(INSN_COST);
7738 format %{ "str $src, $mem\t# ptr" %}
7739
7740 ins_encode(aarch64_enc_str(src, mem));
7741
7742 ins_pipe(istore_reg_mem);
7743 %}
7744
7745 // Store Pointer
7746 instruct storeimmP0(immP0 zero, memory8 mem)
7747 %{
7748 match(Set mem (StoreP mem zero));
7749 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7750
7751 ins_cost(INSN_COST);
7752 format %{ "str zr, $mem\t# ptr" %}
7753
7754 ins_encode(aarch64_enc_str0(mem));
7755
7756 ins_pipe(istore_mem);
7757 %}
7758
7759 // Store Compressed Pointer
7760 instruct storeN(iRegN src, memory4 mem)
7761 %{
7762 match(Set mem (StoreN mem src));
7763 predicate(!needs_releasing_store(n));
7764
7765 ins_cost(INSN_COST);
7766 format %{ "strw $src, $mem\t# compressed ptr" %}
7767
7768 ins_encode(aarch64_enc_strw(src, mem));
7769
7770 ins_pipe(istore_reg_mem);
7771 %}
7772
7773 instruct storeImmN0(immN0 zero, memory4 mem)
7774 %{
7775 match(Set mem (StoreN mem zero));
7776 predicate(!needs_releasing_store(n));
7777
7778 ins_cost(INSN_COST);
7779 format %{ "strw zr, $mem\t# compressed ptr" %}
7780
7781 ins_encode(aarch64_enc_strw0(mem));
7782
7783 ins_pipe(istore_mem);
7784 %}
7785
7786 // Store Float
7787 instruct storeF(vRegF src, memory4 mem)
7788 %{
7789 match(Set mem (StoreF mem src));
7790 predicate(!needs_releasing_store(n));
7791
7792 ins_cost(INSN_COST);
7793 format %{ "strs $src, $mem\t# float" %}
7794
7795 ins_encode( aarch64_enc_strs(src, mem) );
7796
7797 ins_pipe(pipe_class_memory);
7798 %}
7799
7800 // TODO
7801 // implement storeImmF0 and storeFImmPacked
7802
7803 // Store Double
7804 instruct storeD(vRegD src, memory8 mem)
7805 %{
7806 match(Set mem (StoreD mem src));
7807 predicate(!needs_releasing_store(n));
7808
7809 ins_cost(INSN_COST);
7810 format %{ "strd $src, $mem\t# double" %}
7811
7812 ins_encode( aarch64_enc_strd(src, mem) );
7813
7814 ins_pipe(pipe_class_memory);
7815 %}
7816
7817 // Store Compressed Klass Pointer
7818 instruct storeNKlass(iRegN src, memory4 mem)
7819 %{
7820 predicate(!needs_releasing_store(n));
7821 match(Set mem (StoreNKlass mem src));
7822
7823 ins_cost(INSN_COST);
7824 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7825
7826 ins_encode(aarch64_enc_strw(src, mem));
7827
7828 ins_pipe(istore_reg_mem);
7829 %}
7830
7831 // TODO
7832 // implement storeImmD0 and storeDImmPacked
7833
7834 // prefetch instructions
7835 // Must be safe to execute with invalid address (cannot fault).
7836
7837 instruct prefetchalloc( memory8 mem ) %{
7838 match(PrefetchAllocation mem);
7839
7840 ins_cost(INSN_COST);
7841 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7842
7843 ins_encode( aarch64_enc_prefetchw(mem) );
7844
7845 ins_pipe(iload_prefetch);
7846 %}
7847
7848 // ---------------- volatile loads and stores ----------------
7849
7850 // Load Byte (8 bit signed)
7851 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7852 %{
7853 match(Set dst (LoadB mem));
7854
7855 ins_cost(VOLATILE_REF_COST);
7856 format %{ "ldarsb $dst, $mem\t# byte" %}
7857
7858 ins_encode(aarch64_enc_ldarsb(dst, mem));
7859
7860 ins_pipe(pipe_serial);
7861 %}
7862
7863 // Load Byte (8 bit signed) into long
7864 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7865 %{
7866 match(Set dst (ConvI2L (LoadB mem)));
7867
7868 ins_cost(VOLATILE_REF_COST);
7869 format %{ "ldarsb $dst, $mem\t# byte" %}
7870
7871 ins_encode(aarch64_enc_ldarsb(dst, mem));
7872
7873 ins_pipe(pipe_serial);
7874 %}
7875
7876 // Load Byte (8 bit unsigned)
7877 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7878 %{
7879 match(Set dst (LoadUB mem));
7880
7881 ins_cost(VOLATILE_REF_COST);
7882 format %{ "ldarb $dst, $mem\t# byte" %}
7883
7884 ins_encode(aarch64_enc_ldarb(dst, mem));
7885
7886 ins_pipe(pipe_serial);
7887 %}
7888
7889 // Load Byte (8 bit unsigned) into long
7890 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7891 %{
7892 match(Set dst (ConvI2L (LoadUB mem)));
7893
7894 ins_cost(VOLATILE_REF_COST);
7895 format %{ "ldarb $dst, $mem\t# byte" %}
7896
7897 ins_encode(aarch64_enc_ldarb(dst, mem));
7898
7899 ins_pipe(pipe_serial);
7900 %}
7901
7902 // Load Short (16 bit signed)
7903 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7904 %{
7905 match(Set dst (LoadS mem));
7906
7907 ins_cost(VOLATILE_REF_COST);
7908 format %{ "ldarshw $dst, $mem\t# short" %}
7909
7910 ins_encode(aarch64_enc_ldarshw(dst, mem));
7911
7912 ins_pipe(pipe_serial);
7913 %}
7914
7915 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7916 %{
7917 match(Set dst (LoadUS mem));
7918
7919 ins_cost(VOLATILE_REF_COST);
7920 format %{ "ldarhw $dst, $mem\t# short" %}
7921
7922 ins_encode(aarch64_enc_ldarhw(dst, mem));
7923
7924 ins_pipe(pipe_serial);
7925 %}
7926
7927 // Load Short/Char (16 bit unsigned) into long
7928 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7929 %{
7930 match(Set dst (ConvI2L (LoadUS mem)));
7931
7932 ins_cost(VOLATILE_REF_COST);
7933 format %{ "ldarh $dst, $mem\t# short" %}
7934
7935 ins_encode(aarch64_enc_ldarh(dst, mem));
7936
7937 ins_pipe(pipe_serial);
7938 %}
7939
7940 // Load Short/Char (16 bit signed) into long
7941 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7942 %{
7943 match(Set dst (ConvI2L (LoadS mem)));
7944
7945 ins_cost(VOLATILE_REF_COST);
7946 format %{ "ldarh $dst, $mem\t# short" %}
7947
7948 ins_encode(aarch64_enc_ldarsh(dst, mem));
7949
7950 ins_pipe(pipe_serial);
7951 %}
7952
7953 // Load Integer (32 bit signed)
7954 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7955 %{
7956 match(Set dst (LoadI mem));
7957
7958 ins_cost(VOLATILE_REF_COST);
7959 format %{ "ldarw $dst, $mem\t# int" %}
7960
7961 ins_encode(aarch64_enc_ldarw(dst, mem));
7962
7963 ins_pipe(pipe_serial);
7964 %}
7965
7966 // Load Integer (32 bit unsigned) into long
7967 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7968 %{
7969 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7970
7971 ins_cost(VOLATILE_REF_COST);
7972 format %{ "ldarw $dst, $mem\t# int" %}
7973
7974 ins_encode(aarch64_enc_ldarw(dst, mem));
7975
7976 ins_pipe(pipe_serial);
7977 %}
7978
7979 // Load Long (64 bit signed)
7980 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7981 %{
7982 match(Set dst (LoadL mem));
7983
7984 ins_cost(VOLATILE_REF_COST);
7985 format %{ "ldar $dst, $mem\t# int" %}
7986
7987 ins_encode(aarch64_enc_ldar(dst, mem));
7988
7989 ins_pipe(pipe_serial);
7990 %}
7991
7992 // Load Pointer
7993 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7994 %{
7995 match(Set dst (LoadP mem));
7996 predicate(n->as_Load()->barrier_data() == 0);
7997
7998 ins_cost(VOLATILE_REF_COST);
7999 format %{ "ldar $dst, $mem\t# ptr" %}
8000
8001 ins_encode(aarch64_enc_ldar(dst, mem));
8002
8003 ins_pipe(pipe_serial);
8004 %}
8005
8006 // Load Compressed Pointer
8007 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
8008 %{
8009 match(Set dst (LoadN mem));
8010
8011 ins_cost(VOLATILE_REF_COST);
8012 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
8013
8014 ins_encode(aarch64_enc_ldarw(dst, mem));
8015
8016 ins_pipe(pipe_serial);
8017 %}
8018
8019 // Load Float
8020 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
8021 %{
8022 match(Set dst (LoadF mem));
8023
8024 ins_cost(VOLATILE_REF_COST);
8025 format %{ "ldars $dst, $mem\t# float" %}
8026
8027 ins_encode( aarch64_enc_fldars(dst, mem) );
8028
8029 ins_pipe(pipe_serial);
8030 %}
8031
8032 // Load Double
8033 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
8034 %{
8035 match(Set dst (LoadD mem));
8036
8037 ins_cost(VOLATILE_REF_COST);
8038 format %{ "ldard $dst, $mem\t# double" %}
8039
8040 ins_encode( aarch64_enc_fldard(dst, mem) );
8041
8042 ins_pipe(pipe_serial);
8043 %}
8044
8045 // Store Byte
8046 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8047 %{
8048 match(Set mem (StoreB mem src));
8049
8050 ins_cost(VOLATILE_REF_COST);
8051 format %{ "stlrb $src, $mem\t# byte" %}
8052
8053 ins_encode(aarch64_enc_stlrb(src, mem));
8054
8055 ins_pipe(pipe_class_memory);
8056 %}
8057
8058 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8059 %{
8060 match(Set mem (StoreB mem zero));
8061
8062 ins_cost(VOLATILE_REF_COST);
8063 format %{ "stlrb zr, $mem\t# byte" %}
8064
8065 ins_encode(aarch64_enc_stlrb0(mem));
8066
8067 ins_pipe(pipe_class_memory);
8068 %}
8069
8070 // Store Char/Short
8071 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8072 %{
8073 match(Set mem (StoreC mem src));
8074
8075 ins_cost(VOLATILE_REF_COST);
8076 format %{ "stlrh $src, $mem\t# short" %}
8077
8078 ins_encode(aarch64_enc_stlrh(src, mem));
8079
8080 ins_pipe(pipe_class_memory);
8081 %}
8082
8083 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8084 %{
8085 match(Set mem (StoreC mem zero));
8086
8087 ins_cost(VOLATILE_REF_COST);
8088 format %{ "stlrh zr, $mem\t# short" %}
8089
8090 ins_encode(aarch64_enc_stlrh0(mem));
8091
8092 ins_pipe(pipe_class_memory);
8093 %}
8094
8095 // Store Integer
8096
8097 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8098 %{
8099 match(Set mem(StoreI mem src));
8100
8101 ins_cost(VOLATILE_REF_COST);
8102 format %{ "stlrw $src, $mem\t# int" %}
8103
8104 ins_encode(aarch64_enc_stlrw(src, mem));
8105
8106 ins_pipe(pipe_class_memory);
8107 %}
8108
8109 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8110 %{
8111 match(Set mem(StoreI mem zero));
8112
8113 ins_cost(VOLATILE_REF_COST);
8114 format %{ "stlrw zr, $mem\t# int" %}
8115
8116 ins_encode(aarch64_enc_stlrw0(mem));
8117
8118 ins_pipe(pipe_class_memory);
8119 %}
8120
8121 // Store Long (64 bit signed)
8122 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
8123 %{
8124 match(Set mem (StoreL mem src));
8125
8126 ins_cost(VOLATILE_REF_COST);
8127 format %{ "stlr $src, $mem\t# int" %}
8128
8129 ins_encode(aarch64_enc_stlr(src, mem));
8130
8131 ins_pipe(pipe_class_memory);
8132 %}
8133
8134 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
8135 %{
8136 match(Set mem (StoreL mem zero));
8137
8138 ins_cost(VOLATILE_REF_COST);
8139 format %{ "stlr zr, $mem\t# int" %}
8140
8141 ins_encode(aarch64_enc_stlr0(mem));
8142
8143 ins_pipe(pipe_class_memory);
8144 %}
8145
8146 // Store Pointer
8147 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
8148 %{
8149 match(Set mem (StoreP mem src));
8150 predicate(n->as_Store()->barrier_data() == 0);
8151
8152 ins_cost(VOLATILE_REF_COST);
8153 format %{ "stlr $src, $mem\t# ptr" %}
8154
8155 ins_encode(aarch64_enc_stlr(src, mem));
8156
8157 ins_pipe(pipe_class_memory);
8158 %}
8159
8160 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
8161 %{
8162 match(Set mem (StoreP mem zero));
8163 predicate(n->as_Store()->barrier_data() == 0);
8164
8165 ins_cost(VOLATILE_REF_COST);
8166 format %{ "stlr zr, $mem\t# ptr" %}
8167
8168 ins_encode(aarch64_enc_stlr0(mem));
8169
8170 ins_pipe(pipe_class_memory);
8171 %}
8172
8173 // Store Compressed Pointer
8174 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
8175 %{
8176 match(Set mem (StoreN mem src));
8177
8178 ins_cost(VOLATILE_REF_COST);
8179 format %{ "stlrw $src, $mem\t# compressed ptr" %}
8180
8181 ins_encode(aarch64_enc_stlrw(src, mem));
8182
8183 ins_pipe(pipe_class_memory);
8184 %}
8185
8186 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
8187 %{
8188 match(Set mem (StoreN mem zero));
8189
8190 ins_cost(VOLATILE_REF_COST);
8191 format %{ "stlrw zr, $mem\t# compressed ptr" %}
8192
8193 ins_encode(aarch64_enc_stlrw0(mem));
8194
8195 ins_pipe(pipe_class_memory);
8196 %}
8197
8198 // Store Float
8199 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
8200 %{
8201 match(Set mem (StoreF mem src));
8202
8203 ins_cost(VOLATILE_REF_COST);
8204 format %{ "stlrs $src, $mem\t# float" %}
8205
8206 ins_encode( aarch64_enc_fstlrs(src, mem) );
8207
8208 ins_pipe(pipe_class_memory);
8209 %}
8210
8211 // TODO
8212 // implement storeImmF0 and storeFImmPacked
8213
8214 // Store Double
8215 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
8216 %{
8217 match(Set mem (StoreD mem src));
8218
8219 ins_cost(VOLATILE_REF_COST);
8220 format %{ "stlrd $src, $mem\t# double" %}
8221
8222 ins_encode( aarch64_enc_fstlrd(src, mem) );
8223
8224 ins_pipe(pipe_class_memory);
8225 %}
8226
8227 // ---------------- end of volatile loads and stores ----------------
8228
8229 instruct cacheWB(indirect addr)
8230 %{
8231 predicate(VM_Version::supports_data_cache_line_flush());
8232 match(CacheWB addr);
8233
8234 ins_cost(100);
8235 format %{"cache wb $addr" %}
8236 ins_encode %{
8237 assert($addr->index_position() < 0, "should be");
8238 assert($addr$$disp == 0, "should be");
8239 __ cache_wb(Address($addr$$base$$Register, 0));
8240 %}
8241 ins_pipe(pipe_slow); // XXX
8242 %}
8243
8244 instruct cacheWBPreSync()
8245 %{
8246 predicate(VM_Version::supports_data_cache_line_flush());
8247 match(CacheWBPreSync);
8248
8249 ins_cost(100);
8250 format %{"cache wb presync" %}
8251 ins_encode %{
8252 __ cache_wbsync(true);
8253 %}
8254 ins_pipe(pipe_slow); // XXX
8255 %}
8256
8257 instruct cacheWBPostSync()
8258 %{
8259 predicate(VM_Version::supports_data_cache_line_flush());
8260 match(CacheWBPostSync);
8261
8262 ins_cost(100);
8263 format %{"cache wb postsync" %}
8264 ins_encode %{
8265 __ cache_wbsync(false);
8266 %}
8267 ins_pipe(pipe_slow); // XXX
8268 %}
8269
8270 // ============================================================================
8271 // BSWAP Instructions
8272
8273 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
8274 match(Set dst (ReverseBytesI src));
8275
8276 ins_cost(INSN_COST);
8277 format %{ "revw $dst, $src" %}
8278
8279 ins_encode %{
8280 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
8281 %}
8282
8283 ins_pipe(ialu_reg);
8284 %}
8285
8286 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
8287 match(Set dst (ReverseBytesL src));
8288
8289 ins_cost(INSN_COST);
8290 format %{ "rev $dst, $src" %}
8291
8292 ins_encode %{
8293 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
8294 %}
8295
8296 ins_pipe(ialu_reg);
8297 %}
8298
8299 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
8300 match(Set dst (ReverseBytesUS src));
8301
8302 ins_cost(INSN_COST);
8303 format %{ "rev16w $dst, $src" %}
8304
8305 ins_encode %{
8306 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8307 %}
8308
8309 ins_pipe(ialu_reg);
8310 %}
8311
8312 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
8313 match(Set dst (ReverseBytesS src));
8314
8315 ins_cost(INSN_COST);
8316 format %{ "rev16w $dst, $src\n\t"
8317 "sbfmw $dst, $dst, #0, #15" %}
8318
8319 ins_encode %{
8320 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8321 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
8322 %}
8323
8324 ins_pipe(ialu_reg);
8325 %}
8326
8327 // ============================================================================
8328 // Zero Count Instructions
8329
8330 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8331 match(Set dst (CountLeadingZerosI src));
8332
8333 ins_cost(INSN_COST);
8334 format %{ "clzw $dst, $src" %}
8335 ins_encode %{
8336 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
8337 %}
8338
8339 ins_pipe(ialu_reg);
8340 %}
8341
8342 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
8343 match(Set dst (CountLeadingZerosL src));
8344
8345 ins_cost(INSN_COST);
8346 format %{ "clz $dst, $src" %}
8347 ins_encode %{
8348 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
8349 %}
8350
8351 ins_pipe(ialu_reg);
8352 %}
8353
8354 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8355 match(Set dst (CountTrailingZerosI src));
8356
8357 ins_cost(INSN_COST * 2);
8358 format %{ "rbitw $dst, $src\n\t"
8359 "clzw $dst, $dst" %}
8360 ins_encode %{
8361 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
8362 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
8363 %}
8364
8365 ins_pipe(ialu_reg);
8366 %}
8367
8368 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
8369 match(Set dst (CountTrailingZerosL src));
8370
8371 ins_cost(INSN_COST * 2);
8372 format %{ "rbit $dst, $src\n\t"
8373 "clz $dst, $dst" %}
8374 ins_encode %{
8375 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
8376 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
8377 %}
8378
8379 ins_pipe(ialu_reg);
8380 %}
8381
8382 //---------- Population Count Instructions -------------------------------------
8383 //
8384
8385 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
8386 match(Set dst (PopCountI src));
8387 effect(TEMP tmp);
8388 ins_cost(INSN_COST * 13);
8389
8390 format %{ "movw $src, $src\n\t"
8391 "mov $tmp, $src\t# vector (1D)\n\t"
8392 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8393 "addv $tmp, $tmp\t# vector (8B)\n\t"
8394 "mov $dst, $tmp\t# vector (1D)" %}
8395 ins_encode %{
8396 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
8397 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8398 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8399 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8400 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8401 %}
8402
8403 ins_pipe(pipe_class_default);
8404 %}
8405
8406 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
8407 match(Set dst (PopCountI (LoadI mem)));
8408 effect(TEMP tmp);
8409 ins_cost(INSN_COST * 13);
8410
8411 format %{ "ldrs $tmp, $mem\n\t"
8412 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8413 "addv $tmp, $tmp\t# vector (8B)\n\t"
8414 "mov $dst, $tmp\t# vector (1D)" %}
8415 ins_encode %{
8416 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8417 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
8418 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
8419 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8420 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8421 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8422 %}
8423
8424 ins_pipe(pipe_class_default);
8425 %}
8426
8427 // Note: Long.bitCount(long) returns an int.
8428 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
8429 match(Set dst (PopCountL src));
8430 effect(TEMP tmp);
8431 ins_cost(INSN_COST * 13);
8432
8433 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
8434 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8435 "addv $tmp, $tmp\t# vector (8B)\n\t"
8436 "mov $dst, $tmp\t# vector (1D)" %}
8437 ins_encode %{
8438 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8439 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8440 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8441 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8442 %}
8443
8444 ins_pipe(pipe_class_default);
8445 %}
8446
8447 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
8448 match(Set dst (PopCountL (LoadL mem)));
8449 effect(TEMP tmp);
8450 ins_cost(INSN_COST * 13);
8451
8452 format %{ "ldrd $tmp, $mem\n\t"
8453 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8454 "addv $tmp, $tmp\t# vector (8B)\n\t"
8455 "mov $dst, $tmp\t# vector (1D)" %}
8456 ins_encode %{
8457 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8458 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8459 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
8460 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8461 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8462 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8463 %}
8464
8465 ins_pipe(pipe_class_default);
8466 %}
8467
8468 // ============================================================================
8469 // MemBar Instruction
8470
8471 instruct load_fence() %{
8472 match(LoadFence);
8473 ins_cost(VOLATILE_REF_COST);
8474
8475 format %{ "load_fence" %}
8476
8477 ins_encode %{
8478 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8479 %}
8480 ins_pipe(pipe_serial);
8481 %}
8482
8483 instruct unnecessary_membar_acquire() %{
8484 predicate(unnecessary_acquire(n));
8485 match(MemBarAcquire);
8486 ins_cost(0);
8487
8488 format %{ "membar_acquire (elided)" %}
8489
8490 ins_encode %{
8491 __ block_comment("membar_acquire (elided)");
8492 %}
8493
8494 ins_pipe(pipe_class_empty);
8495 %}
8496
8497 instruct membar_acquire() %{
8498 match(MemBarAcquire);
8499 ins_cost(VOLATILE_REF_COST);
8500
8501 format %{ "membar_acquire\n\t"
8502 "dmb ish" %}
8503
8504 ins_encode %{
8505 __ block_comment("membar_acquire");
8506 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8507 %}
8508
8509 ins_pipe(pipe_serial);
8510 %}
8511
8512
8513 instruct membar_acquire_lock() %{
8514 match(MemBarAcquireLock);
8515 ins_cost(VOLATILE_REF_COST);
8516
8517 format %{ "membar_acquire_lock (elided)" %}
8518
8519 ins_encode %{
8520 __ block_comment("membar_acquire_lock (elided)");
8521 %}
8522
8523 ins_pipe(pipe_serial);
8524 %}
8525
8526 instruct store_fence() %{
8527 match(StoreFence);
8528 ins_cost(VOLATILE_REF_COST);
8529
8530 format %{ "store_fence" %}
8531
8532 ins_encode %{
8533 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8534 %}
8535 ins_pipe(pipe_serial);
8536 %}
8537
8538 instruct unnecessary_membar_release() %{
8539 predicate(unnecessary_release(n));
8540 match(MemBarRelease);
8541 ins_cost(0);
8542
8543 format %{ "membar_release (elided)" %}
8544
8545 ins_encode %{
8546 __ block_comment("membar_release (elided)");
8547 %}
8548 ins_pipe(pipe_serial);
8549 %}
8550
8551 instruct membar_release() %{
8552 match(MemBarRelease);
8553 ins_cost(VOLATILE_REF_COST);
8554
8555 format %{ "membar_release\n\t"
8556 "dmb ish" %}
8557
8558 ins_encode %{
8559 __ block_comment("membar_release");
8560 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8561 %}
8562 ins_pipe(pipe_serial);
8563 %}
8564
8565 instruct membar_storestore() %{
8566 match(MemBarStoreStore);
8567 match(StoreStoreFence);
8568 ins_cost(VOLATILE_REF_COST);
8569
8570 format %{ "MEMBAR-store-store" %}
8571
8572 ins_encode %{
8573 __ membar(Assembler::StoreStore);
8574 %}
8575 ins_pipe(pipe_serial);
8576 %}
8577
8578 instruct membar_release_lock() %{
8579 match(MemBarReleaseLock);
8580 ins_cost(VOLATILE_REF_COST);
8581
8582 format %{ "membar_release_lock (elided)" %}
8583
8584 ins_encode %{
8585 __ block_comment("membar_release_lock (elided)");
8586 %}
8587
8588 ins_pipe(pipe_serial);
8589 %}
8590
8591 instruct unnecessary_membar_volatile() %{
8592 predicate(unnecessary_volatile(n));
8593 match(MemBarVolatile);
8594 ins_cost(0);
8595
8596 format %{ "membar_volatile (elided)" %}
8597
8598 ins_encode %{
8599 __ block_comment("membar_volatile (elided)");
8600 %}
8601
8602 ins_pipe(pipe_serial);
8603 %}
8604
8605 instruct membar_volatile() %{
8606 match(MemBarVolatile);
8607 ins_cost(VOLATILE_REF_COST*100);
8608
8609 format %{ "membar_volatile\n\t"
8610 "dmb ish"%}
8611
8612 ins_encode %{
8613 __ block_comment("membar_volatile");
8614 __ membar(Assembler::StoreLoad);
8615 %}
8616
8617 ins_pipe(pipe_serial);
8618 %}
8619
8620 // ============================================================================
8621 // Cast/Convert Instructions
8622
8623 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8624 match(Set dst (CastX2P src));
8625
8626 ins_cost(INSN_COST);
8627 format %{ "mov $dst, $src\t# long -> ptr" %}
8628
8629 ins_encode %{
8630 if ($dst$$reg != $src$$reg) {
8631 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8632 }
8633 %}
8634
8635 ins_pipe(ialu_reg);
8636 %}
8637
8638 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8639 match(Set dst (CastP2X src));
8640
8641 ins_cost(INSN_COST);
8642 format %{ "mov $dst, $src\t# ptr -> long" %}
8643
8644 ins_encode %{
8645 if ($dst$$reg != $src$$reg) {
8646 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8647 }
8648 %}
8649
8650 ins_pipe(ialu_reg);
8651 %}
8652
8653 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8654 match(Set dst (CastP2X src));
8655
8656 ins_cost(INSN_COST);
8657 format %{ "mov $dst, $src\t# ptr -> long" %}
8658
8659 ins_encode %{
8660 if ($dst$$reg != $src$$reg) {
8661 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8662 }
8663 %}
8664
8665 ins_pipe(ialu_reg);
8666 %}
8667
8668 // Convert oop into int for vectors alignment masking
8669 instruct convP2I(iRegINoSp dst, iRegP src) %{
8670 match(Set dst (ConvL2I (CastP2X src)));
8671
8672 ins_cost(INSN_COST);
8673 format %{ "movw $dst, $src\t# ptr -> int" %}
8674 ins_encode %{
8675 __ movw($dst$$Register, $src$$Register);
8676 %}
8677
8678 ins_pipe(ialu_reg);
8679 %}
8680
8681 // Convert compressed oop into int for vectors alignment masking
8682 // in case of 32bit oops (heap < 4Gb).
8683 instruct convN2I(iRegINoSp dst, iRegN src)
8684 %{
8685 predicate(CompressedOops::shift() == 0);
8686 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8687
8688 ins_cost(INSN_COST);
8689 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8690 ins_encode %{
8691 __ movw($dst$$Register, $src$$Register);
8692 %}
8693
8694 ins_pipe(ialu_reg);
8695 %}
8696
8697
8698 // Convert oop pointer into compressed form
8699 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8700 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8701 match(Set dst (EncodeP src));
8702 effect(KILL cr);
8703 ins_cost(INSN_COST * 3);
8704 format %{ "encode_heap_oop $dst, $src" %}
8705 ins_encode %{
8706 Register s = $src$$Register;
8707 Register d = $dst$$Register;
8708 __ encode_heap_oop(d, s);
8709 %}
8710 ins_pipe(ialu_reg);
8711 %}
8712
8713 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8714 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8715 match(Set dst (EncodeP src));
8716 ins_cost(INSN_COST * 3);
8717 format %{ "encode_heap_oop_not_null $dst, $src" %}
8718 ins_encode %{
8719 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8720 %}
8721 ins_pipe(ialu_reg);
8722 %}
8723
8724 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8725 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8726 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8727 match(Set dst (DecodeN src));
8728 ins_cost(INSN_COST * 3);
8729 format %{ "decode_heap_oop $dst, $src" %}
8730 ins_encode %{
8731 Register s = $src$$Register;
8732 Register d = $dst$$Register;
8733 __ decode_heap_oop(d, s);
8734 %}
8735 ins_pipe(ialu_reg);
8736 %}
8737
8738 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8739 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8740 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8741 match(Set dst (DecodeN src));
8742 ins_cost(INSN_COST * 3);
8743 format %{ "decode_heap_oop_not_null $dst, $src" %}
8744 ins_encode %{
8745 Register s = $src$$Register;
8746 Register d = $dst$$Register;
8747 __ decode_heap_oop_not_null(d, s);
8748 %}
8749 ins_pipe(ialu_reg);
8750 %}
8751
8752 // n.b. AArch64 implementations of encode_klass_not_null and
8753 // decode_klass_not_null do not modify the flags register so, unlike
8754 // Intel, we don't kill CR as a side effect here
8755
8756 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8757 match(Set dst (EncodePKlass src));
8758
8759 ins_cost(INSN_COST * 3);
8760 format %{ "encode_klass_not_null $dst,$src" %}
8761
8762 ins_encode %{
8763 Register src_reg = as_Register($src$$reg);
8764 Register dst_reg = as_Register($dst$$reg);
8765 __ encode_klass_not_null(dst_reg, src_reg);
8766 %}
8767
8768 ins_pipe(ialu_reg);
8769 %}
8770
8771 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8772 match(Set dst (DecodeNKlass src));
8773
8774 ins_cost(INSN_COST * 3);
8775 format %{ "decode_klass_not_null $dst,$src" %}
8776
8777 ins_encode %{
8778 Register src_reg = as_Register($src$$reg);
8779 Register dst_reg = as_Register($dst$$reg);
8780 if (dst_reg != src_reg) {
8781 __ decode_klass_not_null(dst_reg, src_reg);
8782 } else {
8783 __ decode_klass_not_null(dst_reg);
8784 }
8785 %}
8786
8787 ins_pipe(ialu_reg);
8788 %}
8789
8790 instruct checkCastPP(iRegPNoSp dst)
8791 %{
8792 match(Set dst (CheckCastPP dst));
8793
8794 size(0);
8795 format %{ "# checkcastPP of $dst" %}
8796 ins_encode(/* empty encoding */);
8797 ins_pipe(pipe_class_empty);
8798 %}
8799
8800 instruct castPP(iRegPNoSp dst)
8801 %{
8802 match(Set dst (CastPP dst));
8803
8804 size(0);
8805 format %{ "# castPP of $dst" %}
8806 ins_encode(/* empty encoding */);
8807 ins_pipe(pipe_class_empty);
8808 %}
8809
8810 instruct castII(iRegI dst)
8811 %{
8812 match(Set dst (CastII dst));
8813
8814 size(0);
8815 format %{ "# castII of $dst" %}
8816 ins_encode(/* empty encoding */);
8817 ins_cost(0);
8818 ins_pipe(pipe_class_empty);
8819 %}
8820
8821 instruct castLL(iRegL dst)
8822 %{
8823 match(Set dst (CastLL dst));
8824
8825 size(0);
8826 format %{ "# castLL of $dst" %}
8827 ins_encode(/* empty encoding */);
8828 ins_cost(0);
8829 ins_pipe(pipe_class_empty);
8830 %}
8831
8832 instruct castFF(vRegF dst)
8833 %{
8834 match(Set dst (CastFF dst));
8835
8836 size(0);
8837 format %{ "# castFF of $dst" %}
8838 ins_encode(/* empty encoding */);
8839 ins_cost(0);
8840 ins_pipe(pipe_class_empty);
8841 %}
8842
8843 instruct castDD(vRegD dst)
8844 %{
8845 match(Set dst (CastDD dst));
8846
8847 size(0);
8848 format %{ "# castDD of $dst" %}
8849 ins_encode(/* empty encoding */);
8850 ins_cost(0);
8851 ins_pipe(pipe_class_empty);
8852 %}
8853
8854 instruct castVV(vReg dst)
8855 %{
8856 match(Set dst (CastVV dst));
8857
8858 size(0);
8859 format %{ "# castVV of $dst" %}
8860 ins_encode(/* empty encoding */);
8861 ins_cost(0);
8862 ins_pipe(pipe_class_empty);
8863 %}
8864
8865 instruct castVVMask(pRegGov dst)
8866 %{
8867 match(Set dst (CastVV dst));
8868
8869 size(0);
8870 format %{ "# castVV of $dst" %}
8871 ins_encode(/* empty encoding */);
8872 ins_cost(0);
8873 ins_pipe(pipe_class_empty);
8874 %}
8875
8876 // ============================================================================
8877 // Atomic operation instructions
8878 //
8879
8880 // standard CompareAndSwapX when we are using barriers
8881 // these have higher priority than the rules selected by a predicate
8882
8883 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8884 // can't match them
8885
8886 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8887
8888 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8889 ins_cost(2 * VOLATILE_REF_COST);
8890
8891 effect(KILL cr);
8892
8893 format %{
8894 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8895 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8896 %}
8897
8898 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8899 aarch64_enc_cset_eq(res));
8900
8901 ins_pipe(pipe_slow);
8902 %}
8903
8904 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8905
8906 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8907 ins_cost(2 * VOLATILE_REF_COST);
8908
8909 effect(KILL cr);
8910
8911 format %{
8912 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8913 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8914 %}
8915
8916 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8917 aarch64_enc_cset_eq(res));
8918
8919 ins_pipe(pipe_slow);
8920 %}
8921
8922 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8923
8924 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8925 ins_cost(2 * VOLATILE_REF_COST);
8926
8927 effect(KILL cr);
8928
8929 format %{
8930 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8931 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8932 %}
8933
8934 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8935 aarch64_enc_cset_eq(res));
8936
8937 ins_pipe(pipe_slow);
8938 %}
8939
8940 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8941
8942 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8943 ins_cost(2 * VOLATILE_REF_COST);
8944
8945 effect(KILL cr);
8946
8947 format %{
8948 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8949 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8950 %}
8951
8952 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8953 aarch64_enc_cset_eq(res));
8954
8955 ins_pipe(pipe_slow);
8956 %}
8957
8958 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8959
8960 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8961 predicate(n->as_LoadStore()->barrier_data() == 0);
8962 ins_cost(2 * VOLATILE_REF_COST);
8963
8964 effect(KILL cr);
8965
8966 format %{
8967 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8968 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8969 %}
8970
8971 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8972 aarch64_enc_cset_eq(res));
8973
8974 ins_pipe(pipe_slow);
8975 %}
8976
8977 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8978
8979 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8980 ins_cost(2 * VOLATILE_REF_COST);
8981
8982 effect(KILL cr);
8983
8984 format %{
8985 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8986 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8987 %}
8988
8989 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8990 aarch64_enc_cset_eq(res));
8991
8992 ins_pipe(pipe_slow);
8993 %}
8994
8995 // alternative CompareAndSwapX when we are eliding barriers
8996
8997 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8998
8999 predicate(needs_acquiring_load_exclusive(n));
9000 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
9001 ins_cost(VOLATILE_REF_COST);
9002
9003 effect(KILL cr);
9004
9005 format %{
9006 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9007 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9008 %}
9009
9010 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
9011 aarch64_enc_cset_eq(res));
9012
9013 ins_pipe(pipe_slow);
9014 %}
9015
9016 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9017
9018 predicate(needs_acquiring_load_exclusive(n));
9019 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
9020 ins_cost(VOLATILE_REF_COST);
9021
9022 effect(KILL cr);
9023
9024 format %{
9025 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9026 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9027 %}
9028
9029 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
9030 aarch64_enc_cset_eq(res));
9031
9032 ins_pipe(pipe_slow);
9033 %}
9034
9035 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9036
9037 predicate(needs_acquiring_load_exclusive(n));
9038 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9039 ins_cost(VOLATILE_REF_COST);
9040
9041 effect(KILL cr);
9042
9043 format %{
9044 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9045 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9046 %}
9047
9048 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9049 aarch64_enc_cset_eq(res));
9050
9051 ins_pipe(pipe_slow);
9052 %}
9053
9054 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9055
9056 predicate(needs_acquiring_load_exclusive(n));
9057 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9058 ins_cost(VOLATILE_REF_COST);
9059
9060 effect(KILL cr);
9061
9062 format %{
9063 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9064 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9065 %}
9066
9067 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9068 aarch64_enc_cset_eq(res));
9069
9070 ins_pipe(pipe_slow);
9071 %}
9072
9073 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9074
9075 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9076 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9077 ins_cost(VOLATILE_REF_COST);
9078
9079 effect(KILL cr);
9080
9081 format %{
9082 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9083 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9084 %}
9085
9086 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9087 aarch64_enc_cset_eq(res));
9088
9089 ins_pipe(pipe_slow);
9090 %}
9091
9092 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9093
9094 predicate(needs_acquiring_load_exclusive(n));
9095 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9096 ins_cost(VOLATILE_REF_COST);
9097
9098 effect(KILL cr);
9099
9100 format %{
9101 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9102 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9103 %}
9104
9105 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9106 aarch64_enc_cset_eq(res));
9107
9108 ins_pipe(pipe_slow);
9109 %}
9110
9111
9112 // ---------------------------------------------------------------------
9113
9114 // BEGIN This section of the file is automatically generated. Do not edit --------------
9115
9116 // Sundry CAS operations. Note that release is always true,
9117 // regardless of the memory ordering of the CAS. This is because we
9118 // need the volatile case to be sequentially consistent but there is
9119 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
9120 // can't check the type of memory ordering here, so we always emit a
9121 // STLXR.
9122
9123 // This section is generated from cas.m4
9124
9125
9126 // This pattern is generated automatically from cas.m4.
9127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9128 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9129 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9130 ins_cost(2 * VOLATILE_REF_COST);
9131 effect(TEMP_DEF res, KILL cr);
9132 format %{
9133 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9134 %}
9135 ins_encode %{
9136 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9137 Assembler::byte, /*acquire*/ false, /*release*/ true,
9138 /*weak*/ false, $res$$Register);
9139 __ sxtbw($res$$Register, $res$$Register);
9140 %}
9141 ins_pipe(pipe_slow);
9142 %}
9143
9144 // This pattern is generated automatically from cas.m4.
9145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9146 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9147 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9148 ins_cost(2 * VOLATILE_REF_COST);
9149 effect(TEMP_DEF res, KILL cr);
9150 format %{
9151 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9152 %}
9153 ins_encode %{
9154 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9155 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9156 /*weak*/ false, $res$$Register);
9157 __ sxthw($res$$Register, $res$$Register);
9158 %}
9159 ins_pipe(pipe_slow);
9160 %}
9161
9162 // This pattern is generated automatically from cas.m4.
9163 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9164 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9165 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9166 ins_cost(2 * VOLATILE_REF_COST);
9167 effect(TEMP_DEF res, KILL cr);
9168 format %{
9169 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9170 %}
9171 ins_encode %{
9172 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9173 Assembler::word, /*acquire*/ false, /*release*/ true,
9174 /*weak*/ false, $res$$Register);
9175 %}
9176 ins_pipe(pipe_slow);
9177 %}
9178
9179 // This pattern is generated automatically from cas.m4.
9180 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9181 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9182 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9183 ins_cost(2 * VOLATILE_REF_COST);
9184 effect(TEMP_DEF res, KILL cr);
9185 format %{
9186 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9187 %}
9188 ins_encode %{
9189 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9190 Assembler::xword, /*acquire*/ false, /*release*/ true,
9191 /*weak*/ false, $res$$Register);
9192 %}
9193 ins_pipe(pipe_slow);
9194 %}
9195
9196 // This pattern is generated automatically from cas.m4.
9197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9198 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9199 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9200 ins_cost(2 * VOLATILE_REF_COST);
9201 effect(TEMP_DEF res, KILL cr);
9202 format %{
9203 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9204 %}
9205 ins_encode %{
9206 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9207 Assembler::word, /*acquire*/ false, /*release*/ true,
9208 /*weak*/ false, $res$$Register);
9209 %}
9210 ins_pipe(pipe_slow);
9211 %}
9212
9213 // This pattern is generated automatically from cas.m4.
9214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9215 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9216 predicate(n->as_LoadStore()->barrier_data() == 0);
9217 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9218 ins_cost(2 * VOLATILE_REF_COST);
9219 effect(TEMP_DEF res, KILL cr);
9220 format %{
9221 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9222 %}
9223 ins_encode %{
9224 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9225 Assembler::xword, /*acquire*/ false, /*release*/ true,
9226 /*weak*/ false, $res$$Register);
9227 %}
9228 ins_pipe(pipe_slow);
9229 %}
9230
9231 // This pattern is generated automatically from cas.m4.
9232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9233 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9234 predicate(needs_acquiring_load_exclusive(n));
9235 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9236 ins_cost(VOLATILE_REF_COST);
9237 effect(TEMP_DEF res, KILL cr);
9238 format %{
9239 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9240 %}
9241 ins_encode %{
9242 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9243 Assembler::byte, /*acquire*/ true, /*release*/ true,
9244 /*weak*/ false, $res$$Register);
9245 __ sxtbw($res$$Register, $res$$Register);
9246 %}
9247 ins_pipe(pipe_slow);
9248 %}
9249
9250 // This pattern is generated automatically from cas.m4.
9251 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9252 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9253 predicate(needs_acquiring_load_exclusive(n));
9254 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9255 ins_cost(VOLATILE_REF_COST);
9256 effect(TEMP_DEF res, KILL cr);
9257 format %{
9258 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9259 %}
9260 ins_encode %{
9261 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9262 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9263 /*weak*/ false, $res$$Register);
9264 __ sxthw($res$$Register, $res$$Register);
9265 %}
9266 ins_pipe(pipe_slow);
9267 %}
9268
9269 // This pattern is generated automatically from cas.m4.
9270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9271 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9272 predicate(needs_acquiring_load_exclusive(n));
9273 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9274 ins_cost(VOLATILE_REF_COST);
9275 effect(TEMP_DEF res, KILL cr);
9276 format %{
9277 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9278 %}
9279 ins_encode %{
9280 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9281 Assembler::word, /*acquire*/ true, /*release*/ true,
9282 /*weak*/ false, $res$$Register);
9283 %}
9284 ins_pipe(pipe_slow);
9285 %}
9286
9287 // This pattern is generated automatically from cas.m4.
9288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9289 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9290 predicate(needs_acquiring_load_exclusive(n));
9291 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9292 ins_cost(VOLATILE_REF_COST);
9293 effect(TEMP_DEF res, KILL cr);
9294 format %{
9295 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9296 %}
9297 ins_encode %{
9298 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9299 Assembler::xword, /*acquire*/ true, /*release*/ true,
9300 /*weak*/ false, $res$$Register);
9301 %}
9302 ins_pipe(pipe_slow);
9303 %}
9304
9305 // This pattern is generated automatically from cas.m4.
9306 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9307 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9308 predicate(needs_acquiring_load_exclusive(n));
9309 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9310 ins_cost(VOLATILE_REF_COST);
9311 effect(TEMP_DEF res, KILL cr);
9312 format %{
9313 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9314 %}
9315 ins_encode %{
9316 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9317 Assembler::word, /*acquire*/ true, /*release*/ true,
9318 /*weak*/ false, $res$$Register);
9319 %}
9320 ins_pipe(pipe_slow);
9321 %}
9322
9323 // This pattern is generated automatically from cas.m4.
9324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9325 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9326 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9327 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9328 ins_cost(VOLATILE_REF_COST);
9329 effect(TEMP_DEF res, KILL cr);
9330 format %{
9331 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9332 %}
9333 ins_encode %{
9334 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9335 Assembler::xword, /*acquire*/ true, /*release*/ true,
9336 /*weak*/ false, $res$$Register);
9337 %}
9338 ins_pipe(pipe_slow);
9339 %}
9340
9341 // This pattern is generated automatically from cas.m4.
9342 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9343 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9344 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9345 ins_cost(2 * VOLATILE_REF_COST);
9346 effect(KILL cr);
9347 format %{
9348 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9349 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9350 %}
9351 ins_encode %{
9352 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9353 Assembler::byte, /*acquire*/ false, /*release*/ true,
9354 /*weak*/ true, noreg);
9355 __ csetw($res$$Register, Assembler::EQ);
9356 %}
9357 ins_pipe(pipe_slow);
9358 %}
9359
9360 // This pattern is generated automatically from cas.m4.
9361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9362 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9363 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9364 ins_cost(2 * VOLATILE_REF_COST);
9365 effect(KILL cr);
9366 format %{
9367 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9368 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9369 %}
9370 ins_encode %{
9371 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9372 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9373 /*weak*/ true, noreg);
9374 __ csetw($res$$Register, Assembler::EQ);
9375 %}
9376 ins_pipe(pipe_slow);
9377 %}
9378
9379 // This pattern is generated automatically from cas.m4.
9380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9381 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9382 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9383 ins_cost(2 * VOLATILE_REF_COST);
9384 effect(KILL cr);
9385 format %{
9386 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9387 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9388 %}
9389 ins_encode %{
9390 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9391 Assembler::word, /*acquire*/ false, /*release*/ true,
9392 /*weak*/ true, noreg);
9393 __ csetw($res$$Register, Assembler::EQ);
9394 %}
9395 ins_pipe(pipe_slow);
9396 %}
9397
9398 // This pattern is generated automatically from cas.m4.
9399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9400 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9401 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9402 ins_cost(2 * VOLATILE_REF_COST);
9403 effect(KILL cr);
9404 format %{
9405 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9406 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9407 %}
9408 ins_encode %{
9409 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9410 Assembler::xword, /*acquire*/ false, /*release*/ true,
9411 /*weak*/ true, noreg);
9412 __ csetw($res$$Register, Assembler::EQ);
9413 %}
9414 ins_pipe(pipe_slow);
9415 %}
9416
9417 // This pattern is generated automatically from cas.m4.
9418 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9419 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9420 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9421 ins_cost(2 * VOLATILE_REF_COST);
9422 effect(KILL cr);
9423 format %{
9424 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9425 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9426 %}
9427 ins_encode %{
9428 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9429 Assembler::word, /*acquire*/ false, /*release*/ true,
9430 /*weak*/ true, noreg);
9431 __ csetw($res$$Register, Assembler::EQ);
9432 %}
9433 ins_pipe(pipe_slow);
9434 %}
9435
9436 // This pattern is generated automatically from cas.m4.
9437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9438 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9439 predicate(n->as_LoadStore()->barrier_data() == 0);
9440 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9441 ins_cost(2 * VOLATILE_REF_COST);
9442 effect(KILL cr);
9443 format %{
9444 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9445 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9446 %}
9447 ins_encode %{
9448 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9449 Assembler::xword, /*acquire*/ false, /*release*/ true,
9450 /*weak*/ true, noreg);
9451 __ csetw($res$$Register, Assembler::EQ);
9452 %}
9453 ins_pipe(pipe_slow);
9454 %}
9455
9456 // This pattern is generated automatically from cas.m4.
9457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9458 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9459 predicate(needs_acquiring_load_exclusive(n));
9460 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9461 ins_cost(VOLATILE_REF_COST);
9462 effect(KILL cr);
9463 format %{
9464 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9465 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9466 %}
9467 ins_encode %{
9468 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9469 Assembler::byte, /*acquire*/ true, /*release*/ true,
9470 /*weak*/ true, noreg);
9471 __ csetw($res$$Register, Assembler::EQ);
9472 %}
9473 ins_pipe(pipe_slow);
9474 %}
9475
9476 // This pattern is generated automatically from cas.m4.
9477 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9478 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9479 predicate(needs_acquiring_load_exclusive(n));
9480 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9481 ins_cost(VOLATILE_REF_COST);
9482 effect(KILL cr);
9483 format %{
9484 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9485 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9486 %}
9487 ins_encode %{
9488 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9489 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9490 /*weak*/ true, noreg);
9491 __ csetw($res$$Register, Assembler::EQ);
9492 %}
9493 ins_pipe(pipe_slow);
9494 %}
9495
9496 // This pattern is generated automatically from cas.m4.
9497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9498 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9499 predicate(needs_acquiring_load_exclusive(n));
9500 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9501 ins_cost(VOLATILE_REF_COST);
9502 effect(KILL cr);
9503 format %{
9504 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9505 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9506 %}
9507 ins_encode %{
9508 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9509 Assembler::word, /*acquire*/ true, /*release*/ true,
9510 /*weak*/ true, noreg);
9511 __ csetw($res$$Register, Assembler::EQ);
9512 %}
9513 ins_pipe(pipe_slow);
9514 %}
9515
9516 // This pattern is generated automatically from cas.m4.
9517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9518 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9519 predicate(needs_acquiring_load_exclusive(n));
9520 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9521 ins_cost(VOLATILE_REF_COST);
9522 effect(KILL cr);
9523 format %{
9524 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9525 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9526 %}
9527 ins_encode %{
9528 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9529 Assembler::xword, /*acquire*/ true, /*release*/ true,
9530 /*weak*/ true, noreg);
9531 __ csetw($res$$Register, Assembler::EQ);
9532 %}
9533 ins_pipe(pipe_slow);
9534 %}
9535
9536 // This pattern is generated automatically from cas.m4.
9537 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9538 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9539 predicate(needs_acquiring_load_exclusive(n));
9540 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9541 ins_cost(VOLATILE_REF_COST);
9542 effect(KILL cr);
9543 format %{
9544 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9545 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9546 %}
9547 ins_encode %{
9548 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9549 Assembler::word, /*acquire*/ true, /*release*/ true,
9550 /*weak*/ true, noreg);
9551 __ csetw($res$$Register, Assembler::EQ);
9552 %}
9553 ins_pipe(pipe_slow);
9554 %}
9555
9556 // This pattern is generated automatically from cas.m4.
9557 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9558 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9559 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9560 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9561 ins_cost(VOLATILE_REF_COST);
9562 effect(KILL cr);
9563 format %{
9564 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9565 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9566 %}
9567 ins_encode %{
9568 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9569 Assembler::xword, /*acquire*/ true, /*release*/ true,
9570 /*weak*/ true, noreg);
9571 __ csetw($res$$Register, Assembler::EQ);
9572 %}
9573 ins_pipe(pipe_slow);
9574 %}
9575
9576 // END This section of the file is automatically generated. Do not edit --------------
9577 // ---------------------------------------------------------------------
9578
9579 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9580 match(Set prev (GetAndSetI mem newv));
9581 ins_cost(2 * VOLATILE_REF_COST);
9582 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9583 ins_encode %{
9584 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9585 %}
9586 ins_pipe(pipe_serial);
9587 %}
9588
9589 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9590 match(Set prev (GetAndSetL mem newv));
9591 ins_cost(2 * VOLATILE_REF_COST);
9592 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9593 ins_encode %{
9594 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9595 %}
9596 ins_pipe(pipe_serial);
9597 %}
9598
9599 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9600 match(Set prev (GetAndSetN mem newv));
9601 ins_cost(2 * VOLATILE_REF_COST);
9602 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9603 ins_encode %{
9604 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9605 %}
9606 ins_pipe(pipe_serial);
9607 %}
9608
9609 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9610 predicate(n->as_LoadStore()->barrier_data() == 0);
9611 match(Set prev (GetAndSetP mem newv));
9612 ins_cost(2 * VOLATILE_REF_COST);
9613 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9614 ins_encode %{
9615 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9616 %}
9617 ins_pipe(pipe_serial);
9618 %}
9619
9620 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9621 predicate(needs_acquiring_load_exclusive(n));
9622 match(Set prev (GetAndSetI mem newv));
9623 ins_cost(VOLATILE_REF_COST);
9624 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9625 ins_encode %{
9626 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9627 %}
9628 ins_pipe(pipe_serial);
9629 %}
9630
9631 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9632 predicate(needs_acquiring_load_exclusive(n));
9633 match(Set prev (GetAndSetL mem newv));
9634 ins_cost(VOLATILE_REF_COST);
9635 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9636 ins_encode %{
9637 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9638 %}
9639 ins_pipe(pipe_serial);
9640 %}
9641
9642 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9643 predicate(needs_acquiring_load_exclusive(n));
9644 match(Set prev (GetAndSetN mem newv));
9645 ins_cost(VOLATILE_REF_COST);
9646 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9647 ins_encode %{
9648 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9649 %}
9650 ins_pipe(pipe_serial);
9651 %}
9652
9653 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9654 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9655 match(Set prev (GetAndSetP mem newv));
9656 ins_cost(VOLATILE_REF_COST);
9657 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9658 ins_encode %{
9659 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9660 %}
9661 ins_pipe(pipe_serial);
9662 %}
9663
9664
9665 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9666 match(Set newval (GetAndAddL mem incr));
9667 ins_cost(2 * VOLATILE_REF_COST + 1);
9668 format %{ "get_and_addL $newval, [$mem], $incr" %}
9669 ins_encode %{
9670 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9671 %}
9672 ins_pipe(pipe_serial);
9673 %}
9674
9675 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9676 predicate(n->as_LoadStore()->result_not_used());
9677 match(Set dummy (GetAndAddL mem incr));
9678 ins_cost(2 * VOLATILE_REF_COST);
9679 format %{ "get_and_addL [$mem], $incr" %}
9680 ins_encode %{
9681 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9682 %}
9683 ins_pipe(pipe_serial);
9684 %}
9685
9686 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9687 match(Set newval (GetAndAddL mem incr));
9688 ins_cost(2 * VOLATILE_REF_COST + 1);
9689 format %{ "get_and_addL $newval, [$mem], $incr" %}
9690 ins_encode %{
9691 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9692 %}
9693 ins_pipe(pipe_serial);
9694 %}
9695
9696 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9697 predicate(n->as_LoadStore()->result_not_used());
9698 match(Set dummy (GetAndAddL mem incr));
9699 ins_cost(2 * VOLATILE_REF_COST);
9700 format %{ "get_and_addL [$mem], $incr" %}
9701 ins_encode %{
9702 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9703 %}
9704 ins_pipe(pipe_serial);
9705 %}
9706
9707 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9708 match(Set newval (GetAndAddI mem incr));
9709 ins_cost(2 * VOLATILE_REF_COST + 1);
9710 format %{ "get_and_addI $newval, [$mem], $incr" %}
9711 ins_encode %{
9712 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9713 %}
9714 ins_pipe(pipe_serial);
9715 %}
9716
9717 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9718 predicate(n->as_LoadStore()->result_not_used());
9719 match(Set dummy (GetAndAddI mem incr));
9720 ins_cost(2 * VOLATILE_REF_COST);
9721 format %{ "get_and_addI [$mem], $incr" %}
9722 ins_encode %{
9723 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9724 %}
9725 ins_pipe(pipe_serial);
9726 %}
9727
9728 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9729 match(Set newval (GetAndAddI mem incr));
9730 ins_cost(2 * VOLATILE_REF_COST + 1);
9731 format %{ "get_and_addI $newval, [$mem], $incr" %}
9732 ins_encode %{
9733 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9734 %}
9735 ins_pipe(pipe_serial);
9736 %}
9737
9738 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9739 predicate(n->as_LoadStore()->result_not_used());
9740 match(Set dummy (GetAndAddI mem incr));
9741 ins_cost(2 * VOLATILE_REF_COST);
9742 format %{ "get_and_addI [$mem], $incr" %}
9743 ins_encode %{
9744 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9745 %}
9746 ins_pipe(pipe_serial);
9747 %}
9748
9749 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9750 predicate(needs_acquiring_load_exclusive(n));
9751 match(Set newval (GetAndAddL mem incr));
9752 ins_cost(VOLATILE_REF_COST + 1);
9753 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9754 ins_encode %{
9755 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9756 %}
9757 ins_pipe(pipe_serial);
9758 %}
9759
9760 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9761 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9762 match(Set dummy (GetAndAddL mem incr));
9763 ins_cost(VOLATILE_REF_COST);
9764 format %{ "get_and_addL_acq [$mem], $incr" %}
9765 ins_encode %{
9766 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9767 %}
9768 ins_pipe(pipe_serial);
9769 %}
9770
9771 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9772 predicate(needs_acquiring_load_exclusive(n));
9773 match(Set newval (GetAndAddL mem incr));
9774 ins_cost(VOLATILE_REF_COST + 1);
9775 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9776 ins_encode %{
9777 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9778 %}
9779 ins_pipe(pipe_serial);
9780 %}
9781
9782 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9783 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9784 match(Set dummy (GetAndAddL mem incr));
9785 ins_cost(VOLATILE_REF_COST);
9786 format %{ "get_and_addL_acq [$mem], $incr" %}
9787 ins_encode %{
9788 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9789 %}
9790 ins_pipe(pipe_serial);
9791 %}
9792
9793 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9794 predicate(needs_acquiring_load_exclusive(n));
9795 match(Set newval (GetAndAddI mem incr));
9796 ins_cost(VOLATILE_REF_COST + 1);
9797 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9798 ins_encode %{
9799 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9800 %}
9801 ins_pipe(pipe_serial);
9802 %}
9803
9804 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9805 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9806 match(Set dummy (GetAndAddI mem incr));
9807 ins_cost(VOLATILE_REF_COST);
9808 format %{ "get_and_addI_acq [$mem], $incr" %}
9809 ins_encode %{
9810 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9811 %}
9812 ins_pipe(pipe_serial);
9813 %}
9814
9815 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9816 predicate(needs_acquiring_load_exclusive(n));
9817 match(Set newval (GetAndAddI mem incr));
9818 ins_cost(VOLATILE_REF_COST + 1);
9819 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9820 ins_encode %{
9821 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9822 %}
9823 ins_pipe(pipe_serial);
9824 %}
9825
9826 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9827 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9828 match(Set dummy (GetAndAddI mem incr));
9829 ins_cost(VOLATILE_REF_COST);
9830 format %{ "get_and_addI_acq [$mem], $incr" %}
9831 ins_encode %{
9832 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9833 %}
9834 ins_pipe(pipe_serial);
9835 %}
9836
9837 // Manifest a CmpU result in an integer register.
9838 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9839 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9840 %{
9841 match(Set dst (CmpU3 src1 src2));
9842 effect(KILL flags);
9843
9844 ins_cost(INSN_COST * 3);
9845 format %{
9846 "cmpw $src1, $src2\n\t"
9847 "csetw $dst, ne\n\t"
9848 "cnegw $dst, lo\t# CmpU3(reg)"
9849 %}
9850 ins_encode %{
9851 __ cmpw($src1$$Register, $src2$$Register);
9852 __ csetw($dst$$Register, Assembler::NE);
9853 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9854 %}
9855
9856 ins_pipe(pipe_class_default);
9857 %}
9858
9859 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9860 %{
9861 match(Set dst (CmpU3 src1 src2));
9862 effect(KILL flags);
9863
9864 ins_cost(INSN_COST * 3);
9865 format %{
9866 "subsw zr, $src1, $src2\n\t"
9867 "csetw $dst, ne\n\t"
9868 "cnegw $dst, lo\t# CmpU3(imm)"
9869 %}
9870 ins_encode %{
9871 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9872 __ csetw($dst$$Register, Assembler::NE);
9873 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9874 %}
9875
9876 ins_pipe(pipe_class_default);
9877 %}
9878
9879 // Manifest a CmpUL result in an integer register.
9880 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9881 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9882 %{
9883 match(Set dst (CmpUL3 src1 src2));
9884 effect(KILL flags);
9885
9886 ins_cost(INSN_COST * 3);
9887 format %{
9888 "cmp $src1, $src2\n\t"
9889 "csetw $dst, ne\n\t"
9890 "cnegw $dst, lo\t# CmpUL3(reg)"
9891 %}
9892 ins_encode %{
9893 __ cmp($src1$$Register, $src2$$Register);
9894 __ csetw($dst$$Register, Assembler::NE);
9895 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9896 %}
9897
9898 ins_pipe(pipe_class_default);
9899 %}
9900
9901 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9902 %{
9903 match(Set dst (CmpUL3 src1 src2));
9904 effect(KILL flags);
9905
9906 ins_cost(INSN_COST * 3);
9907 format %{
9908 "subs zr, $src1, $src2\n\t"
9909 "csetw $dst, ne\n\t"
9910 "cnegw $dst, lo\t# CmpUL3(imm)"
9911 %}
9912 ins_encode %{
9913 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9914 __ csetw($dst$$Register, Assembler::NE);
9915 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9916 %}
9917
9918 ins_pipe(pipe_class_default);
9919 %}
9920
9921 // Manifest a CmpL result in an integer register.
9922 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9923 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9924 %{
9925 match(Set dst (CmpL3 src1 src2));
9926 effect(KILL flags);
9927
9928 ins_cost(INSN_COST * 3);
9929 format %{
9930 "cmp $src1, $src2\n\t"
9931 "csetw $dst, ne\n\t"
9932 "cnegw $dst, lt\t# CmpL3(reg)"
9933 %}
9934 ins_encode %{
9935 __ cmp($src1$$Register, $src2$$Register);
9936 __ csetw($dst$$Register, Assembler::NE);
9937 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9938 %}
9939
9940 ins_pipe(pipe_class_default);
9941 %}
9942
9943 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9944 %{
9945 match(Set dst (CmpL3 src1 src2));
9946 effect(KILL flags);
9947
9948 ins_cost(INSN_COST * 3);
9949 format %{
9950 "subs zr, $src1, $src2\n\t"
9951 "csetw $dst, ne\n\t"
9952 "cnegw $dst, lt\t# CmpL3(imm)"
9953 %}
9954 ins_encode %{
9955 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9956 __ csetw($dst$$Register, Assembler::NE);
9957 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9958 %}
9959
9960 ins_pipe(pipe_class_default);
9961 %}
9962
9963 // ============================================================================
9964 // Conditional Move Instructions
9965
9966 // n.b. we have identical rules for both a signed compare op (cmpOp)
9967 // and an unsigned compare op (cmpOpU). it would be nice if we could
9968 // define an op class which merged both inputs and use it to type the
9969 // argument to a single rule. unfortunatelyt his fails because the
9970 // opclass does not live up to the COND_INTER interface of its
9971 // component operands. When the generic code tries to negate the
9972 // operand it ends up running the generci Machoper::negate method
9973 // which throws a ShouldNotHappen. So, we have to provide two flavours
9974 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9975
9976 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9977 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9978
9979 ins_cost(INSN_COST * 2);
9980 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9981
9982 ins_encode %{
9983 __ cselw(as_Register($dst$$reg),
9984 as_Register($src2$$reg),
9985 as_Register($src1$$reg),
9986 (Assembler::Condition)$cmp$$cmpcode);
9987 %}
9988
9989 ins_pipe(icond_reg_reg);
9990 %}
9991
9992 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9993 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9994
9995 ins_cost(INSN_COST * 2);
9996 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9997
9998 ins_encode %{
9999 __ cselw(as_Register($dst$$reg),
10000 as_Register($src2$$reg),
10001 as_Register($src1$$reg),
10002 (Assembler::Condition)$cmp$$cmpcode);
10003 %}
10004
10005 ins_pipe(icond_reg_reg);
10006 %}
10007
10008 // special cases where one arg is zero
10009
10010 // n.b. this is selected in preference to the rule above because it
10011 // avoids loading constant 0 into a source register
10012
10013 // TODO
10014 // we ought only to be able to cull one of these variants as the ideal
10015 // transforms ought always to order the zero consistently (to left/right?)
10016
10017 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10018 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10019
10020 ins_cost(INSN_COST * 2);
10021 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
10022
10023 ins_encode %{
10024 __ cselw(as_Register($dst$$reg),
10025 as_Register($src$$reg),
10026 zr,
10027 (Assembler::Condition)$cmp$$cmpcode);
10028 %}
10029
10030 ins_pipe(icond_reg);
10031 %}
10032
10033 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10034 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10035
10036 ins_cost(INSN_COST * 2);
10037 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
10038
10039 ins_encode %{
10040 __ cselw(as_Register($dst$$reg),
10041 as_Register($src$$reg),
10042 zr,
10043 (Assembler::Condition)$cmp$$cmpcode);
10044 %}
10045
10046 ins_pipe(icond_reg);
10047 %}
10048
10049 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10050 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10051
10052 ins_cost(INSN_COST * 2);
10053 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
10054
10055 ins_encode %{
10056 __ cselw(as_Register($dst$$reg),
10057 zr,
10058 as_Register($src$$reg),
10059 (Assembler::Condition)$cmp$$cmpcode);
10060 %}
10061
10062 ins_pipe(icond_reg);
10063 %}
10064
10065 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10066 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10067
10068 ins_cost(INSN_COST * 2);
10069 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
10070
10071 ins_encode %{
10072 __ cselw(as_Register($dst$$reg),
10073 zr,
10074 as_Register($src$$reg),
10075 (Assembler::Condition)$cmp$$cmpcode);
10076 %}
10077
10078 ins_pipe(icond_reg);
10079 %}
10080
10081 // special case for creating a boolean 0 or 1
10082
10083 // n.b. this is selected in preference to the rule above because it
10084 // avoids loading constants 0 and 1 into a source register
10085
10086 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10087 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10088
10089 ins_cost(INSN_COST * 2);
10090 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
10091
10092 ins_encode %{
10093 // equivalently
10094 // cset(as_Register($dst$$reg),
10095 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10096 __ csincw(as_Register($dst$$reg),
10097 zr,
10098 zr,
10099 (Assembler::Condition)$cmp$$cmpcode);
10100 %}
10101
10102 ins_pipe(icond_none);
10103 %}
10104
10105 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10106 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10107
10108 ins_cost(INSN_COST * 2);
10109 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
10110
10111 ins_encode %{
10112 // equivalently
10113 // cset(as_Register($dst$$reg),
10114 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10115 __ csincw(as_Register($dst$$reg),
10116 zr,
10117 zr,
10118 (Assembler::Condition)$cmp$$cmpcode);
10119 %}
10120
10121 ins_pipe(icond_none);
10122 %}
10123
10124 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10125 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10126
10127 ins_cost(INSN_COST * 2);
10128 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
10129
10130 ins_encode %{
10131 __ csel(as_Register($dst$$reg),
10132 as_Register($src2$$reg),
10133 as_Register($src1$$reg),
10134 (Assembler::Condition)$cmp$$cmpcode);
10135 %}
10136
10137 ins_pipe(icond_reg_reg);
10138 %}
10139
10140 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10141 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10142
10143 ins_cost(INSN_COST * 2);
10144 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
10145
10146 ins_encode %{
10147 __ csel(as_Register($dst$$reg),
10148 as_Register($src2$$reg),
10149 as_Register($src1$$reg),
10150 (Assembler::Condition)$cmp$$cmpcode);
10151 %}
10152
10153 ins_pipe(icond_reg_reg);
10154 %}
10155
10156 // special cases where one arg is zero
10157
10158 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10159 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10160
10161 ins_cost(INSN_COST * 2);
10162 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
10163
10164 ins_encode %{
10165 __ csel(as_Register($dst$$reg),
10166 zr,
10167 as_Register($src$$reg),
10168 (Assembler::Condition)$cmp$$cmpcode);
10169 %}
10170
10171 ins_pipe(icond_reg);
10172 %}
10173
10174 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10175 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10176
10177 ins_cost(INSN_COST * 2);
10178 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
10179
10180 ins_encode %{
10181 __ csel(as_Register($dst$$reg),
10182 zr,
10183 as_Register($src$$reg),
10184 (Assembler::Condition)$cmp$$cmpcode);
10185 %}
10186
10187 ins_pipe(icond_reg);
10188 %}
10189
10190 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10191 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10192
10193 ins_cost(INSN_COST * 2);
10194 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
10195
10196 ins_encode %{
10197 __ csel(as_Register($dst$$reg),
10198 as_Register($src$$reg),
10199 zr,
10200 (Assembler::Condition)$cmp$$cmpcode);
10201 %}
10202
10203 ins_pipe(icond_reg);
10204 %}
10205
10206 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10207 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10208
10209 ins_cost(INSN_COST * 2);
10210 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
10211
10212 ins_encode %{
10213 __ csel(as_Register($dst$$reg),
10214 as_Register($src$$reg),
10215 zr,
10216 (Assembler::Condition)$cmp$$cmpcode);
10217 %}
10218
10219 ins_pipe(icond_reg);
10220 %}
10221
10222 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10223 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10224
10225 ins_cost(INSN_COST * 2);
10226 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
10227
10228 ins_encode %{
10229 __ csel(as_Register($dst$$reg),
10230 as_Register($src2$$reg),
10231 as_Register($src1$$reg),
10232 (Assembler::Condition)$cmp$$cmpcode);
10233 %}
10234
10235 ins_pipe(icond_reg_reg);
10236 %}
10237
10238 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10239 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10240
10241 ins_cost(INSN_COST * 2);
10242 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
10243
10244 ins_encode %{
10245 __ csel(as_Register($dst$$reg),
10246 as_Register($src2$$reg),
10247 as_Register($src1$$reg),
10248 (Assembler::Condition)$cmp$$cmpcode);
10249 %}
10250
10251 ins_pipe(icond_reg_reg);
10252 %}
10253
10254 // special cases where one arg is zero
10255
10256 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10257 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10258
10259 ins_cost(INSN_COST * 2);
10260 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
10261
10262 ins_encode %{
10263 __ csel(as_Register($dst$$reg),
10264 zr,
10265 as_Register($src$$reg),
10266 (Assembler::Condition)$cmp$$cmpcode);
10267 %}
10268
10269 ins_pipe(icond_reg);
10270 %}
10271
10272 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10273 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10274
10275 ins_cost(INSN_COST * 2);
10276 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
10277
10278 ins_encode %{
10279 __ csel(as_Register($dst$$reg),
10280 zr,
10281 as_Register($src$$reg),
10282 (Assembler::Condition)$cmp$$cmpcode);
10283 %}
10284
10285 ins_pipe(icond_reg);
10286 %}
10287
10288 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10289 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10290
10291 ins_cost(INSN_COST * 2);
10292 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
10293
10294 ins_encode %{
10295 __ csel(as_Register($dst$$reg),
10296 as_Register($src$$reg),
10297 zr,
10298 (Assembler::Condition)$cmp$$cmpcode);
10299 %}
10300
10301 ins_pipe(icond_reg);
10302 %}
10303
10304 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10305 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10306
10307 ins_cost(INSN_COST * 2);
10308 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
10309
10310 ins_encode %{
10311 __ csel(as_Register($dst$$reg),
10312 as_Register($src$$reg),
10313 zr,
10314 (Assembler::Condition)$cmp$$cmpcode);
10315 %}
10316
10317 ins_pipe(icond_reg);
10318 %}
10319
10320 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10321 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10322
10323 ins_cost(INSN_COST * 2);
10324 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10325
10326 ins_encode %{
10327 __ cselw(as_Register($dst$$reg),
10328 as_Register($src2$$reg),
10329 as_Register($src1$$reg),
10330 (Assembler::Condition)$cmp$$cmpcode);
10331 %}
10332
10333 ins_pipe(icond_reg_reg);
10334 %}
10335
10336 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10337 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10338
10339 ins_cost(INSN_COST * 2);
10340 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10341
10342 ins_encode %{
10343 __ cselw(as_Register($dst$$reg),
10344 as_Register($src2$$reg),
10345 as_Register($src1$$reg),
10346 (Assembler::Condition)$cmp$$cmpcode);
10347 %}
10348
10349 ins_pipe(icond_reg_reg);
10350 %}
10351
10352 // special cases where one arg is zero
10353
10354 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10355 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10356
10357 ins_cost(INSN_COST * 2);
10358 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
10359
10360 ins_encode %{
10361 __ cselw(as_Register($dst$$reg),
10362 zr,
10363 as_Register($src$$reg),
10364 (Assembler::Condition)$cmp$$cmpcode);
10365 %}
10366
10367 ins_pipe(icond_reg);
10368 %}
10369
10370 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10371 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10372
10373 ins_cost(INSN_COST * 2);
10374 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
10375
10376 ins_encode %{
10377 __ cselw(as_Register($dst$$reg),
10378 zr,
10379 as_Register($src$$reg),
10380 (Assembler::Condition)$cmp$$cmpcode);
10381 %}
10382
10383 ins_pipe(icond_reg);
10384 %}
10385
10386 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10387 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10388
10389 ins_cost(INSN_COST * 2);
10390 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
10391
10392 ins_encode %{
10393 __ cselw(as_Register($dst$$reg),
10394 as_Register($src$$reg),
10395 zr,
10396 (Assembler::Condition)$cmp$$cmpcode);
10397 %}
10398
10399 ins_pipe(icond_reg);
10400 %}
10401
10402 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10403 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10404
10405 ins_cost(INSN_COST * 2);
10406 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
10407
10408 ins_encode %{
10409 __ cselw(as_Register($dst$$reg),
10410 as_Register($src$$reg),
10411 zr,
10412 (Assembler::Condition)$cmp$$cmpcode);
10413 %}
10414
10415 ins_pipe(icond_reg);
10416 %}
10417
10418 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
10419 %{
10420 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10421
10422 ins_cost(INSN_COST * 3);
10423
10424 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10425 ins_encode %{
10426 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10427 __ fcsels(as_FloatRegister($dst$$reg),
10428 as_FloatRegister($src2$$reg),
10429 as_FloatRegister($src1$$reg),
10430 cond);
10431 %}
10432
10433 ins_pipe(fp_cond_reg_reg_s);
10434 %}
10435
10436 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
10437 %{
10438 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10439
10440 ins_cost(INSN_COST * 3);
10441
10442 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10443 ins_encode %{
10444 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10445 __ fcsels(as_FloatRegister($dst$$reg),
10446 as_FloatRegister($src2$$reg),
10447 as_FloatRegister($src1$$reg),
10448 cond);
10449 %}
10450
10451 ins_pipe(fp_cond_reg_reg_s);
10452 %}
10453
10454 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10455 %{
10456 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10457
10458 ins_cost(INSN_COST * 3);
10459
10460 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10461 ins_encode %{
10462 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10463 __ fcseld(as_FloatRegister($dst$$reg),
10464 as_FloatRegister($src2$$reg),
10465 as_FloatRegister($src1$$reg),
10466 cond);
10467 %}
10468
10469 ins_pipe(fp_cond_reg_reg_d);
10470 %}
10471
10472 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10473 %{
10474 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10475
10476 ins_cost(INSN_COST * 3);
10477
10478 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10479 ins_encode %{
10480 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10481 __ fcseld(as_FloatRegister($dst$$reg),
10482 as_FloatRegister($src2$$reg),
10483 as_FloatRegister($src1$$reg),
10484 cond);
10485 %}
10486
10487 ins_pipe(fp_cond_reg_reg_d);
10488 %}
10489
10490 // ============================================================================
10491 // Arithmetic Instructions
10492 //
10493
10494 // Integer Addition
10495
10496 // TODO
10497 // these currently employ operations which do not set CR and hence are
10498 // not flagged as killing CR but we would like to isolate the cases
10499 // where we want to set flags from those where we don't. need to work
10500 // out how to do that.
10501
10502 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10503 match(Set dst (AddI src1 src2));
10504
10505 ins_cost(INSN_COST);
10506 format %{ "addw $dst, $src1, $src2" %}
10507
10508 ins_encode %{
10509 __ addw(as_Register($dst$$reg),
10510 as_Register($src1$$reg),
10511 as_Register($src2$$reg));
10512 %}
10513
10514 ins_pipe(ialu_reg_reg);
10515 %}
10516
10517 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10518 match(Set dst (AddI src1 src2));
10519
10520 ins_cost(INSN_COST);
10521 format %{ "addw $dst, $src1, $src2" %}
10522
10523 // use opcode to indicate that this is an add not a sub
10524 opcode(0x0);
10525
10526 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10527
10528 ins_pipe(ialu_reg_imm);
10529 %}
10530
10531 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10532 match(Set dst (AddI (ConvL2I src1) src2));
10533
10534 ins_cost(INSN_COST);
10535 format %{ "addw $dst, $src1, $src2" %}
10536
10537 // use opcode to indicate that this is an add not a sub
10538 opcode(0x0);
10539
10540 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10541
10542 ins_pipe(ialu_reg_imm);
10543 %}
10544
10545 // Pointer Addition
10546 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10547 match(Set dst (AddP src1 src2));
10548
10549 ins_cost(INSN_COST);
10550 format %{ "add $dst, $src1, $src2\t# ptr" %}
10551
10552 ins_encode %{
10553 __ add(as_Register($dst$$reg),
10554 as_Register($src1$$reg),
10555 as_Register($src2$$reg));
10556 %}
10557
10558 ins_pipe(ialu_reg_reg);
10559 %}
10560
10561 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10562 match(Set dst (AddP src1 (ConvI2L src2)));
10563
10564 ins_cost(1.9 * INSN_COST);
10565 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10566
10567 ins_encode %{
10568 __ add(as_Register($dst$$reg),
10569 as_Register($src1$$reg),
10570 as_Register($src2$$reg), ext::sxtw);
10571 %}
10572
10573 ins_pipe(ialu_reg_reg);
10574 %}
10575
10576 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10577 match(Set dst (AddP src1 (LShiftL src2 scale)));
10578
10579 ins_cost(1.9 * INSN_COST);
10580 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10581
10582 ins_encode %{
10583 __ lea(as_Register($dst$$reg),
10584 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10585 Address::lsl($scale$$constant)));
10586 %}
10587
10588 ins_pipe(ialu_reg_reg_shift);
10589 %}
10590
10591 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10592 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10593
10594 ins_cost(1.9 * INSN_COST);
10595 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10596
10597 ins_encode %{
10598 __ lea(as_Register($dst$$reg),
10599 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10600 Address::sxtw($scale$$constant)));
10601 %}
10602
10603 ins_pipe(ialu_reg_reg_shift);
10604 %}
10605
10606 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10607 match(Set dst (LShiftL (ConvI2L src) scale));
10608
10609 ins_cost(INSN_COST);
10610 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10611
10612 ins_encode %{
10613 __ sbfiz(as_Register($dst$$reg),
10614 as_Register($src$$reg),
10615 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10616 %}
10617
10618 ins_pipe(ialu_reg_shift);
10619 %}
10620
10621 // Pointer Immediate Addition
10622 // n.b. this needs to be more expensive than using an indirect memory
10623 // operand
10624 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10625 match(Set dst (AddP src1 src2));
10626
10627 ins_cost(INSN_COST);
10628 format %{ "add $dst, $src1, $src2\t# ptr" %}
10629
10630 // use opcode to indicate that this is an add not a sub
10631 opcode(0x0);
10632
10633 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10634
10635 ins_pipe(ialu_reg_imm);
10636 %}
10637
10638 // Long Addition
10639 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10640
10641 match(Set dst (AddL src1 src2));
10642
10643 ins_cost(INSN_COST);
10644 format %{ "add $dst, $src1, $src2" %}
10645
10646 ins_encode %{
10647 __ add(as_Register($dst$$reg),
10648 as_Register($src1$$reg),
10649 as_Register($src2$$reg));
10650 %}
10651
10652 ins_pipe(ialu_reg_reg);
10653 %}
10654
10655 // No constant pool entries requiredLong Immediate Addition.
10656 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10657 match(Set dst (AddL src1 src2));
10658
10659 ins_cost(INSN_COST);
10660 format %{ "add $dst, $src1, $src2" %}
10661
10662 // use opcode to indicate that this is an add not a sub
10663 opcode(0x0);
10664
10665 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10666
10667 ins_pipe(ialu_reg_imm);
10668 %}
10669
10670 // Integer Subtraction
10671 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10672 match(Set dst (SubI src1 src2));
10673
10674 ins_cost(INSN_COST);
10675 format %{ "subw $dst, $src1, $src2" %}
10676
10677 ins_encode %{
10678 __ subw(as_Register($dst$$reg),
10679 as_Register($src1$$reg),
10680 as_Register($src2$$reg));
10681 %}
10682
10683 ins_pipe(ialu_reg_reg);
10684 %}
10685
10686 // Immediate Subtraction
10687 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10688 match(Set dst (SubI src1 src2));
10689
10690 ins_cost(INSN_COST);
10691 format %{ "subw $dst, $src1, $src2" %}
10692
10693 // use opcode to indicate that this is a sub not an add
10694 opcode(0x1);
10695
10696 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10697
10698 ins_pipe(ialu_reg_imm);
10699 %}
10700
10701 // Long Subtraction
10702 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10703
10704 match(Set dst (SubL src1 src2));
10705
10706 ins_cost(INSN_COST);
10707 format %{ "sub $dst, $src1, $src2" %}
10708
10709 ins_encode %{
10710 __ sub(as_Register($dst$$reg),
10711 as_Register($src1$$reg),
10712 as_Register($src2$$reg));
10713 %}
10714
10715 ins_pipe(ialu_reg_reg);
10716 %}
10717
10718 // No constant pool entries requiredLong Immediate Subtraction.
10719 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10720 match(Set dst (SubL src1 src2));
10721
10722 ins_cost(INSN_COST);
10723 format %{ "sub$dst, $src1, $src2" %}
10724
10725 // use opcode to indicate that this is a sub not an add
10726 opcode(0x1);
10727
10728 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10729
10730 ins_pipe(ialu_reg_imm);
10731 %}
10732
10733 // Integer Negation (special case for sub)
10734
10735 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10736 match(Set dst (SubI zero src));
10737
10738 ins_cost(INSN_COST);
10739 format %{ "negw $dst, $src\t# int" %}
10740
10741 ins_encode %{
10742 __ negw(as_Register($dst$$reg),
10743 as_Register($src$$reg));
10744 %}
10745
10746 ins_pipe(ialu_reg);
10747 %}
10748
10749 // Long Negation
10750
10751 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10752 match(Set dst (SubL zero src));
10753
10754 ins_cost(INSN_COST);
10755 format %{ "neg $dst, $src\t# long" %}
10756
10757 ins_encode %{
10758 __ neg(as_Register($dst$$reg),
10759 as_Register($src$$reg));
10760 %}
10761
10762 ins_pipe(ialu_reg);
10763 %}
10764
10765 // Integer Multiply
10766
10767 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10768 match(Set dst (MulI src1 src2));
10769
10770 ins_cost(INSN_COST * 3);
10771 format %{ "mulw $dst, $src1, $src2" %}
10772
10773 ins_encode %{
10774 __ mulw(as_Register($dst$$reg),
10775 as_Register($src1$$reg),
10776 as_Register($src2$$reg));
10777 %}
10778
10779 ins_pipe(imul_reg_reg);
10780 %}
10781
10782 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10783 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10784
10785 ins_cost(INSN_COST * 3);
10786 format %{ "smull $dst, $src1, $src2" %}
10787
10788 ins_encode %{
10789 __ smull(as_Register($dst$$reg),
10790 as_Register($src1$$reg),
10791 as_Register($src2$$reg));
10792 %}
10793
10794 ins_pipe(imul_reg_reg);
10795 %}
10796
10797 // Long Multiply
10798
10799 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10800 match(Set dst (MulL src1 src2));
10801
10802 ins_cost(INSN_COST * 5);
10803 format %{ "mul $dst, $src1, $src2" %}
10804
10805 ins_encode %{
10806 __ mul(as_Register($dst$$reg),
10807 as_Register($src1$$reg),
10808 as_Register($src2$$reg));
10809 %}
10810
10811 ins_pipe(lmul_reg_reg);
10812 %}
10813
10814 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10815 %{
10816 match(Set dst (MulHiL src1 src2));
10817
10818 ins_cost(INSN_COST * 7);
10819 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10820
10821 ins_encode %{
10822 __ smulh(as_Register($dst$$reg),
10823 as_Register($src1$$reg),
10824 as_Register($src2$$reg));
10825 %}
10826
10827 ins_pipe(lmul_reg_reg);
10828 %}
10829
10830 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10831 %{
10832 match(Set dst (UMulHiL src1 src2));
10833
10834 ins_cost(INSN_COST * 7);
10835 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10836
10837 ins_encode %{
10838 __ umulh(as_Register($dst$$reg),
10839 as_Register($src1$$reg),
10840 as_Register($src2$$reg));
10841 %}
10842
10843 ins_pipe(lmul_reg_reg);
10844 %}
10845
10846 // Combined Integer Multiply & Add/Sub
10847
10848 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10849 match(Set dst (AddI src3 (MulI src1 src2)));
10850
10851 ins_cost(INSN_COST * 3);
10852 format %{ "madd $dst, $src1, $src2, $src3" %}
10853
10854 ins_encode %{
10855 __ maddw(as_Register($dst$$reg),
10856 as_Register($src1$$reg),
10857 as_Register($src2$$reg),
10858 as_Register($src3$$reg));
10859 %}
10860
10861 ins_pipe(imac_reg_reg);
10862 %}
10863
10864 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10865 match(Set dst (SubI src3 (MulI src1 src2)));
10866
10867 ins_cost(INSN_COST * 3);
10868 format %{ "msub $dst, $src1, $src2, $src3" %}
10869
10870 ins_encode %{
10871 __ msubw(as_Register($dst$$reg),
10872 as_Register($src1$$reg),
10873 as_Register($src2$$reg),
10874 as_Register($src3$$reg));
10875 %}
10876
10877 ins_pipe(imac_reg_reg);
10878 %}
10879
10880 // Combined Integer Multiply & Neg
10881
10882 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10883 match(Set dst (MulI (SubI zero src1) src2));
10884
10885 ins_cost(INSN_COST * 3);
10886 format %{ "mneg $dst, $src1, $src2" %}
10887
10888 ins_encode %{
10889 __ mnegw(as_Register($dst$$reg),
10890 as_Register($src1$$reg),
10891 as_Register($src2$$reg));
10892 %}
10893
10894 ins_pipe(imac_reg_reg);
10895 %}
10896
10897 // Combined Long Multiply & Add/Sub
10898
10899 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10900 match(Set dst (AddL src3 (MulL src1 src2)));
10901
10902 ins_cost(INSN_COST * 5);
10903 format %{ "madd $dst, $src1, $src2, $src3" %}
10904
10905 ins_encode %{
10906 __ madd(as_Register($dst$$reg),
10907 as_Register($src1$$reg),
10908 as_Register($src2$$reg),
10909 as_Register($src3$$reg));
10910 %}
10911
10912 ins_pipe(lmac_reg_reg);
10913 %}
10914
10915 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10916 match(Set dst (SubL src3 (MulL src1 src2)));
10917
10918 ins_cost(INSN_COST * 5);
10919 format %{ "msub $dst, $src1, $src2, $src3" %}
10920
10921 ins_encode %{
10922 __ msub(as_Register($dst$$reg),
10923 as_Register($src1$$reg),
10924 as_Register($src2$$reg),
10925 as_Register($src3$$reg));
10926 %}
10927
10928 ins_pipe(lmac_reg_reg);
10929 %}
10930
10931 // Combined Long Multiply & Neg
10932
10933 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10934 match(Set dst (MulL (SubL zero src1) src2));
10935
10936 ins_cost(INSN_COST * 5);
10937 format %{ "mneg $dst, $src1, $src2" %}
10938
10939 ins_encode %{
10940 __ mneg(as_Register($dst$$reg),
10941 as_Register($src1$$reg),
10942 as_Register($src2$$reg));
10943 %}
10944
10945 ins_pipe(lmac_reg_reg);
10946 %}
10947
10948 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10949
10950 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10951 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10952
10953 ins_cost(INSN_COST * 3);
10954 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10955
10956 ins_encode %{
10957 __ smaddl(as_Register($dst$$reg),
10958 as_Register($src1$$reg),
10959 as_Register($src2$$reg),
10960 as_Register($src3$$reg));
10961 %}
10962
10963 ins_pipe(imac_reg_reg);
10964 %}
10965
10966 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10967 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10968
10969 ins_cost(INSN_COST * 3);
10970 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10971
10972 ins_encode %{
10973 __ smsubl(as_Register($dst$$reg),
10974 as_Register($src1$$reg),
10975 as_Register($src2$$reg),
10976 as_Register($src3$$reg));
10977 %}
10978
10979 ins_pipe(imac_reg_reg);
10980 %}
10981
10982 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10983 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10984
10985 ins_cost(INSN_COST * 3);
10986 format %{ "smnegl $dst, $src1, $src2" %}
10987
10988 ins_encode %{
10989 __ smnegl(as_Register($dst$$reg),
10990 as_Register($src1$$reg),
10991 as_Register($src2$$reg));
10992 %}
10993
10994 ins_pipe(imac_reg_reg);
10995 %}
10996
10997 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10998
10999 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11000 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11001
11002 ins_cost(INSN_COST * 5);
11003 format %{ "mulw rscratch1, $src1, $src2\n\t"
11004 "maddw $dst, $src3, $src4, rscratch1" %}
11005
11006 ins_encode %{
11007 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11008 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11009
11010 ins_pipe(imac_reg_reg);
11011 %}
11012
11013 // Integer Divide
11014
11015 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11016 match(Set dst (DivI src1 src2));
11017
11018 ins_cost(INSN_COST * 19);
11019 format %{ "sdivw $dst, $src1, $src2" %}
11020
11021 ins_encode(aarch64_enc_divw(dst, src1, src2));
11022 ins_pipe(idiv_reg_reg);
11023 %}
11024
11025 // Long Divide
11026
11027 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11028 match(Set dst (DivL src1 src2));
11029
11030 ins_cost(INSN_COST * 35);
11031 format %{ "sdiv $dst, $src1, $src2" %}
11032
11033 ins_encode(aarch64_enc_div(dst, src1, src2));
11034 ins_pipe(ldiv_reg_reg);
11035 %}
11036
11037 // Integer Remainder
11038
11039 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11040 match(Set dst (ModI src1 src2));
11041
11042 ins_cost(INSN_COST * 22);
11043 format %{ "sdivw rscratch1, $src1, $src2\n\t"
11044 "msubw $dst, rscratch1, $src2, $src1" %}
11045
11046 ins_encode(aarch64_enc_modw(dst, src1, src2));
11047 ins_pipe(idiv_reg_reg);
11048 %}
11049
11050 // Long Remainder
11051
11052 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11053 match(Set dst (ModL src1 src2));
11054
11055 ins_cost(INSN_COST * 38);
11056 format %{ "sdiv rscratch1, $src1, $src2\n"
11057 "msub $dst, rscratch1, $src2, $src1" %}
11058
11059 ins_encode(aarch64_enc_mod(dst, src1, src2));
11060 ins_pipe(ldiv_reg_reg);
11061 %}
11062
11063 // Unsigned Integer Divide
11064
11065 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11066 match(Set dst (UDivI src1 src2));
11067
11068 ins_cost(INSN_COST * 19);
11069 format %{ "udivw $dst, $src1, $src2" %}
11070
11071 ins_encode %{
11072 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
11073 %}
11074
11075 ins_pipe(idiv_reg_reg);
11076 %}
11077
11078 // Unsigned Long Divide
11079
11080 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11081 match(Set dst (UDivL src1 src2));
11082
11083 ins_cost(INSN_COST * 35);
11084 format %{ "udiv $dst, $src1, $src2" %}
11085
11086 ins_encode %{
11087 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
11088 %}
11089
11090 ins_pipe(ldiv_reg_reg);
11091 %}
11092
11093 // Unsigned Integer Remainder
11094
11095 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11096 match(Set dst (UModI src1 src2));
11097
11098 ins_cost(INSN_COST * 22);
11099 format %{ "udivw rscratch1, $src1, $src2\n\t"
11100 "msubw $dst, rscratch1, $src2, $src1" %}
11101
11102 ins_encode %{
11103 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
11104 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
11105 %}
11106
11107 ins_pipe(idiv_reg_reg);
11108 %}
11109
11110 // Unsigned Long Remainder
11111
11112 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11113 match(Set dst (UModL src1 src2));
11114
11115 ins_cost(INSN_COST * 38);
11116 format %{ "udiv rscratch1, $src1, $src2\n"
11117 "msub $dst, rscratch1, $src2, $src1" %}
11118
11119 ins_encode %{
11120 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
11121 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
11122 %}
11123
11124 ins_pipe(ldiv_reg_reg);
11125 %}
11126
11127 // Integer Shifts
11128
11129 // Shift Left Register
11130 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11131 match(Set dst (LShiftI src1 src2));
11132
11133 ins_cost(INSN_COST * 2);
11134 format %{ "lslvw $dst, $src1, $src2" %}
11135
11136 ins_encode %{
11137 __ lslvw(as_Register($dst$$reg),
11138 as_Register($src1$$reg),
11139 as_Register($src2$$reg));
11140 %}
11141
11142 ins_pipe(ialu_reg_reg_vshift);
11143 %}
11144
11145 // Shift Left Immediate
11146 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11147 match(Set dst (LShiftI src1 src2));
11148
11149 ins_cost(INSN_COST);
11150 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11151
11152 ins_encode %{
11153 __ lslw(as_Register($dst$$reg),
11154 as_Register($src1$$reg),
11155 $src2$$constant & 0x1f);
11156 %}
11157
11158 ins_pipe(ialu_reg_shift);
11159 %}
11160
11161 // Shift Right Logical Register
11162 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11163 match(Set dst (URShiftI src1 src2));
11164
11165 ins_cost(INSN_COST * 2);
11166 format %{ "lsrvw $dst, $src1, $src2" %}
11167
11168 ins_encode %{
11169 __ lsrvw(as_Register($dst$$reg),
11170 as_Register($src1$$reg),
11171 as_Register($src2$$reg));
11172 %}
11173
11174 ins_pipe(ialu_reg_reg_vshift);
11175 %}
11176
11177 // Shift Right Logical Immediate
11178 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11179 match(Set dst (URShiftI src1 src2));
11180
11181 ins_cost(INSN_COST);
11182 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11183
11184 ins_encode %{
11185 __ lsrw(as_Register($dst$$reg),
11186 as_Register($src1$$reg),
11187 $src2$$constant & 0x1f);
11188 %}
11189
11190 ins_pipe(ialu_reg_shift);
11191 %}
11192
11193 // Shift Right Arithmetic Register
11194 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11195 match(Set dst (RShiftI src1 src2));
11196
11197 ins_cost(INSN_COST * 2);
11198 format %{ "asrvw $dst, $src1, $src2" %}
11199
11200 ins_encode %{
11201 __ asrvw(as_Register($dst$$reg),
11202 as_Register($src1$$reg),
11203 as_Register($src2$$reg));
11204 %}
11205
11206 ins_pipe(ialu_reg_reg_vshift);
11207 %}
11208
11209 // Shift Right Arithmetic Immediate
11210 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11211 match(Set dst (RShiftI src1 src2));
11212
11213 ins_cost(INSN_COST);
11214 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11215
11216 ins_encode %{
11217 __ asrw(as_Register($dst$$reg),
11218 as_Register($src1$$reg),
11219 $src2$$constant & 0x1f);
11220 %}
11221
11222 ins_pipe(ialu_reg_shift);
11223 %}
11224
11225 // Combined Int Mask and Right Shift (using UBFM)
11226 // TODO
11227
11228 // Long Shifts
11229
11230 // Shift Left Register
11231 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11232 match(Set dst (LShiftL src1 src2));
11233
11234 ins_cost(INSN_COST * 2);
11235 format %{ "lslv $dst, $src1, $src2" %}
11236
11237 ins_encode %{
11238 __ lslv(as_Register($dst$$reg),
11239 as_Register($src1$$reg),
11240 as_Register($src2$$reg));
11241 %}
11242
11243 ins_pipe(ialu_reg_reg_vshift);
11244 %}
11245
11246 // Shift Left Immediate
11247 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11248 match(Set dst (LShiftL src1 src2));
11249
11250 ins_cost(INSN_COST);
11251 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11252
11253 ins_encode %{
11254 __ lsl(as_Register($dst$$reg),
11255 as_Register($src1$$reg),
11256 $src2$$constant & 0x3f);
11257 %}
11258
11259 ins_pipe(ialu_reg_shift);
11260 %}
11261
11262 // Shift Right Logical Register
11263 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11264 match(Set dst (URShiftL src1 src2));
11265
11266 ins_cost(INSN_COST * 2);
11267 format %{ "lsrv $dst, $src1, $src2" %}
11268
11269 ins_encode %{
11270 __ lsrv(as_Register($dst$$reg),
11271 as_Register($src1$$reg),
11272 as_Register($src2$$reg));
11273 %}
11274
11275 ins_pipe(ialu_reg_reg_vshift);
11276 %}
11277
11278 // Shift Right Logical Immediate
11279 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11280 match(Set dst (URShiftL src1 src2));
11281
11282 ins_cost(INSN_COST);
11283 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11284
11285 ins_encode %{
11286 __ lsr(as_Register($dst$$reg),
11287 as_Register($src1$$reg),
11288 $src2$$constant & 0x3f);
11289 %}
11290
11291 ins_pipe(ialu_reg_shift);
11292 %}
11293
11294 // A special-case pattern for card table stores.
11295 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11296 match(Set dst (URShiftL (CastP2X src1) src2));
11297
11298 ins_cost(INSN_COST);
11299 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11300
11301 ins_encode %{
11302 __ lsr(as_Register($dst$$reg),
11303 as_Register($src1$$reg),
11304 $src2$$constant & 0x3f);
11305 %}
11306
11307 ins_pipe(ialu_reg_shift);
11308 %}
11309
11310 // Shift Right Arithmetic Register
11311 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11312 match(Set dst (RShiftL src1 src2));
11313
11314 ins_cost(INSN_COST * 2);
11315 format %{ "asrv $dst, $src1, $src2" %}
11316
11317 ins_encode %{
11318 __ asrv(as_Register($dst$$reg),
11319 as_Register($src1$$reg),
11320 as_Register($src2$$reg));
11321 %}
11322
11323 ins_pipe(ialu_reg_reg_vshift);
11324 %}
11325
11326 // Shift Right Arithmetic Immediate
11327 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11328 match(Set dst (RShiftL src1 src2));
11329
11330 ins_cost(INSN_COST);
11331 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11332
11333 ins_encode %{
11334 __ asr(as_Register($dst$$reg),
11335 as_Register($src1$$reg),
11336 $src2$$constant & 0x3f);
11337 %}
11338
11339 ins_pipe(ialu_reg_shift);
11340 %}
11341
11342 // BEGIN This section of the file is automatically generated. Do not edit --------------
11343 // This section is generated from aarch64_ad.m4
11344
11345 // This pattern is automatically generated from aarch64_ad.m4
11346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11347 instruct regL_not_reg(iRegLNoSp dst,
11348 iRegL src1, immL_M1 m1,
11349 rFlagsReg cr) %{
11350 match(Set dst (XorL src1 m1));
11351 ins_cost(INSN_COST);
11352 format %{ "eon $dst, $src1, zr" %}
11353
11354 ins_encode %{
11355 __ eon(as_Register($dst$$reg),
11356 as_Register($src1$$reg),
11357 zr,
11358 Assembler::LSL, 0);
11359 %}
11360
11361 ins_pipe(ialu_reg);
11362 %}
11363
11364 // This pattern is automatically generated from aarch64_ad.m4
11365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11366 instruct regI_not_reg(iRegINoSp dst,
11367 iRegIorL2I src1, immI_M1 m1,
11368 rFlagsReg cr) %{
11369 match(Set dst (XorI src1 m1));
11370 ins_cost(INSN_COST);
11371 format %{ "eonw $dst, $src1, zr" %}
11372
11373 ins_encode %{
11374 __ eonw(as_Register($dst$$reg),
11375 as_Register($src1$$reg),
11376 zr,
11377 Assembler::LSL, 0);
11378 %}
11379
11380 ins_pipe(ialu_reg);
11381 %}
11382
11383 // This pattern is automatically generated from aarch64_ad.m4
11384 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11385 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11386 immI0 zero, iRegIorL2I src1, immI src2) %{
11387 match(Set dst (SubI zero (URShiftI src1 src2)));
11388
11389 ins_cost(1.9 * INSN_COST);
11390 format %{ "negw $dst, $src1, LSR $src2" %}
11391
11392 ins_encode %{
11393 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11394 Assembler::LSR, $src2$$constant & 0x1f);
11395 %}
11396
11397 ins_pipe(ialu_reg_shift);
11398 %}
11399
11400 // This pattern is automatically generated from aarch64_ad.m4
11401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11402 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11403 immI0 zero, iRegIorL2I src1, immI src2) %{
11404 match(Set dst (SubI zero (RShiftI src1 src2)));
11405
11406 ins_cost(1.9 * INSN_COST);
11407 format %{ "negw $dst, $src1, ASR $src2" %}
11408
11409 ins_encode %{
11410 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11411 Assembler::ASR, $src2$$constant & 0x1f);
11412 %}
11413
11414 ins_pipe(ialu_reg_shift);
11415 %}
11416
11417 // This pattern is automatically generated from aarch64_ad.m4
11418 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11419 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11420 immI0 zero, iRegIorL2I src1, immI src2) %{
11421 match(Set dst (SubI zero (LShiftI src1 src2)));
11422
11423 ins_cost(1.9 * INSN_COST);
11424 format %{ "negw $dst, $src1, LSL $src2" %}
11425
11426 ins_encode %{
11427 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11428 Assembler::LSL, $src2$$constant & 0x1f);
11429 %}
11430
11431 ins_pipe(ialu_reg_shift);
11432 %}
11433
11434 // This pattern is automatically generated from aarch64_ad.m4
11435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11436 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11437 immL0 zero, iRegL src1, immI src2) %{
11438 match(Set dst (SubL zero (URShiftL src1 src2)));
11439
11440 ins_cost(1.9 * INSN_COST);
11441 format %{ "neg $dst, $src1, LSR $src2" %}
11442
11443 ins_encode %{
11444 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11445 Assembler::LSR, $src2$$constant & 0x3f);
11446 %}
11447
11448 ins_pipe(ialu_reg_shift);
11449 %}
11450
11451 // This pattern is automatically generated from aarch64_ad.m4
11452 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11453 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11454 immL0 zero, iRegL src1, immI src2) %{
11455 match(Set dst (SubL zero (RShiftL src1 src2)));
11456
11457 ins_cost(1.9 * INSN_COST);
11458 format %{ "neg $dst, $src1, ASR $src2" %}
11459
11460 ins_encode %{
11461 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11462 Assembler::ASR, $src2$$constant & 0x3f);
11463 %}
11464
11465 ins_pipe(ialu_reg_shift);
11466 %}
11467
11468 // This pattern is automatically generated from aarch64_ad.m4
11469 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11470 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11471 immL0 zero, iRegL src1, immI src2) %{
11472 match(Set dst (SubL zero (LShiftL src1 src2)));
11473
11474 ins_cost(1.9 * INSN_COST);
11475 format %{ "neg $dst, $src1, LSL $src2" %}
11476
11477 ins_encode %{
11478 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11479 Assembler::LSL, $src2$$constant & 0x3f);
11480 %}
11481
11482 ins_pipe(ialu_reg_shift);
11483 %}
11484
11485 // This pattern is automatically generated from aarch64_ad.m4
11486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11487 instruct AndI_reg_not_reg(iRegINoSp dst,
11488 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11489 match(Set dst (AndI src1 (XorI src2 m1)));
11490 ins_cost(INSN_COST);
11491 format %{ "bicw $dst, $src1, $src2" %}
11492
11493 ins_encode %{
11494 __ bicw(as_Register($dst$$reg),
11495 as_Register($src1$$reg),
11496 as_Register($src2$$reg),
11497 Assembler::LSL, 0);
11498 %}
11499
11500 ins_pipe(ialu_reg_reg);
11501 %}
11502
11503 // This pattern is automatically generated from aarch64_ad.m4
11504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11505 instruct AndL_reg_not_reg(iRegLNoSp dst,
11506 iRegL src1, iRegL src2, immL_M1 m1) %{
11507 match(Set dst (AndL src1 (XorL src2 m1)));
11508 ins_cost(INSN_COST);
11509 format %{ "bic $dst, $src1, $src2" %}
11510
11511 ins_encode %{
11512 __ bic(as_Register($dst$$reg),
11513 as_Register($src1$$reg),
11514 as_Register($src2$$reg),
11515 Assembler::LSL, 0);
11516 %}
11517
11518 ins_pipe(ialu_reg_reg);
11519 %}
11520
11521 // This pattern is automatically generated from aarch64_ad.m4
11522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11523 instruct OrI_reg_not_reg(iRegINoSp dst,
11524 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11525 match(Set dst (OrI src1 (XorI src2 m1)));
11526 ins_cost(INSN_COST);
11527 format %{ "ornw $dst, $src1, $src2" %}
11528
11529 ins_encode %{
11530 __ ornw(as_Register($dst$$reg),
11531 as_Register($src1$$reg),
11532 as_Register($src2$$reg),
11533 Assembler::LSL, 0);
11534 %}
11535
11536 ins_pipe(ialu_reg_reg);
11537 %}
11538
11539 // This pattern is automatically generated from aarch64_ad.m4
11540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11541 instruct OrL_reg_not_reg(iRegLNoSp dst,
11542 iRegL src1, iRegL src2, immL_M1 m1) %{
11543 match(Set dst (OrL src1 (XorL src2 m1)));
11544 ins_cost(INSN_COST);
11545 format %{ "orn $dst, $src1, $src2" %}
11546
11547 ins_encode %{
11548 __ orn(as_Register($dst$$reg),
11549 as_Register($src1$$reg),
11550 as_Register($src2$$reg),
11551 Assembler::LSL, 0);
11552 %}
11553
11554 ins_pipe(ialu_reg_reg);
11555 %}
11556
11557 // This pattern is automatically generated from aarch64_ad.m4
11558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11559 instruct XorI_reg_not_reg(iRegINoSp dst,
11560 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11561 match(Set dst (XorI m1 (XorI src2 src1)));
11562 ins_cost(INSN_COST);
11563 format %{ "eonw $dst, $src1, $src2" %}
11564
11565 ins_encode %{
11566 __ eonw(as_Register($dst$$reg),
11567 as_Register($src1$$reg),
11568 as_Register($src2$$reg),
11569 Assembler::LSL, 0);
11570 %}
11571
11572 ins_pipe(ialu_reg_reg);
11573 %}
11574
11575 // This pattern is automatically generated from aarch64_ad.m4
11576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11577 instruct XorL_reg_not_reg(iRegLNoSp dst,
11578 iRegL src1, iRegL src2, immL_M1 m1) %{
11579 match(Set dst (XorL m1 (XorL src2 src1)));
11580 ins_cost(INSN_COST);
11581 format %{ "eon $dst, $src1, $src2" %}
11582
11583 ins_encode %{
11584 __ eon(as_Register($dst$$reg),
11585 as_Register($src1$$reg),
11586 as_Register($src2$$reg),
11587 Assembler::LSL, 0);
11588 %}
11589
11590 ins_pipe(ialu_reg_reg);
11591 %}
11592
11593 // This pattern is automatically generated from aarch64_ad.m4
11594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11595 // val & (-1 ^ (val >>> shift)) ==> bicw
11596 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11597 iRegIorL2I src1, iRegIorL2I src2,
11598 immI src3, immI_M1 src4) %{
11599 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11600 ins_cost(1.9 * INSN_COST);
11601 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11602
11603 ins_encode %{
11604 __ bicw(as_Register($dst$$reg),
11605 as_Register($src1$$reg),
11606 as_Register($src2$$reg),
11607 Assembler::LSR,
11608 $src3$$constant & 0x1f);
11609 %}
11610
11611 ins_pipe(ialu_reg_reg_shift);
11612 %}
11613
11614 // This pattern is automatically generated from aarch64_ad.m4
11615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11616 // val & (-1 ^ (val >>> shift)) ==> bic
11617 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11618 iRegL src1, iRegL src2,
11619 immI src3, immL_M1 src4) %{
11620 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11621 ins_cost(1.9 * INSN_COST);
11622 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11623
11624 ins_encode %{
11625 __ bic(as_Register($dst$$reg),
11626 as_Register($src1$$reg),
11627 as_Register($src2$$reg),
11628 Assembler::LSR,
11629 $src3$$constant & 0x3f);
11630 %}
11631
11632 ins_pipe(ialu_reg_reg_shift);
11633 %}
11634
11635 // This pattern is automatically generated from aarch64_ad.m4
11636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11637 // val & (-1 ^ (val >> shift)) ==> bicw
11638 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11639 iRegIorL2I src1, iRegIorL2I src2,
11640 immI src3, immI_M1 src4) %{
11641 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11642 ins_cost(1.9 * INSN_COST);
11643 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11644
11645 ins_encode %{
11646 __ bicw(as_Register($dst$$reg),
11647 as_Register($src1$$reg),
11648 as_Register($src2$$reg),
11649 Assembler::ASR,
11650 $src3$$constant & 0x1f);
11651 %}
11652
11653 ins_pipe(ialu_reg_reg_shift);
11654 %}
11655
11656 // This pattern is automatically generated from aarch64_ad.m4
11657 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11658 // val & (-1 ^ (val >> shift)) ==> bic
11659 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11660 iRegL src1, iRegL src2,
11661 immI src3, immL_M1 src4) %{
11662 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11663 ins_cost(1.9 * INSN_COST);
11664 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11665
11666 ins_encode %{
11667 __ bic(as_Register($dst$$reg),
11668 as_Register($src1$$reg),
11669 as_Register($src2$$reg),
11670 Assembler::ASR,
11671 $src3$$constant & 0x3f);
11672 %}
11673
11674 ins_pipe(ialu_reg_reg_shift);
11675 %}
11676
11677 // This pattern is automatically generated from aarch64_ad.m4
11678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11679 // val & (-1 ^ (val ror shift)) ==> bicw
11680 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11681 iRegIorL2I src1, iRegIorL2I src2,
11682 immI src3, immI_M1 src4) %{
11683 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11684 ins_cost(1.9 * INSN_COST);
11685 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11686
11687 ins_encode %{
11688 __ bicw(as_Register($dst$$reg),
11689 as_Register($src1$$reg),
11690 as_Register($src2$$reg),
11691 Assembler::ROR,
11692 $src3$$constant & 0x1f);
11693 %}
11694
11695 ins_pipe(ialu_reg_reg_shift);
11696 %}
11697
11698 // This pattern is automatically generated from aarch64_ad.m4
11699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11700 // val & (-1 ^ (val ror shift)) ==> bic
11701 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11702 iRegL src1, iRegL src2,
11703 immI src3, immL_M1 src4) %{
11704 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11705 ins_cost(1.9 * INSN_COST);
11706 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11707
11708 ins_encode %{
11709 __ bic(as_Register($dst$$reg),
11710 as_Register($src1$$reg),
11711 as_Register($src2$$reg),
11712 Assembler::ROR,
11713 $src3$$constant & 0x3f);
11714 %}
11715
11716 ins_pipe(ialu_reg_reg_shift);
11717 %}
11718
11719 // This pattern is automatically generated from aarch64_ad.m4
11720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11721 // val & (-1 ^ (val << shift)) ==> bicw
11722 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11723 iRegIorL2I src1, iRegIorL2I src2,
11724 immI src3, immI_M1 src4) %{
11725 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11726 ins_cost(1.9 * INSN_COST);
11727 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11728
11729 ins_encode %{
11730 __ bicw(as_Register($dst$$reg),
11731 as_Register($src1$$reg),
11732 as_Register($src2$$reg),
11733 Assembler::LSL,
11734 $src3$$constant & 0x1f);
11735 %}
11736
11737 ins_pipe(ialu_reg_reg_shift);
11738 %}
11739
11740 // This pattern is automatically generated from aarch64_ad.m4
11741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11742 // val & (-1 ^ (val << shift)) ==> bic
11743 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11744 iRegL src1, iRegL src2,
11745 immI src3, immL_M1 src4) %{
11746 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11747 ins_cost(1.9 * INSN_COST);
11748 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11749
11750 ins_encode %{
11751 __ bic(as_Register($dst$$reg),
11752 as_Register($src1$$reg),
11753 as_Register($src2$$reg),
11754 Assembler::LSL,
11755 $src3$$constant & 0x3f);
11756 %}
11757
11758 ins_pipe(ialu_reg_reg_shift);
11759 %}
11760
11761 // This pattern is automatically generated from aarch64_ad.m4
11762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11763 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11764 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11765 iRegIorL2I src1, iRegIorL2I src2,
11766 immI src3, immI_M1 src4) %{
11767 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11768 ins_cost(1.9 * INSN_COST);
11769 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11770
11771 ins_encode %{
11772 __ eonw(as_Register($dst$$reg),
11773 as_Register($src1$$reg),
11774 as_Register($src2$$reg),
11775 Assembler::LSR,
11776 $src3$$constant & 0x1f);
11777 %}
11778
11779 ins_pipe(ialu_reg_reg_shift);
11780 %}
11781
11782 // This pattern is automatically generated from aarch64_ad.m4
11783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11784 // val ^ (-1 ^ (val >>> shift)) ==> eon
11785 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11786 iRegL src1, iRegL src2,
11787 immI src3, immL_M1 src4) %{
11788 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11789 ins_cost(1.9 * INSN_COST);
11790 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11791
11792 ins_encode %{
11793 __ eon(as_Register($dst$$reg),
11794 as_Register($src1$$reg),
11795 as_Register($src2$$reg),
11796 Assembler::LSR,
11797 $src3$$constant & 0x3f);
11798 %}
11799
11800 ins_pipe(ialu_reg_reg_shift);
11801 %}
11802
11803 // This pattern is automatically generated from aarch64_ad.m4
11804 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11805 // val ^ (-1 ^ (val >> shift)) ==> eonw
11806 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11807 iRegIorL2I src1, iRegIorL2I src2,
11808 immI src3, immI_M1 src4) %{
11809 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11810 ins_cost(1.9 * INSN_COST);
11811 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11812
11813 ins_encode %{
11814 __ eonw(as_Register($dst$$reg),
11815 as_Register($src1$$reg),
11816 as_Register($src2$$reg),
11817 Assembler::ASR,
11818 $src3$$constant & 0x1f);
11819 %}
11820
11821 ins_pipe(ialu_reg_reg_shift);
11822 %}
11823
11824 // This pattern is automatically generated from aarch64_ad.m4
11825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11826 // val ^ (-1 ^ (val >> shift)) ==> eon
11827 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11828 iRegL src1, iRegL src2,
11829 immI src3, immL_M1 src4) %{
11830 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11831 ins_cost(1.9 * INSN_COST);
11832 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11833
11834 ins_encode %{
11835 __ eon(as_Register($dst$$reg),
11836 as_Register($src1$$reg),
11837 as_Register($src2$$reg),
11838 Assembler::ASR,
11839 $src3$$constant & 0x3f);
11840 %}
11841
11842 ins_pipe(ialu_reg_reg_shift);
11843 %}
11844
11845 // This pattern is automatically generated from aarch64_ad.m4
11846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11847 // val ^ (-1 ^ (val ror shift)) ==> eonw
11848 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11849 iRegIorL2I src1, iRegIorL2I src2,
11850 immI src3, immI_M1 src4) %{
11851 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11852 ins_cost(1.9 * INSN_COST);
11853 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11854
11855 ins_encode %{
11856 __ eonw(as_Register($dst$$reg),
11857 as_Register($src1$$reg),
11858 as_Register($src2$$reg),
11859 Assembler::ROR,
11860 $src3$$constant & 0x1f);
11861 %}
11862
11863 ins_pipe(ialu_reg_reg_shift);
11864 %}
11865
11866 // This pattern is automatically generated from aarch64_ad.m4
11867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11868 // val ^ (-1 ^ (val ror shift)) ==> eon
11869 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11870 iRegL src1, iRegL src2,
11871 immI src3, immL_M1 src4) %{
11872 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11873 ins_cost(1.9 * INSN_COST);
11874 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11875
11876 ins_encode %{
11877 __ eon(as_Register($dst$$reg),
11878 as_Register($src1$$reg),
11879 as_Register($src2$$reg),
11880 Assembler::ROR,
11881 $src3$$constant & 0x3f);
11882 %}
11883
11884 ins_pipe(ialu_reg_reg_shift);
11885 %}
11886
11887 // This pattern is automatically generated from aarch64_ad.m4
11888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11889 // val ^ (-1 ^ (val << shift)) ==> eonw
11890 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11891 iRegIorL2I src1, iRegIorL2I src2,
11892 immI src3, immI_M1 src4) %{
11893 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11894 ins_cost(1.9 * INSN_COST);
11895 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11896
11897 ins_encode %{
11898 __ eonw(as_Register($dst$$reg),
11899 as_Register($src1$$reg),
11900 as_Register($src2$$reg),
11901 Assembler::LSL,
11902 $src3$$constant & 0x1f);
11903 %}
11904
11905 ins_pipe(ialu_reg_reg_shift);
11906 %}
11907
11908 // This pattern is automatically generated from aarch64_ad.m4
11909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11910 // val ^ (-1 ^ (val << shift)) ==> eon
11911 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11912 iRegL src1, iRegL src2,
11913 immI src3, immL_M1 src4) %{
11914 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11915 ins_cost(1.9 * INSN_COST);
11916 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11917
11918 ins_encode %{
11919 __ eon(as_Register($dst$$reg),
11920 as_Register($src1$$reg),
11921 as_Register($src2$$reg),
11922 Assembler::LSL,
11923 $src3$$constant & 0x3f);
11924 %}
11925
11926 ins_pipe(ialu_reg_reg_shift);
11927 %}
11928
11929 // This pattern is automatically generated from aarch64_ad.m4
11930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11931 // val | (-1 ^ (val >>> shift)) ==> ornw
11932 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11933 iRegIorL2I src1, iRegIorL2I src2,
11934 immI src3, immI_M1 src4) %{
11935 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11936 ins_cost(1.9 * INSN_COST);
11937 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11938
11939 ins_encode %{
11940 __ ornw(as_Register($dst$$reg),
11941 as_Register($src1$$reg),
11942 as_Register($src2$$reg),
11943 Assembler::LSR,
11944 $src3$$constant & 0x1f);
11945 %}
11946
11947 ins_pipe(ialu_reg_reg_shift);
11948 %}
11949
11950 // This pattern is automatically generated from aarch64_ad.m4
11951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11952 // val | (-1 ^ (val >>> shift)) ==> orn
11953 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11954 iRegL src1, iRegL src2,
11955 immI src3, immL_M1 src4) %{
11956 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11957 ins_cost(1.9 * INSN_COST);
11958 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11959
11960 ins_encode %{
11961 __ orn(as_Register($dst$$reg),
11962 as_Register($src1$$reg),
11963 as_Register($src2$$reg),
11964 Assembler::LSR,
11965 $src3$$constant & 0x3f);
11966 %}
11967
11968 ins_pipe(ialu_reg_reg_shift);
11969 %}
11970
11971 // This pattern is automatically generated from aarch64_ad.m4
11972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11973 // val | (-1 ^ (val >> shift)) ==> ornw
11974 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11975 iRegIorL2I src1, iRegIorL2I src2,
11976 immI src3, immI_M1 src4) %{
11977 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11978 ins_cost(1.9 * INSN_COST);
11979 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11980
11981 ins_encode %{
11982 __ ornw(as_Register($dst$$reg),
11983 as_Register($src1$$reg),
11984 as_Register($src2$$reg),
11985 Assembler::ASR,
11986 $src3$$constant & 0x1f);
11987 %}
11988
11989 ins_pipe(ialu_reg_reg_shift);
11990 %}
11991
11992 // This pattern is automatically generated from aarch64_ad.m4
11993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11994 // val | (-1 ^ (val >> shift)) ==> orn
11995 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11996 iRegL src1, iRegL src2,
11997 immI src3, immL_M1 src4) %{
11998 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11999 ins_cost(1.9 * INSN_COST);
12000 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
12001
12002 ins_encode %{
12003 __ orn(as_Register($dst$$reg),
12004 as_Register($src1$$reg),
12005 as_Register($src2$$reg),
12006 Assembler::ASR,
12007 $src3$$constant & 0x3f);
12008 %}
12009
12010 ins_pipe(ialu_reg_reg_shift);
12011 %}
12012
12013 // This pattern is automatically generated from aarch64_ad.m4
12014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12015 // val | (-1 ^ (val ror shift)) ==> ornw
12016 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
12017 iRegIorL2I src1, iRegIorL2I src2,
12018 immI src3, immI_M1 src4) %{
12019 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
12020 ins_cost(1.9 * INSN_COST);
12021 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
12022
12023 ins_encode %{
12024 __ ornw(as_Register($dst$$reg),
12025 as_Register($src1$$reg),
12026 as_Register($src2$$reg),
12027 Assembler::ROR,
12028 $src3$$constant & 0x1f);
12029 %}
12030
12031 ins_pipe(ialu_reg_reg_shift);
12032 %}
12033
12034 // This pattern is automatically generated from aarch64_ad.m4
12035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12036 // val | (-1 ^ (val ror shift)) ==> orn
12037 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12038 iRegL src1, iRegL src2,
12039 immI src3, immL_M1 src4) %{
12040 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12041 ins_cost(1.9 * INSN_COST);
12042 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
12043
12044 ins_encode %{
12045 __ orn(as_Register($dst$$reg),
12046 as_Register($src1$$reg),
12047 as_Register($src2$$reg),
12048 Assembler::ROR,
12049 $src3$$constant & 0x3f);
12050 %}
12051
12052 ins_pipe(ialu_reg_reg_shift);
12053 %}
12054
12055 // This pattern is automatically generated from aarch64_ad.m4
12056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12057 // val | (-1 ^ (val << shift)) ==> ornw
12058 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12059 iRegIorL2I src1, iRegIorL2I src2,
12060 immI src3, immI_M1 src4) %{
12061 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12062 ins_cost(1.9 * INSN_COST);
12063 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
12064
12065 ins_encode %{
12066 __ ornw(as_Register($dst$$reg),
12067 as_Register($src1$$reg),
12068 as_Register($src2$$reg),
12069 Assembler::LSL,
12070 $src3$$constant & 0x1f);
12071 %}
12072
12073 ins_pipe(ialu_reg_reg_shift);
12074 %}
12075
12076 // This pattern is automatically generated from aarch64_ad.m4
12077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12078 // val | (-1 ^ (val << shift)) ==> orn
12079 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12080 iRegL src1, iRegL src2,
12081 immI src3, immL_M1 src4) %{
12082 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12083 ins_cost(1.9 * INSN_COST);
12084 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
12085
12086 ins_encode %{
12087 __ orn(as_Register($dst$$reg),
12088 as_Register($src1$$reg),
12089 as_Register($src2$$reg),
12090 Assembler::LSL,
12091 $src3$$constant & 0x3f);
12092 %}
12093
12094 ins_pipe(ialu_reg_reg_shift);
12095 %}
12096
12097 // This pattern is automatically generated from aarch64_ad.m4
12098 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12099 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12100 iRegIorL2I src1, iRegIorL2I src2,
12101 immI src3) %{
12102 match(Set dst (AndI src1 (URShiftI src2 src3)));
12103
12104 ins_cost(1.9 * INSN_COST);
12105 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
12106
12107 ins_encode %{
12108 __ andw(as_Register($dst$$reg),
12109 as_Register($src1$$reg),
12110 as_Register($src2$$reg),
12111 Assembler::LSR,
12112 $src3$$constant & 0x1f);
12113 %}
12114
12115 ins_pipe(ialu_reg_reg_shift);
12116 %}
12117
12118 // This pattern is automatically generated from aarch64_ad.m4
12119 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12120 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12121 iRegL src1, iRegL src2,
12122 immI src3) %{
12123 match(Set dst (AndL src1 (URShiftL src2 src3)));
12124
12125 ins_cost(1.9 * INSN_COST);
12126 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
12127
12128 ins_encode %{
12129 __ andr(as_Register($dst$$reg),
12130 as_Register($src1$$reg),
12131 as_Register($src2$$reg),
12132 Assembler::LSR,
12133 $src3$$constant & 0x3f);
12134 %}
12135
12136 ins_pipe(ialu_reg_reg_shift);
12137 %}
12138
12139 // This pattern is automatically generated from aarch64_ad.m4
12140 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12141 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12142 iRegIorL2I src1, iRegIorL2I src2,
12143 immI src3) %{
12144 match(Set dst (AndI src1 (RShiftI src2 src3)));
12145
12146 ins_cost(1.9 * INSN_COST);
12147 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
12148
12149 ins_encode %{
12150 __ andw(as_Register($dst$$reg),
12151 as_Register($src1$$reg),
12152 as_Register($src2$$reg),
12153 Assembler::ASR,
12154 $src3$$constant & 0x1f);
12155 %}
12156
12157 ins_pipe(ialu_reg_reg_shift);
12158 %}
12159
12160 // This pattern is automatically generated from aarch64_ad.m4
12161 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12162 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12163 iRegL src1, iRegL src2,
12164 immI src3) %{
12165 match(Set dst (AndL src1 (RShiftL src2 src3)));
12166
12167 ins_cost(1.9 * INSN_COST);
12168 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
12169
12170 ins_encode %{
12171 __ andr(as_Register($dst$$reg),
12172 as_Register($src1$$reg),
12173 as_Register($src2$$reg),
12174 Assembler::ASR,
12175 $src3$$constant & 0x3f);
12176 %}
12177
12178 ins_pipe(ialu_reg_reg_shift);
12179 %}
12180
12181 // This pattern is automatically generated from aarch64_ad.m4
12182 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12183 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12184 iRegIorL2I src1, iRegIorL2I src2,
12185 immI src3) %{
12186 match(Set dst (AndI src1 (LShiftI src2 src3)));
12187
12188 ins_cost(1.9 * INSN_COST);
12189 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
12190
12191 ins_encode %{
12192 __ andw(as_Register($dst$$reg),
12193 as_Register($src1$$reg),
12194 as_Register($src2$$reg),
12195 Assembler::LSL,
12196 $src3$$constant & 0x1f);
12197 %}
12198
12199 ins_pipe(ialu_reg_reg_shift);
12200 %}
12201
12202 // This pattern is automatically generated from aarch64_ad.m4
12203 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12204 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12205 iRegL src1, iRegL src2,
12206 immI src3) %{
12207 match(Set dst (AndL src1 (LShiftL src2 src3)));
12208
12209 ins_cost(1.9 * INSN_COST);
12210 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
12211
12212 ins_encode %{
12213 __ andr(as_Register($dst$$reg),
12214 as_Register($src1$$reg),
12215 as_Register($src2$$reg),
12216 Assembler::LSL,
12217 $src3$$constant & 0x3f);
12218 %}
12219
12220 ins_pipe(ialu_reg_reg_shift);
12221 %}
12222
12223 // This pattern is automatically generated from aarch64_ad.m4
12224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12225 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12226 iRegIorL2I src1, iRegIorL2I src2,
12227 immI src3) %{
12228 match(Set dst (AndI src1 (RotateRight src2 src3)));
12229
12230 ins_cost(1.9 * INSN_COST);
12231 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
12232
12233 ins_encode %{
12234 __ andw(as_Register($dst$$reg),
12235 as_Register($src1$$reg),
12236 as_Register($src2$$reg),
12237 Assembler::ROR,
12238 $src3$$constant & 0x1f);
12239 %}
12240
12241 ins_pipe(ialu_reg_reg_shift);
12242 %}
12243
12244 // This pattern is automatically generated from aarch64_ad.m4
12245 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12246 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12247 iRegL src1, iRegL src2,
12248 immI src3) %{
12249 match(Set dst (AndL src1 (RotateRight src2 src3)));
12250
12251 ins_cost(1.9 * INSN_COST);
12252 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
12253
12254 ins_encode %{
12255 __ andr(as_Register($dst$$reg),
12256 as_Register($src1$$reg),
12257 as_Register($src2$$reg),
12258 Assembler::ROR,
12259 $src3$$constant & 0x3f);
12260 %}
12261
12262 ins_pipe(ialu_reg_reg_shift);
12263 %}
12264
12265 // This pattern is automatically generated from aarch64_ad.m4
12266 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12267 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12268 iRegIorL2I src1, iRegIorL2I src2,
12269 immI src3) %{
12270 match(Set dst (XorI src1 (URShiftI src2 src3)));
12271
12272 ins_cost(1.9 * INSN_COST);
12273 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
12274
12275 ins_encode %{
12276 __ eorw(as_Register($dst$$reg),
12277 as_Register($src1$$reg),
12278 as_Register($src2$$reg),
12279 Assembler::LSR,
12280 $src3$$constant & 0x1f);
12281 %}
12282
12283 ins_pipe(ialu_reg_reg_shift);
12284 %}
12285
12286 // This pattern is automatically generated from aarch64_ad.m4
12287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12288 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12289 iRegL src1, iRegL src2,
12290 immI src3) %{
12291 match(Set dst (XorL src1 (URShiftL src2 src3)));
12292
12293 ins_cost(1.9 * INSN_COST);
12294 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
12295
12296 ins_encode %{
12297 __ eor(as_Register($dst$$reg),
12298 as_Register($src1$$reg),
12299 as_Register($src2$$reg),
12300 Assembler::LSR,
12301 $src3$$constant & 0x3f);
12302 %}
12303
12304 ins_pipe(ialu_reg_reg_shift);
12305 %}
12306
12307 // This pattern is automatically generated from aarch64_ad.m4
12308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12309 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12310 iRegIorL2I src1, iRegIorL2I src2,
12311 immI src3) %{
12312 match(Set dst (XorI src1 (RShiftI src2 src3)));
12313
12314 ins_cost(1.9 * INSN_COST);
12315 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
12316
12317 ins_encode %{
12318 __ eorw(as_Register($dst$$reg),
12319 as_Register($src1$$reg),
12320 as_Register($src2$$reg),
12321 Assembler::ASR,
12322 $src3$$constant & 0x1f);
12323 %}
12324
12325 ins_pipe(ialu_reg_reg_shift);
12326 %}
12327
12328 // This pattern is automatically generated from aarch64_ad.m4
12329 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12330 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12331 iRegL src1, iRegL src2,
12332 immI src3) %{
12333 match(Set dst (XorL src1 (RShiftL src2 src3)));
12334
12335 ins_cost(1.9 * INSN_COST);
12336 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
12337
12338 ins_encode %{
12339 __ eor(as_Register($dst$$reg),
12340 as_Register($src1$$reg),
12341 as_Register($src2$$reg),
12342 Assembler::ASR,
12343 $src3$$constant & 0x3f);
12344 %}
12345
12346 ins_pipe(ialu_reg_reg_shift);
12347 %}
12348
12349 // This pattern is automatically generated from aarch64_ad.m4
12350 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12351 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12352 iRegIorL2I src1, iRegIorL2I src2,
12353 immI src3) %{
12354 match(Set dst (XorI src1 (LShiftI src2 src3)));
12355
12356 ins_cost(1.9 * INSN_COST);
12357 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
12358
12359 ins_encode %{
12360 __ eorw(as_Register($dst$$reg),
12361 as_Register($src1$$reg),
12362 as_Register($src2$$reg),
12363 Assembler::LSL,
12364 $src3$$constant & 0x1f);
12365 %}
12366
12367 ins_pipe(ialu_reg_reg_shift);
12368 %}
12369
12370 // This pattern is automatically generated from aarch64_ad.m4
12371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12372 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12373 iRegL src1, iRegL src2,
12374 immI src3) %{
12375 match(Set dst (XorL src1 (LShiftL src2 src3)));
12376
12377 ins_cost(1.9 * INSN_COST);
12378 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
12379
12380 ins_encode %{
12381 __ eor(as_Register($dst$$reg),
12382 as_Register($src1$$reg),
12383 as_Register($src2$$reg),
12384 Assembler::LSL,
12385 $src3$$constant & 0x3f);
12386 %}
12387
12388 ins_pipe(ialu_reg_reg_shift);
12389 %}
12390
12391 // This pattern is automatically generated from aarch64_ad.m4
12392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12393 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12394 iRegIorL2I src1, iRegIorL2I src2,
12395 immI src3) %{
12396 match(Set dst (XorI src1 (RotateRight src2 src3)));
12397
12398 ins_cost(1.9 * INSN_COST);
12399 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
12400
12401 ins_encode %{
12402 __ eorw(as_Register($dst$$reg),
12403 as_Register($src1$$reg),
12404 as_Register($src2$$reg),
12405 Assembler::ROR,
12406 $src3$$constant & 0x1f);
12407 %}
12408
12409 ins_pipe(ialu_reg_reg_shift);
12410 %}
12411
12412 // This pattern is automatically generated from aarch64_ad.m4
12413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12414 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12415 iRegL src1, iRegL src2,
12416 immI src3) %{
12417 match(Set dst (XorL src1 (RotateRight src2 src3)));
12418
12419 ins_cost(1.9 * INSN_COST);
12420 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
12421
12422 ins_encode %{
12423 __ eor(as_Register($dst$$reg),
12424 as_Register($src1$$reg),
12425 as_Register($src2$$reg),
12426 Assembler::ROR,
12427 $src3$$constant & 0x3f);
12428 %}
12429
12430 ins_pipe(ialu_reg_reg_shift);
12431 %}
12432
12433 // This pattern is automatically generated from aarch64_ad.m4
12434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12435 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12436 iRegIorL2I src1, iRegIorL2I src2,
12437 immI src3) %{
12438 match(Set dst (OrI src1 (URShiftI src2 src3)));
12439
12440 ins_cost(1.9 * INSN_COST);
12441 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
12442
12443 ins_encode %{
12444 __ orrw(as_Register($dst$$reg),
12445 as_Register($src1$$reg),
12446 as_Register($src2$$reg),
12447 Assembler::LSR,
12448 $src3$$constant & 0x1f);
12449 %}
12450
12451 ins_pipe(ialu_reg_reg_shift);
12452 %}
12453
12454 // This pattern is automatically generated from aarch64_ad.m4
12455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12456 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12457 iRegL src1, iRegL src2,
12458 immI src3) %{
12459 match(Set dst (OrL src1 (URShiftL src2 src3)));
12460
12461 ins_cost(1.9 * INSN_COST);
12462 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12463
12464 ins_encode %{
12465 __ orr(as_Register($dst$$reg),
12466 as_Register($src1$$reg),
12467 as_Register($src2$$reg),
12468 Assembler::LSR,
12469 $src3$$constant & 0x3f);
12470 %}
12471
12472 ins_pipe(ialu_reg_reg_shift);
12473 %}
12474
12475 // This pattern is automatically generated from aarch64_ad.m4
12476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12477 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12478 iRegIorL2I src1, iRegIorL2I src2,
12479 immI src3) %{
12480 match(Set dst (OrI src1 (RShiftI src2 src3)));
12481
12482 ins_cost(1.9 * INSN_COST);
12483 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12484
12485 ins_encode %{
12486 __ orrw(as_Register($dst$$reg),
12487 as_Register($src1$$reg),
12488 as_Register($src2$$reg),
12489 Assembler::ASR,
12490 $src3$$constant & 0x1f);
12491 %}
12492
12493 ins_pipe(ialu_reg_reg_shift);
12494 %}
12495
12496 // This pattern is automatically generated from aarch64_ad.m4
12497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12498 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12499 iRegL src1, iRegL src2,
12500 immI src3) %{
12501 match(Set dst (OrL src1 (RShiftL src2 src3)));
12502
12503 ins_cost(1.9 * INSN_COST);
12504 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12505
12506 ins_encode %{
12507 __ orr(as_Register($dst$$reg),
12508 as_Register($src1$$reg),
12509 as_Register($src2$$reg),
12510 Assembler::ASR,
12511 $src3$$constant & 0x3f);
12512 %}
12513
12514 ins_pipe(ialu_reg_reg_shift);
12515 %}
12516
12517 // This pattern is automatically generated from aarch64_ad.m4
12518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12519 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12520 iRegIorL2I src1, iRegIorL2I src2,
12521 immI src3) %{
12522 match(Set dst (OrI src1 (LShiftI src2 src3)));
12523
12524 ins_cost(1.9 * INSN_COST);
12525 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12526
12527 ins_encode %{
12528 __ orrw(as_Register($dst$$reg),
12529 as_Register($src1$$reg),
12530 as_Register($src2$$reg),
12531 Assembler::LSL,
12532 $src3$$constant & 0x1f);
12533 %}
12534
12535 ins_pipe(ialu_reg_reg_shift);
12536 %}
12537
12538 // This pattern is automatically generated from aarch64_ad.m4
12539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12540 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12541 iRegL src1, iRegL src2,
12542 immI src3) %{
12543 match(Set dst (OrL src1 (LShiftL src2 src3)));
12544
12545 ins_cost(1.9 * INSN_COST);
12546 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12547
12548 ins_encode %{
12549 __ orr(as_Register($dst$$reg),
12550 as_Register($src1$$reg),
12551 as_Register($src2$$reg),
12552 Assembler::LSL,
12553 $src3$$constant & 0x3f);
12554 %}
12555
12556 ins_pipe(ialu_reg_reg_shift);
12557 %}
12558
12559 // This pattern is automatically generated from aarch64_ad.m4
12560 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12561 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12562 iRegIorL2I src1, iRegIorL2I src2,
12563 immI src3) %{
12564 match(Set dst (OrI src1 (RotateRight src2 src3)));
12565
12566 ins_cost(1.9 * INSN_COST);
12567 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12568
12569 ins_encode %{
12570 __ orrw(as_Register($dst$$reg),
12571 as_Register($src1$$reg),
12572 as_Register($src2$$reg),
12573 Assembler::ROR,
12574 $src3$$constant & 0x1f);
12575 %}
12576
12577 ins_pipe(ialu_reg_reg_shift);
12578 %}
12579
12580 // This pattern is automatically generated from aarch64_ad.m4
12581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12582 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12583 iRegL src1, iRegL src2,
12584 immI src3) %{
12585 match(Set dst (OrL src1 (RotateRight src2 src3)));
12586
12587 ins_cost(1.9 * INSN_COST);
12588 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12589
12590 ins_encode %{
12591 __ orr(as_Register($dst$$reg),
12592 as_Register($src1$$reg),
12593 as_Register($src2$$reg),
12594 Assembler::ROR,
12595 $src3$$constant & 0x3f);
12596 %}
12597
12598 ins_pipe(ialu_reg_reg_shift);
12599 %}
12600
12601 // This pattern is automatically generated from aarch64_ad.m4
12602 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12603 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12604 iRegIorL2I src1, iRegIorL2I src2,
12605 immI src3) %{
12606 match(Set dst (AddI src1 (URShiftI src2 src3)));
12607
12608 ins_cost(1.9 * INSN_COST);
12609 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12610
12611 ins_encode %{
12612 __ addw(as_Register($dst$$reg),
12613 as_Register($src1$$reg),
12614 as_Register($src2$$reg),
12615 Assembler::LSR,
12616 $src3$$constant & 0x1f);
12617 %}
12618
12619 ins_pipe(ialu_reg_reg_shift);
12620 %}
12621
12622 // This pattern is automatically generated from aarch64_ad.m4
12623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12624 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12625 iRegL src1, iRegL src2,
12626 immI src3) %{
12627 match(Set dst (AddL src1 (URShiftL src2 src3)));
12628
12629 ins_cost(1.9 * INSN_COST);
12630 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12631
12632 ins_encode %{
12633 __ add(as_Register($dst$$reg),
12634 as_Register($src1$$reg),
12635 as_Register($src2$$reg),
12636 Assembler::LSR,
12637 $src3$$constant & 0x3f);
12638 %}
12639
12640 ins_pipe(ialu_reg_reg_shift);
12641 %}
12642
12643 // This pattern is automatically generated from aarch64_ad.m4
12644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12645 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12646 iRegIorL2I src1, iRegIorL2I src2,
12647 immI src3) %{
12648 match(Set dst (AddI src1 (RShiftI src2 src3)));
12649
12650 ins_cost(1.9 * INSN_COST);
12651 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12652
12653 ins_encode %{
12654 __ addw(as_Register($dst$$reg),
12655 as_Register($src1$$reg),
12656 as_Register($src2$$reg),
12657 Assembler::ASR,
12658 $src3$$constant & 0x1f);
12659 %}
12660
12661 ins_pipe(ialu_reg_reg_shift);
12662 %}
12663
12664 // This pattern is automatically generated from aarch64_ad.m4
12665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12666 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12667 iRegL src1, iRegL src2,
12668 immI src3) %{
12669 match(Set dst (AddL src1 (RShiftL src2 src3)));
12670
12671 ins_cost(1.9 * INSN_COST);
12672 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12673
12674 ins_encode %{
12675 __ add(as_Register($dst$$reg),
12676 as_Register($src1$$reg),
12677 as_Register($src2$$reg),
12678 Assembler::ASR,
12679 $src3$$constant & 0x3f);
12680 %}
12681
12682 ins_pipe(ialu_reg_reg_shift);
12683 %}
12684
12685 // This pattern is automatically generated from aarch64_ad.m4
12686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12687 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12688 iRegIorL2I src1, iRegIorL2I src2,
12689 immI src3) %{
12690 match(Set dst (AddI src1 (LShiftI src2 src3)));
12691
12692 ins_cost(1.9 * INSN_COST);
12693 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12694
12695 ins_encode %{
12696 __ addw(as_Register($dst$$reg),
12697 as_Register($src1$$reg),
12698 as_Register($src2$$reg),
12699 Assembler::LSL,
12700 $src3$$constant & 0x1f);
12701 %}
12702
12703 ins_pipe(ialu_reg_reg_shift);
12704 %}
12705
12706 // This pattern is automatically generated from aarch64_ad.m4
12707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12708 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12709 iRegL src1, iRegL src2,
12710 immI src3) %{
12711 match(Set dst (AddL src1 (LShiftL src2 src3)));
12712
12713 ins_cost(1.9 * INSN_COST);
12714 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12715
12716 ins_encode %{
12717 __ add(as_Register($dst$$reg),
12718 as_Register($src1$$reg),
12719 as_Register($src2$$reg),
12720 Assembler::LSL,
12721 $src3$$constant & 0x3f);
12722 %}
12723
12724 ins_pipe(ialu_reg_reg_shift);
12725 %}
12726
12727 // This pattern is automatically generated from aarch64_ad.m4
12728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12729 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12730 iRegIorL2I src1, iRegIorL2I src2,
12731 immI src3) %{
12732 match(Set dst (SubI src1 (URShiftI src2 src3)));
12733
12734 ins_cost(1.9 * INSN_COST);
12735 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12736
12737 ins_encode %{
12738 __ subw(as_Register($dst$$reg),
12739 as_Register($src1$$reg),
12740 as_Register($src2$$reg),
12741 Assembler::LSR,
12742 $src3$$constant & 0x1f);
12743 %}
12744
12745 ins_pipe(ialu_reg_reg_shift);
12746 %}
12747
12748 // This pattern is automatically generated from aarch64_ad.m4
12749 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12750 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12751 iRegL src1, iRegL src2,
12752 immI src3) %{
12753 match(Set dst (SubL src1 (URShiftL src2 src3)));
12754
12755 ins_cost(1.9 * INSN_COST);
12756 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12757
12758 ins_encode %{
12759 __ sub(as_Register($dst$$reg),
12760 as_Register($src1$$reg),
12761 as_Register($src2$$reg),
12762 Assembler::LSR,
12763 $src3$$constant & 0x3f);
12764 %}
12765
12766 ins_pipe(ialu_reg_reg_shift);
12767 %}
12768
12769 // This pattern is automatically generated from aarch64_ad.m4
12770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12771 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12772 iRegIorL2I src1, iRegIorL2I src2,
12773 immI src3) %{
12774 match(Set dst (SubI src1 (RShiftI src2 src3)));
12775
12776 ins_cost(1.9 * INSN_COST);
12777 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12778
12779 ins_encode %{
12780 __ subw(as_Register($dst$$reg),
12781 as_Register($src1$$reg),
12782 as_Register($src2$$reg),
12783 Assembler::ASR,
12784 $src3$$constant & 0x1f);
12785 %}
12786
12787 ins_pipe(ialu_reg_reg_shift);
12788 %}
12789
12790 // This pattern is automatically generated from aarch64_ad.m4
12791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12792 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12793 iRegL src1, iRegL src2,
12794 immI src3) %{
12795 match(Set dst (SubL src1 (RShiftL src2 src3)));
12796
12797 ins_cost(1.9 * INSN_COST);
12798 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12799
12800 ins_encode %{
12801 __ sub(as_Register($dst$$reg),
12802 as_Register($src1$$reg),
12803 as_Register($src2$$reg),
12804 Assembler::ASR,
12805 $src3$$constant & 0x3f);
12806 %}
12807
12808 ins_pipe(ialu_reg_reg_shift);
12809 %}
12810
12811 // This pattern is automatically generated from aarch64_ad.m4
12812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12813 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12814 iRegIorL2I src1, iRegIorL2I src2,
12815 immI src3) %{
12816 match(Set dst (SubI src1 (LShiftI src2 src3)));
12817
12818 ins_cost(1.9 * INSN_COST);
12819 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12820
12821 ins_encode %{
12822 __ subw(as_Register($dst$$reg),
12823 as_Register($src1$$reg),
12824 as_Register($src2$$reg),
12825 Assembler::LSL,
12826 $src3$$constant & 0x1f);
12827 %}
12828
12829 ins_pipe(ialu_reg_reg_shift);
12830 %}
12831
12832 // This pattern is automatically generated from aarch64_ad.m4
12833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12834 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12835 iRegL src1, iRegL src2,
12836 immI src3) %{
12837 match(Set dst (SubL src1 (LShiftL src2 src3)));
12838
12839 ins_cost(1.9 * INSN_COST);
12840 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12841
12842 ins_encode %{
12843 __ sub(as_Register($dst$$reg),
12844 as_Register($src1$$reg),
12845 as_Register($src2$$reg),
12846 Assembler::LSL,
12847 $src3$$constant & 0x3f);
12848 %}
12849
12850 ins_pipe(ialu_reg_reg_shift);
12851 %}
12852
12853 // This pattern is automatically generated from aarch64_ad.m4
12854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12855
12856 // Shift Left followed by Shift Right.
12857 // This idiom is used by the compiler for the i2b bytecode etc.
12858 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12859 %{
12860 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12861 ins_cost(INSN_COST * 2);
12862 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12863 ins_encode %{
12864 int lshift = $lshift_count$$constant & 63;
12865 int rshift = $rshift_count$$constant & 63;
12866 int s = 63 - lshift;
12867 int r = (rshift - lshift) & 63;
12868 __ sbfm(as_Register($dst$$reg),
12869 as_Register($src$$reg),
12870 r, s);
12871 %}
12872
12873 ins_pipe(ialu_reg_shift);
12874 %}
12875
12876 // This pattern is automatically generated from aarch64_ad.m4
12877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12878
12879 // Shift Left followed by Shift Right.
12880 // This idiom is used by the compiler for the i2b bytecode etc.
12881 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12882 %{
12883 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12884 ins_cost(INSN_COST * 2);
12885 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12886 ins_encode %{
12887 int lshift = $lshift_count$$constant & 31;
12888 int rshift = $rshift_count$$constant & 31;
12889 int s = 31 - lshift;
12890 int r = (rshift - lshift) & 31;
12891 __ sbfmw(as_Register($dst$$reg),
12892 as_Register($src$$reg),
12893 r, s);
12894 %}
12895
12896 ins_pipe(ialu_reg_shift);
12897 %}
12898
12899 // This pattern is automatically generated from aarch64_ad.m4
12900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12901
12902 // Shift Left followed by Shift Right.
12903 // This idiom is used by the compiler for the i2b bytecode etc.
12904 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12905 %{
12906 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12907 ins_cost(INSN_COST * 2);
12908 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12909 ins_encode %{
12910 int lshift = $lshift_count$$constant & 63;
12911 int rshift = $rshift_count$$constant & 63;
12912 int s = 63 - lshift;
12913 int r = (rshift - lshift) & 63;
12914 __ ubfm(as_Register($dst$$reg),
12915 as_Register($src$$reg),
12916 r, s);
12917 %}
12918
12919 ins_pipe(ialu_reg_shift);
12920 %}
12921
12922 // This pattern is automatically generated from aarch64_ad.m4
12923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12924
12925 // Shift Left followed by Shift Right.
12926 // This idiom is used by the compiler for the i2b bytecode etc.
12927 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12928 %{
12929 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12930 ins_cost(INSN_COST * 2);
12931 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12932 ins_encode %{
12933 int lshift = $lshift_count$$constant & 31;
12934 int rshift = $rshift_count$$constant & 31;
12935 int s = 31 - lshift;
12936 int r = (rshift - lshift) & 31;
12937 __ ubfmw(as_Register($dst$$reg),
12938 as_Register($src$$reg),
12939 r, s);
12940 %}
12941
12942 ins_pipe(ialu_reg_shift);
12943 %}
12944
12945 // Bitfield extract with shift & mask
12946
12947 // This pattern is automatically generated from aarch64_ad.m4
12948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12949 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12950 %{
12951 match(Set dst (AndI (URShiftI src rshift) mask));
12952 // Make sure we are not going to exceed what ubfxw can do.
12953 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12954
12955 ins_cost(INSN_COST);
12956 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12957 ins_encode %{
12958 int rshift = $rshift$$constant & 31;
12959 intptr_t mask = $mask$$constant;
12960 int width = exact_log2(mask+1);
12961 __ ubfxw(as_Register($dst$$reg),
12962 as_Register($src$$reg), rshift, width);
12963 %}
12964 ins_pipe(ialu_reg_shift);
12965 %}
12966
12967 // This pattern is automatically generated from aarch64_ad.m4
12968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12969 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12970 %{
12971 match(Set dst (AndL (URShiftL src rshift) mask));
12972 // Make sure we are not going to exceed what ubfx can do.
12973 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12974
12975 ins_cost(INSN_COST);
12976 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12977 ins_encode %{
12978 int rshift = $rshift$$constant & 63;
12979 intptr_t mask = $mask$$constant;
12980 int width = exact_log2_long(mask+1);
12981 __ ubfx(as_Register($dst$$reg),
12982 as_Register($src$$reg), rshift, width);
12983 %}
12984 ins_pipe(ialu_reg_shift);
12985 %}
12986
12987
12988 // This pattern is automatically generated from aarch64_ad.m4
12989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12990
12991 // We can use ubfx when extending an And with a mask when we know mask
12992 // is positive. We know that because immI_bitmask guarantees it.
12993 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12994 %{
12995 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12996 // Make sure we are not going to exceed what ubfxw can do.
12997 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12998
12999 ins_cost(INSN_COST * 2);
13000 format %{ "ubfx $dst, $src, $rshift, $mask" %}
13001 ins_encode %{
13002 int rshift = $rshift$$constant & 31;
13003 intptr_t mask = $mask$$constant;
13004 int width = exact_log2(mask+1);
13005 __ ubfx(as_Register($dst$$reg),
13006 as_Register($src$$reg), rshift, width);
13007 %}
13008 ins_pipe(ialu_reg_shift);
13009 %}
13010
13011
13012 // This pattern is automatically generated from aarch64_ad.m4
13013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13014
13015 // We can use ubfiz when masking by a positive number and then left shifting the result.
13016 // We know that the mask is positive because immI_bitmask guarantees it.
13017 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13018 %{
13019 match(Set dst (LShiftI (AndI src mask) lshift));
13020 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
13021
13022 ins_cost(INSN_COST);
13023 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13024 ins_encode %{
13025 int lshift = $lshift$$constant & 31;
13026 intptr_t mask = $mask$$constant;
13027 int width = exact_log2(mask+1);
13028 __ ubfizw(as_Register($dst$$reg),
13029 as_Register($src$$reg), lshift, width);
13030 %}
13031 ins_pipe(ialu_reg_shift);
13032 %}
13033
13034 // This pattern is automatically generated from aarch64_ad.m4
13035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13036
13037 // We can use ubfiz when masking by a positive number and then left shifting the result.
13038 // We know that the mask is positive because immL_bitmask guarantees it.
13039 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13040 %{
13041 match(Set dst (LShiftL (AndL src mask) lshift));
13042 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13043
13044 ins_cost(INSN_COST);
13045 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13046 ins_encode %{
13047 int lshift = $lshift$$constant & 63;
13048 intptr_t mask = $mask$$constant;
13049 int width = exact_log2_long(mask+1);
13050 __ ubfiz(as_Register($dst$$reg),
13051 as_Register($src$$reg), lshift, width);
13052 %}
13053 ins_pipe(ialu_reg_shift);
13054 %}
13055
13056 // This pattern is automatically generated from aarch64_ad.m4
13057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13058
13059 // We can use ubfiz when masking by a positive number and then left shifting the result.
13060 // We know that the mask is positive because immI_bitmask guarantees it.
13061 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13062 %{
13063 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13064 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13065
13066 ins_cost(INSN_COST);
13067 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13068 ins_encode %{
13069 int lshift = $lshift$$constant & 31;
13070 intptr_t mask = $mask$$constant;
13071 int width = exact_log2(mask+1);
13072 __ ubfizw(as_Register($dst$$reg),
13073 as_Register($src$$reg), lshift, width);
13074 %}
13075 ins_pipe(ialu_reg_shift);
13076 %}
13077
13078 // This pattern is automatically generated from aarch64_ad.m4
13079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13080
13081 // We can use ubfiz when masking by a positive number and then left shifting the result.
13082 // We know that the mask is positive because immL_bitmask guarantees it.
13083 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13084 %{
13085 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13086 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13087
13088 ins_cost(INSN_COST);
13089 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13090 ins_encode %{
13091 int lshift = $lshift$$constant & 63;
13092 intptr_t mask = $mask$$constant;
13093 int width = exact_log2_long(mask+1);
13094 __ ubfiz(as_Register($dst$$reg),
13095 as_Register($src$$reg), lshift, width);
13096 %}
13097 ins_pipe(ialu_reg_shift);
13098 %}
13099
13100
13101 // This pattern is automatically generated from aarch64_ad.m4
13102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13103
13104 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13105 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13106 %{
13107 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13108 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13109
13110 ins_cost(INSN_COST);
13111 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13112 ins_encode %{
13113 int lshift = $lshift$$constant & 63;
13114 intptr_t mask = $mask$$constant;
13115 int width = exact_log2(mask+1);
13116 __ ubfiz(as_Register($dst$$reg),
13117 as_Register($src$$reg), lshift, width);
13118 %}
13119 ins_pipe(ialu_reg_shift);
13120 %}
13121
13122 // This pattern is automatically generated from aarch64_ad.m4
13123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13124
13125 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13126 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13127 %{
13128 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13129 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13130
13131 ins_cost(INSN_COST);
13132 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13133 ins_encode %{
13134 int lshift = $lshift$$constant & 31;
13135 intptr_t mask = $mask$$constant;
13136 int width = exact_log2(mask+1);
13137 __ ubfiz(as_Register($dst$$reg),
13138 as_Register($src$$reg), lshift, width);
13139 %}
13140 ins_pipe(ialu_reg_shift);
13141 %}
13142
13143 // This pattern is automatically generated from aarch64_ad.m4
13144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13145
13146 // Can skip int2long conversions after AND with small bitmask
13147 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13148 %{
13149 match(Set dst (ConvI2L (AndI src msk)));
13150 ins_cost(INSN_COST);
13151 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13152 ins_encode %{
13153 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13154 %}
13155 ins_pipe(ialu_reg_shift);
13156 %}
13157
13158
13159 // Rotations
13160
13161 // This pattern is automatically generated from aarch64_ad.m4
13162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13163 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13164 %{
13165 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13166 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13167
13168 ins_cost(INSN_COST);
13169 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13170
13171 ins_encode %{
13172 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13173 $rshift$$constant & 63);
13174 %}
13175 ins_pipe(ialu_reg_reg_extr);
13176 %}
13177
13178
13179 // This pattern is automatically generated from aarch64_ad.m4
13180 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13181 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13182 %{
13183 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13184 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13185
13186 ins_cost(INSN_COST);
13187 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13188
13189 ins_encode %{
13190 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13191 $rshift$$constant & 31);
13192 %}
13193 ins_pipe(ialu_reg_reg_extr);
13194 %}
13195
13196
13197 // This pattern is automatically generated from aarch64_ad.m4
13198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13199 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13200 %{
13201 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13202 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13203
13204 ins_cost(INSN_COST);
13205 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13206
13207 ins_encode %{
13208 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13209 $rshift$$constant & 63);
13210 %}
13211 ins_pipe(ialu_reg_reg_extr);
13212 %}
13213
13214
13215 // This pattern is automatically generated from aarch64_ad.m4
13216 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13217 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13218 %{
13219 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13220 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13221
13222 ins_cost(INSN_COST);
13223 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13224
13225 ins_encode %{
13226 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13227 $rshift$$constant & 31);
13228 %}
13229 ins_pipe(ialu_reg_reg_extr);
13230 %}
13231
13232 // This pattern is automatically generated from aarch64_ad.m4
13233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13234 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13235 %{
13236 match(Set dst (RotateRight src shift));
13237
13238 ins_cost(INSN_COST);
13239 format %{ "ror $dst, $src, $shift" %}
13240
13241 ins_encode %{
13242 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13243 $shift$$constant & 0x1f);
13244 %}
13245 ins_pipe(ialu_reg_reg_vshift);
13246 %}
13247
13248 // This pattern is automatically generated from aarch64_ad.m4
13249 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13250 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13251 %{
13252 match(Set dst (RotateRight src shift));
13253
13254 ins_cost(INSN_COST);
13255 format %{ "ror $dst, $src, $shift" %}
13256
13257 ins_encode %{
13258 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13259 $shift$$constant & 0x3f);
13260 %}
13261 ins_pipe(ialu_reg_reg_vshift);
13262 %}
13263
13264 // This pattern is automatically generated from aarch64_ad.m4
13265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13266 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13267 %{
13268 match(Set dst (RotateRight src shift));
13269
13270 ins_cost(INSN_COST);
13271 format %{ "ror $dst, $src, $shift" %}
13272
13273 ins_encode %{
13274 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13275 %}
13276 ins_pipe(ialu_reg_reg_vshift);
13277 %}
13278
13279 // This pattern is automatically generated from aarch64_ad.m4
13280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13281 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13282 %{
13283 match(Set dst (RotateRight src shift));
13284
13285 ins_cost(INSN_COST);
13286 format %{ "ror $dst, $src, $shift" %}
13287
13288 ins_encode %{
13289 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13290 %}
13291 ins_pipe(ialu_reg_reg_vshift);
13292 %}
13293
13294 // This pattern is automatically generated from aarch64_ad.m4
13295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13296 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13297 %{
13298 match(Set dst (RotateLeft src shift));
13299
13300 ins_cost(INSN_COST);
13301 format %{ "rol $dst, $src, $shift" %}
13302
13303 ins_encode %{
13304 __ subw(rscratch1, zr, as_Register($shift$$reg));
13305 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13306 %}
13307 ins_pipe(ialu_reg_reg_vshift);
13308 %}
13309
13310 // This pattern is automatically generated from aarch64_ad.m4
13311 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13312 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13313 %{
13314 match(Set dst (RotateLeft src shift));
13315
13316 ins_cost(INSN_COST);
13317 format %{ "rol $dst, $src, $shift" %}
13318
13319 ins_encode %{
13320 __ subw(rscratch1, zr, as_Register($shift$$reg));
13321 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13322 %}
13323 ins_pipe(ialu_reg_reg_vshift);
13324 %}
13325
13326
13327 // Add/subtract (extended)
13328
13329 // This pattern is automatically generated from aarch64_ad.m4
13330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13331 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13332 %{
13333 match(Set dst (AddL src1 (ConvI2L src2)));
13334 ins_cost(INSN_COST);
13335 format %{ "add $dst, $src1, $src2, sxtw" %}
13336
13337 ins_encode %{
13338 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13339 as_Register($src2$$reg), ext::sxtw);
13340 %}
13341 ins_pipe(ialu_reg_reg);
13342 %}
13343
13344 // This pattern is automatically generated from aarch64_ad.m4
13345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13346 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13347 %{
13348 match(Set dst (SubL src1 (ConvI2L src2)));
13349 ins_cost(INSN_COST);
13350 format %{ "sub $dst, $src1, $src2, sxtw" %}
13351
13352 ins_encode %{
13353 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13354 as_Register($src2$$reg), ext::sxtw);
13355 %}
13356 ins_pipe(ialu_reg_reg);
13357 %}
13358
13359 // This pattern is automatically generated from aarch64_ad.m4
13360 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13361 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13362 %{
13363 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13364 ins_cost(INSN_COST);
13365 format %{ "add $dst, $src1, $src2, sxth" %}
13366
13367 ins_encode %{
13368 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13369 as_Register($src2$$reg), ext::sxth);
13370 %}
13371 ins_pipe(ialu_reg_reg);
13372 %}
13373
13374 // This pattern is automatically generated from aarch64_ad.m4
13375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13376 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13377 %{
13378 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13379 ins_cost(INSN_COST);
13380 format %{ "add $dst, $src1, $src2, sxtb" %}
13381
13382 ins_encode %{
13383 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13384 as_Register($src2$$reg), ext::sxtb);
13385 %}
13386 ins_pipe(ialu_reg_reg);
13387 %}
13388
13389 // This pattern is automatically generated from aarch64_ad.m4
13390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13391 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13392 %{
13393 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13394 ins_cost(INSN_COST);
13395 format %{ "add $dst, $src1, $src2, uxtb" %}
13396
13397 ins_encode %{
13398 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13399 as_Register($src2$$reg), ext::uxtb);
13400 %}
13401 ins_pipe(ialu_reg_reg);
13402 %}
13403
13404 // This pattern is automatically generated from aarch64_ad.m4
13405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13406 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13407 %{
13408 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13409 ins_cost(INSN_COST);
13410 format %{ "add $dst, $src1, $src2, sxth" %}
13411
13412 ins_encode %{
13413 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13414 as_Register($src2$$reg), ext::sxth);
13415 %}
13416 ins_pipe(ialu_reg_reg);
13417 %}
13418
13419 // This pattern is automatically generated from aarch64_ad.m4
13420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13421 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13422 %{
13423 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13424 ins_cost(INSN_COST);
13425 format %{ "add $dst, $src1, $src2, sxtw" %}
13426
13427 ins_encode %{
13428 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13429 as_Register($src2$$reg), ext::sxtw);
13430 %}
13431 ins_pipe(ialu_reg_reg);
13432 %}
13433
13434 // This pattern is automatically generated from aarch64_ad.m4
13435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13436 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13437 %{
13438 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13439 ins_cost(INSN_COST);
13440 format %{ "add $dst, $src1, $src2, sxtb" %}
13441
13442 ins_encode %{
13443 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13444 as_Register($src2$$reg), ext::sxtb);
13445 %}
13446 ins_pipe(ialu_reg_reg);
13447 %}
13448
13449 // This pattern is automatically generated from aarch64_ad.m4
13450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13451 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13452 %{
13453 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13454 ins_cost(INSN_COST);
13455 format %{ "add $dst, $src1, $src2, uxtb" %}
13456
13457 ins_encode %{
13458 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13459 as_Register($src2$$reg), ext::uxtb);
13460 %}
13461 ins_pipe(ialu_reg_reg);
13462 %}
13463
13464 // This pattern is automatically generated from aarch64_ad.m4
13465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13466 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13467 %{
13468 match(Set dst (AddI src1 (AndI src2 mask)));
13469 ins_cost(INSN_COST);
13470 format %{ "addw $dst, $src1, $src2, uxtb" %}
13471
13472 ins_encode %{
13473 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13474 as_Register($src2$$reg), ext::uxtb);
13475 %}
13476 ins_pipe(ialu_reg_reg);
13477 %}
13478
13479 // This pattern is automatically generated from aarch64_ad.m4
13480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13481 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13482 %{
13483 match(Set dst (AddI src1 (AndI src2 mask)));
13484 ins_cost(INSN_COST);
13485 format %{ "addw $dst, $src1, $src2, uxth" %}
13486
13487 ins_encode %{
13488 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13489 as_Register($src2$$reg), ext::uxth);
13490 %}
13491 ins_pipe(ialu_reg_reg);
13492 %}
13493
13494 // This pattern is automatically generated from aarch64_ad.m4
13495 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13496 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13497 %{
13498 match(Set dst (AddL src1 (AndL src2 mask)));
13499 ins_cost(INSN_COST);
13500 format %{ "add $dst, $src1, $src2, uxtb" %}
13501
13502 ins_encode %{
13503 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13504 as_Register($src2$$reg), ext::uxtb);
13505 %}
13506 ins_pipe(ialu_reg_reg);
13507 %}
13508
13509 // This pattern is automatically generated from aarch64_ad.m4
13510 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13511 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13512 %{
13513 match(Set dst (AddL src1 (AndL src2 mask)));
13514 ins_cost(INSN_COST);
13515 format %{ "add $dst, $src1, $src2, uxth" %}
13516
13517 ins_encode %{
13518 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13519 as_Register($src2$$reg), ext::uxth);
13520 %}
13521 ins_pipe(ialu_reg_reg);
13522 %}
13523
13524 // This pattern is automatically generated from aarch64_ad.m4
13525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13526 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13527 %{
13528 match(Set dst (AddL src1 (AndL src2 mask)));
13529 ins_cost(INSN_COST);
13530 format %{ "add $dst, $src1, $src2, uxtw" %}
13531
13532 ins_encode %{
13533 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13534 as_Register($src2$$reg), ext::uxtw);
13535 %}
13536 ins_pipe(ialu_reg_reg);
13537 %}
13538
13539 // This pattern is automatically generated from aarch64_ad.m4
13540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13541 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13542 %{
13543 match(Set dst (SubI src1 (AndI src2 mask)));
13544 ins_cost(INSN_COST);
13545 format %{ "subw $dst, $src1, $src2, uxtb" %}
13546
13547 ins_encode %{
13548 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13549 as_Register($src2$$reg), ext::uxtb);
13550 %}
13551 ins_pipe(ialu_reg_reg);
13552 %}
13553
13554 // This pattern is automatically generated from aarch64_ad.m4
13555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13556 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13557 %{
13558 match(Set dst (SubI src1 (AndI src2 mask)));
13559 ins_cost(INSN_COST);
13560 format %{ "subw $dst, $src1, $src2, uxth" %}
13561
13562 ins_encode %{
13563 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13564 as_Register($src2$$reg), ext::uxth);
13565 %}
13566 ins_pipe(ialu_reg_reg);
13567 %}
13568
13569 // This pattern is automatically generated from aarch64_ad.m4
13570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13571 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13572 %{
13573 match(Set dst (SubL src1 (AndL src2 mask)));
13574 ins_cost(INSN_COST);
13575 format %{ "sub $dst, $src1, $src2, uxtb" %}
13576
13577 ins_encode %{
13578 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13579 as_Register($src2$$reg), ext::uxtb);
13580 %}
13581 ins_pipe(ialu_reg_reg);
13582 %}
13583
13584 // This pattern is automatically generated from aarch64_ad.m4
13585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13586 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13587 %{
13588 match(Set dst (SubL src1 (AndL src2 mask)));
13589 ins_cost(INSN_COST);
13590 format %{ "sub $dst, $src1, $src2, uxth" %}
13591
13592 ins_encode %{
13593 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13594 as_Register($src2$$reg), ext::uxth);
13595 %}
13596 ins_pipe(ialu_reg_reg);
13597 %}
13598
13599 // This pattern is automatically generated from aarch64_ad.m4
13600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13601 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13602 %{
13603 match(Set dst (SubL src1 (AndL src2 mask)));
13604 ins_cost(INSN_COST);
13605 format %{ "sub $dst, $src1, $src2, uxtw" %}
13606
13607 ins_encode %{
13608 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13609 as_Register($src2$$reg), ext::uxtw);
13610 %}
13611 ins_pipe(ialu_reg_reg);
13612 %}
13613
13614
13615 // This pattern is automatically generated from aarch64_ad.m4
13616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13617 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13618 %{
13619 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13620 ins_cost(1.9 * INSN_COST);
13621 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13622
13623 ins_encode %{
13624 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13625 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13626 %}
13627 ins_pipe(ialu_reg_reg_shift);
13628 %}
13629
13630 // This pattern is automatically generated from aarch64_ad.m4
13631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13632 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13633 %{
13634 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13635 ins_cost(1.9 * INSN_COST);
13636 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13637
13638 ins_encode %{
13639 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13640 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13641 %}
13642 ins_pipe(ialu_reg_reg_shift);
13643 %}
13644
13645 // This pattern is automatically generated from aarch64_ad.m4
13646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13647 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13648 %{
13649 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13650 ins_cost(1.9 * INSN_COST);
13651 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13652
13653 ins_encode %{
13654 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13655 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13656 %}
13657 ins_pipe(ialu_reg_reg_shift);
13658 %}
13659
13660 // This pattern is automatically generated from aarch64_ad.m4
13661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13662 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13663 %{
13664 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13665 ins_cost(1.9 * INSN_COST);
13666 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13667
13668 ins_encode %{
13669 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13670 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13671 %}
13672 ins_pipe(ialu_reg_reg_shift);
13673 %}
13674
13675 // This pattern is automatically generated from aarch64_ad.m4
13676 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13677 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13678 %{
13679 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13680 ins_cost(1.9 * INSN_COST);
13681 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13682
13683 ins_encode %{
13684 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13685 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13686 %}
13687 ins_pipe(ialu_reg_reg_shift);
13688 %}
13689
13690 // This pattern is automatically generated from aarch64_ad.m4
13691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13692 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13693 %{
13694 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13695 ins_cost(1.9 * INSN_COST);
13696 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13697
13698 ins_encode %{
13699 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13700 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13701 %}
13702 ins_pipe(ialu_reg_reg_shift);
13703 %}
13704
13705 // This pattern is automatically generated from aarch64_ad.m4
13706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13707 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13708 %{
13709 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13710 ins_cost(1.9 * INSN_COST);
13711 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13712
13713 ins_encode %{
13714 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13715 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13716 %}
13717 ins_pipe(ialu_reg_reg_shift);
13718 %}
13719
13720 // This pattern is automatically generated from aarch64_ad.m4
13721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13722 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13723 %{
13724 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13725 ins_cost(1.9 * INSN_COST);
13726 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13727
13728 ins_encode %{
13729 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13730 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13731 %}
13732 ins_pipe(ialu_reg_reg_shift);
13733 %}
13734
13735 // This pattern is automatically generated from aarch64_ad.m4
13736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13737 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13738 %{
13739 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13740 ins_cost(1.9 * INSN_COST);
13741 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13742
13743 ins_encode %{
13744 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13745 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13746 %}
13747 ins_pipe(ialu_reg_reg_shift);
13748 %}
13749
13750 // This pattern is automatically generated from aarch64_ad.m4
13751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13752 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13753 %{
13754 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13755 ins_cost(1.9 * INSN_COST);
13756 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13757
13758 ins_encode %{
13759 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13760 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13761 %}
13762 ins_pipe(ialu_reg_reg_shift);
13763 %}
13764
13765 // This pattern is automatically generated from aarch64_ad.m4
13766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13767 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13768 %{
13769 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13770 ins_cost(1.9 * INSN_COST);
13771 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13772
13773 ins_encode %{
13774 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13775 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13776 %}
13777 ins_pipe(ialu_reg_reg_shift);
13778 %}
13779
13780 // This pattern is automatically generated from aarch64_ad.m4
13781 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13782 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13783 %{
13784 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13785 ins_cost(1.9 * INSN_COST);
13786 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13787
13788 ins_encode %{
13789 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13790 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13791 %}
13792 ins_pipe(ialu_reg_reg_shift);
13793 %}
13794
13795 // This pattern is automatically generated from aarch64_ad.m4
13796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13797 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13798 %{
13799 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13800 ins_cost(1.9 * INSN_COST);
13801 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13802
13803 ins_encode %{
13804 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13805 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13806 %}
13807 ins_pipe(ialu_reg_reg_shift);
13808 %}
13809
13810 // This pattern is automatically generated from aarch64_ad.m4
13811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13812 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13813 %{
13814 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13815 ins_cost(1.9 * INSN_COST);
13816 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13817
13818 ins_encode %{
13819 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13820 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13821 %}
13822 ins_pipe(ialu_reg_reg_shift);
13823 %}
13824
13825 // This pattern is automatically generated from aarch64_ad.m4
13826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13827 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13828 %{
13829 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13830 ins_cost(1.9 * INSN_COST);
13831 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13832
13833 ins_encode %{
13834 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13835 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13836 %}
13837 ins_pipe(ialu_reg_reg_shift);
13838 %}
13839
13840 // This pattern is automatically generated from aarch64_ad.m4
13841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13842 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13843 %{
13844 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13845 ins_cost(1.9 * INSN_COST);
13846 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13847
13848 ins_encode %{
13849 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13850 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13851 %}
13852 ins_pipe(ialu_reg_reg_shift);
13853 %}
13854
13855 // This pattern is automatically generated from aarch64_ad.m4
13856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13857 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13858 %{
13859 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13860 ins_cost(1.9 * INSN_COST);
13861 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13862
13863 ins_encode %{
13864 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13865 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13866 %}
13867 ins_pipe(ialu_reg_reg_shift);
13868 %}
13869
13870 // This pattern is automatically generated from aarch64_ad.m4
13871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13872 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13873 %{
13874 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13875 ins_cost(1.9 * INSN_COST);
13876 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13877
13878 ins_encode %{
13879 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13880 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13881 %}
13882 ins_pipe(ialu_reg_reg_shift);
13883 %}
13884
13885 // This pattern is automatically generated from aarch64_ad.m4
13886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13887 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13888 %{
13889 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13890 ins_cost(1.9 * INSN_COST);
13891 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13892
13893 ins_encode %{
13894 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13895 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13896 %}
13897 ins_pipe(ialu_reg_reg_shift);
13898 %}
13899
13900 // This pattern is automatically generated from aarch64_ad.m4
13901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13902 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13903 %{
13904 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13905 ins_cost(1.9 * INSN_COST);
13906 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13907
13908 ins_encode %{
13909 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13910 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13911 %}
13912 ins_pipe(ialu_reg_reg_shift);
13913 %}
13914
13915 // This pattern is automatically generated from aarch64_ad.m4
13916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13917 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13918 %{
13919 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13920 ins_cost(1.9 * INSN_COST);
13921 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13922
13923 ins_encode %{
13924 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13925 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13926 %}
13927 ins_pipe(ialu_reg_reg_shift);
13928 %}
13929
13930 // This pattern is automatically generated from aarch64_ad.m4
13931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13932 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13933 %{
13934 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13935 ins_cost(1.9 * INSN_COST);
13936 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13937
13938 ins_encode %{
13939 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13940 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13941 %}
13942 ins_pipe(ialu_reg_reg_shift);
13943 %}
13944
13945 // This pattern is automatically generated from aarch64_ad.m4
13946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13947 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13948 %{
13949 effect(DEF dst, USE src1, USE src2, USE cr);
13950 ins_cost(INSN_COST * 2);
13951 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13952
13953 ins_encode %{
13954 __ cselw($dst$$Register,
13955 $src1$$Register,
13956 $src2$$Register,
13957 Assembler::LT);
13958 %}
13959 ins_pipe(icond_reg_reg);
13960 %}
13961
13962 // This pattern is automatically generated from aarch64_ad.m4
13963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13964 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13965 %{
13966 effect(DEF dst, USE src1, USE src2, USE cr);
13967 ins_cost(INSN_COST * 2);
13968 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13969
13970 ins_encode %{
13971 __ cselw($dst$$Register,
13972 $src1$$Register,
13973 $src2$$Register,
13974 Assembler::GT);
13975 %}
13976 ins_pipe(icond_reg_reg);
13977 %}
13978
13979 // This pattern is automatically generated from aarch64_ad.m4
13980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13981 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13982 %{
13983 effect(DEF dst, USE src1, USE cr);
13984 ins_cost(INSN_COST * 2);
13985 format %{ "cselw $dst, $src1, zr lt\t" %}
13986
13987 ins_encode %{
13988 __ cselw($dst$$Register,
13989 $src1$$Register,
13990 zr,
13991 Assembler::LT);
13992 %}
13993 ins_pipe(icond_reg);
13994 %}
13995
13996 // This pattern is automatically generated from aarch64_ad.m4
13997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13998 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13999 %{
14000 effect(DEF dst, USE src1, USE cr);
14001 ins_cost(INSN_COST * 2);
14002 format %{ "cselw $dst, $src1, zr gt\t" %}
14003
14004 ins_encode %{
14005 __ cselw($dst$$Register,
14006 $src1$$Register,
14007 zr,
14008 Assembler::GT);
14009 %}
14010 ins_pipe(icond_reg);
14011 %}
14012
14013 // This pattern is automatically generated from aarch64_ad.m4
14014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14015 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14016 %{
14017 effect(DEF dst, USE src1, USE cr);
14018 ins_cost(INSN_COST * 2);
14019 format %{ "csincw $dst, $src1, zr le\t" %}
14020
14021 ins_encode %{
14022 __ csincw($dst$$Register,
14023 $src1$$Register,
14024 zr,
14025 Assembler::LE);
14026 %}
14027 ins_pipe(icond_reg);
14028 %}
14029
14030 // This pattern is automatically generated from aarch64_ad.m4
14031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14032 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14033 %{
14034 effect(DEF dst, USE src1, USE cr);
14035 ins_cost(INSN_COST * 2);
14036 format %{ "csincw $dst, $src1, zr gt\t" %}
14037
14038 ins_encode %{
14039 __ csincw($dst$$Register,
14040 $src1$$Register,
14041 zr,
14042 Assembler::GT);
14043 %}
14044 ins_pipe(icond_reg);
14045 %}
14046
14047 // This pattern is automatically generated from aarch64_ad.m4
14048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14049 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14050 %{
14051 effect(DEF dst, USE src1, USE cr);
14052 ins_cost(INSN_COST * 2);
14053 format %{ "csinvw $dst, $src1, zr lt\t" %}
14054
14055 ins_encode %{
14056 __ csinvw($dst$$Register,
14057 $src1$$Register,
14058 zr,
14059 Assembler::LT);
14060 %}
14061 ins_pipe(icond_reg);
14062 %}
14063
14064 // This pattern is automatically generated from aarch64_ad.m4
14065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14066 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14067 %{
14068 effect(DEF dst, USE src1, USE cr);
14069 ins_cost(INSN_COST * 2);
14070 format %{ "csinvw $dst, $src1, zr ge\t" %}
14071
14072 ins_encode %{
14073 __ csinvw($dst$$Register,
14074 $src1$$Register,
14075 zr,
14076 Assembler::GE);
14077 %}
14078 ins_pipe(icond_reg);
14079 %}
14080
14081 // This pattern is automatically generated from aarch64_ad.m4
14082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14083 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14084 %{
14085 match(Set dst (MinI src imm));
14086 ins_cost(INSN_COST * 3);
14087 expand %{
14088 rFlagsReg cr;
14089 compI_reg_imm0(cr, src);
14090 cmovI_reg_imm0_lt(dst, src, cr);
14091 %}
14092 %}
14093
14094 // This pattern is automatically generated from aarch64_ad.m4
14095 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14096 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14097 %{
14098 match(Set dst (MinI imm src));
14099 ins_cost(INSN_COST * 3);
14100 expand %{
14101 rFlagsReg cr;
14102 compI_reg_imm0(cr, src);
14103 cmovI_reg_imm0_lt(dst, src, cr);
14104 %}
14105 %}
14106
14107 // This pattern is automatically generated from aarch64_ad.m4
14108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14109 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14110 %{
14111 match(Set dst (MinI src imm));
14112 ins_cost(INSN_COST * 3);
14113 expand %{
14114 rFlagsReg cr;
14115 compI_reg_imm0(cr, src);
14116 cmovI_reg_imm1_le(dst, src, cr);
14117 %}
14118 %}
14119
14120 // This pattern is automatically generated from aarch64_ad.m4
14121 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14122 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14123 %{
14124 match(Set dst (MinI imm src));
14125 ins_cost(INSN_COST * 3);
14126 expand %{
14127 rFlagsReg cr;
14128 compI_reg_imm0(cr, src);
14129 cmovI_reg_imm1_le(dst, src, cr);
14130 %}
14131 %}
14132
14133 // This pattern is automatically generated from aarch64_ad.m4
14134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14135 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14136 %{
14137 match(Set dst (MinI src imm));
14138 ins_cost(INSN_COST * 3);
14139 expand %{
14140 rFlagsReg cr;
14141 compI_reg_imm0(cr, src);
14142 cmovI_reg_immM1_lt(dst, src, cr);
14143 %}
14144 %}
14145
14146 // This pattern is automatically generated from aarch64_ad.m4
14147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14148 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14149 %{
14150 match(Set dst (MinI imm src));
14151 ins_cost(INSN_COST * 3);
14152 expand %{
14153 rFlagsReg cr;
14154 compI_reg_imm0(cr, src);
14155 cmovI_reg_immM1_lt(dst, src, cr);
14156 %}
14157 %}
14158
14159 // This pattern is automatically generated from aarch64_ad.m4
14160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14161 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14162 %{
14163 match(Set dst (MaxI src imm));
14164 ins_cost(INSN_COST * 3);
14165 expand %{
14166 rFlagsReg cr;
14167 compI_reg_imm0(cr, src);
14168 cmovI_reg_imm0_gt(dst, src, cr);
14169 %}
14170 %}
14171
14172 // This pattern is automatically generated from aarch64_ad.m4
14173 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14174 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14175 %{
14176 match(Set dst (MaxI imm src));
14177 ins_cost(INSN_COST * 3);
14178 expand %{
14179 rFlagsReg cr;
14180 compI_reg_imm0(cr, src);
14181 cmovI_reg_imm0_gt(dst, src, cr);
14182 %}
14183 %}
14184
14185 // This pattern is automatically generated from aarch64_ad.m4
14186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14187 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14188 %{
14189 match(Set dst (MaxI src imm));
14190 ins_cost(INSN_COST * 3);
14191 expand %{
14192 rFlagsReg cr;
14193 compI_reg_imm0(cr, src);
14194 cmovI_reg_imm1_gt(dst, src, cr);
14195 %}
14196 %}
14197
14198 // This pattern is automatically generated from aarch64_ad.m4
14199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14200 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14201 %{
14202 match(Set dst (MaxI imm src));
14203 ins_cost(INSN_COST * 3);
14204 expand %{
14205 rFlagsReg cr;
14206 compI_reg_imm0(cr, src);
14207 cmovI_reg_imm1_gt(dst, src, cr);
14208 %}
14209 %}
14210
14211 // This pattern is automatically generated from aarch64_ad.m4
14212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14213 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14214 %{
14215 match(Set dst (MaxI src imm));
14216 ins_cost(INSN_COST * 3);
14217 expand %{
14218 rFlagsReg cr;
14219 compI_reg_imm0(cr, src);
14220 cmovI_reg_immM1_ge(dst, src, cr);
14221 %}
14222 %}
14223
14224 // This pattern is automatically generated from aarch64_ad.m4
14225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14226 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14227 %{
14228 match(Set dst (MaxI imm src));
14229 ins_cost(INSN_COST * 3);
14230 expand %{
14231 rFlagsReg cr;
14232 compI_reg_imm0(cr, src);
14233 cmovI_reg_immM1_ge(dst, src, cr);
14234 %}
14235 %}
14236
14237 // This pattern is automatically generated from aarch64_ad.m4
14238 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14239 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
14240 %{
14241 match(Set dst (ReverseI src));
14242 ins_cost(INSN_COST);
14243 format %{ "rbitw $dst, $src" %}
14244 ins_encode %{
14245 __ rbitw($dst$$Register, $src$$Register);
14246 %}
14247 ins_pipe(ialu_reg);
14248 %}
14249
14250 // This pattern is automatically generated from aarch64_ad.m4
14251 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14252 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
14253 %{
14254 match(Set dst (ReverseL src));
14255 ins_cost(INSN_COST);
14256 format %{ "rbit $dst, $src" %}
14257 ins_encode %{
14258 __ rbit($dst$$Register, $src$$Register);
14259 %}
14260 ins_pipe(ialu_reg);
14261 %}
14262
14263
14264 // END This section of the file is automatically generated. Do not edit --------------
14265
14266
14267 // ============================================================================
14268 // Floating Point Arithmetic Instructions
14269
14270 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14271 match(Set dst (AddF src1 src2));
14272
14273 ins_cost(INSN_COST * 5);
14274 format %{ "fadds $dst, $src1, $src2" %}
14275
14276 ins_encode %{
14277 __ fadds(as_FloatRegister($dst$$reg),
14278 as_FloatRegister($src1$$reg),
14279 as_FloatRegister($src2$$reg));
14280 %}
14281
14282 ins_pipe(fp_dop_reg_reg_s);
14283 %}
14284
14285 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14286 match(Set dst (AddD src1 src2));
14287
14288 ins_cost(INSN_COST * 5);
14289 format %{ "faddd $dst, $src1, $src2" %}
14290
14291 ins_encode %{
14292 __ faddd(as_FloatRegister($dst$$reg),
14293 as_FloatRegister($src1$$reg),
14294 as_FloatRegister($src2$$reg));
14295 %}
14296
14297 ins_pipe(fp_dop_reg_reg_d);
14298 %}
14299
14300 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14301 match(Set dst (SubF src1 src2));
14302
14303 ins_cost(INSN_COST * 5);
14304 format %{ "fsubs $dst, $src1, $src2" %}
14305
14306 ins_encode %{
14307 __ fsubs(as_FloatRegister($dst$$reg),
14308 as_FloatRegister($src1$$reg),
14309 as_FloatRegister($src2$$reg));
14310 %}
14311
14312 ins_pipe(fp_dop_reg_reg_s);
14313 %}
14314
14315 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14316 match(Set dst (SubD src1 src2));
14317
14318 ins_cost(INSN_COST * 5);
14319 format %{ "fsubd $dst, $src1, $src2" %}
14320
14321 ins_encode %{
14322 __ fsubd(as_FloatRegister($dst$$reg),
14323 as_FloatRegister($src1$$reg),
14324 as_FloatRegister($src2$$reg));
14325 %}
14326
14327 ins_pipe(fp_dop_reg_reg_d);
14328 %}
14329
14330 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14331 match(Set dst (MulF src1 src2));
14332
14333 ins_cost(INSN_COST * 6);
14334 format %{ "fmuls $dst, $src1, $src2" %}
14335
14336 ins_encode %{
14337 __ fmuls(as_FloatRegister($dst$$reg),
14338 as_FloatRegister($src1$$reg),
14339 as_FloatRegister($src2$$reg));
14340 %}
14341
14342 ins_pipe(fp_dop_reg_reg_s);
14343 %}
14344
14345 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14346 match(Set dst (MulD src1 src2));
14347
14348 ins_cost(INSN_COST * 6);
14349 format %{ "fmuld $dst, $src1, $src2" %}
14350
14351 ins_encode %{
14352 __ fmuld(as_FloatRegister($dst$$reg),
14353 as_FloatRegister($src1$$reg),
14354 as_FloatRegister($src2$$reg));
14355 %}
14356
14357 ins_pipe(fp_dop_reg_reg_d);
14358 %}
14359
14360 // src1 * src2 + src3
14361 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14362 predicate(UseFMA);
14363 match(Set dst (FmaF src3 (Binary src1 src2)));
14364
14365 format %{ "fmadds $dst, $src1, $src2, $src3" %}
14366
14367 ins_encode %{
14368 __ fmadds(as_FloatRegister($dst$$reg),
14369 as_FloatRegister($src1$$reg),
14370 as_FloatRegister($src2$$reg),
14371 as_FloatRegister($src3$$reg));
14372 %}
14373
14374 ins_pipe(pipe_class_default);
14375 %}
14376
14377 // src1 * src2 + src3
14378 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14379 predicate(UseFMA);
14380 match(Set dst (FmaD src3 (Binary src1 src2)));
14381
14382 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
14383
14384 ins_encode %{
14385 __ fmaddd(as_FloatRegister($dst$$reg),
14386 as_FloatRegister($src1$$reg),
14387 as_FloatRegister($src2$$reg),
14388 as_FloatRegister($src3$$reg));
14389 %}
14390
14391 ins_pipe(pipe_class_default);
14392 %}
14393
14394 // -src1 * src2 + src3
14395 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14396 predicate(UseFMA);
14397 match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14398 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14399
14400 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
14401
14402 ins_encode %{
14403 __ fmsubs(as_FloatRegister($dst$$reg),
14404 as_FloatRegister($src1$$reg),
14405 as_FloatRegister($src2$$reg),
14406 as_FloatRegister($src3$$reg));
14407 %}
14408
14409 ins_pipe(pipe_class_default);
14410 %}
14411
14412 // -src1 * src2 + src3
14413 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14414 predicate(UseFMA);
14415 match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14416 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14417
14418 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
14419
14420 ins_encode %{
14421 __ fmsubd(as_FloatRegister($dst$$reg),
14422 as_FloatRegister($src1$$reg),
14423 as_FloatRegister($src2$$reg),
14424 as_FloatRegister($src3$$reg));
14425 %}
14426
14427 ins_pipe(pipe_class_default);
14428 %}
14429
14430 // -src1 * src2 - src3
14431 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14432 predicate(UseFMA);
14433 match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14434 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14435
14436 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
14437
14438 ins_encode %{
14439 __ fnmadds(as_FloatRegister($dst$$reg),
14440 as_FloatRegister($src1$$reg),
14441 as_FloatRegister($src2$$reg),
14442 as_FloatRegister($src3$$reg));
14443 %}
14444
14445 ins_pipe(pipe_class_default);
14446 %}
14447
14448 // -src1 * src2 - src3
14449 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14450 predicate(UseFMA);
14451 match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14452 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14453
14454 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14455
14456 ins_encode %{
14457 __ fnmaddd(as_FloatRegister($dst$$reg),
14458 as_FloatRegister($src1$$reg),
14459 as_FloatRegister($src2$$reg),
14460 as_FloatRegister($src3$$reg));
14461 %}
14462
14463 ins_pipe(pipe_class_default);
14464 %}
14465
14466 // src1 * src2 - src3
14467 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14468 predicate(UseFMA);
14469 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14470
14471 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14472
14473 ins_encode %{
14474 __ fnmsubs(as_FloatRegister($dst$$reg),
14475 as_FloatRegister($src1$$reg),
14476 as_FloatRegister($src2$$reg),
14477 as_FloatRegister($src3$$reg));
14478 %}
14479
14480 ins_pipe(pipe_class_default);
14481 %}
14482
14483 // src1 * src2 - src3
14484 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14485 predicate(UseFMA);
14486 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14487
14488 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14489
14490 ins_encode %{
14491 // n.b. insn name should be fnmsubd
14492 __ fnmsub(as_FloatRegister($dst$$reg),
14493 as_FloatRegister($src1$$reg),
14494 as_FloatRegister($src2$$reg),
14495 as_FloatRegister($src3$$reg));
14496 %}
14497
14498 ins_pipe(pipe_class_default);
14499 %}
14500
14501
14502 // Math.max(FF)F
14503 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14504 match(Set dst (MaxF src1 src2));
14505
14506 format %{ "fmaxs $dst, $src1, $src2" %}
14507 ins_encode %{
14508 __ fmaxs(as_FloatRegister($dst$$reg),
14509 as_FloatRegister($src1$$reg),
14510 as_FloatRegister($src2$$reg));
14511 %}
14512
14513 ins_pipe(fp_dop_reg_reg_s);
14514 %}
14515
14516 // Math.min(FF)F
14517 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14518 match(Set dst (MinF src1 src2));
14519
14520 format %{ "fmins $dst, $src1, $src2" %}
14521 ins_encode %{
14522 __ fmins(as_FloatRegister($dst$$reg),
14523 as_FloatRegister($src1$$reg),
14524 as_FloatRegister($src2$$reg));
14525 %}
14526
14527 ins_pipe(fp_dop_reg_reg_s);
14528 %}
14529
14530 // Math.max(DD)D
14531 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14532 match(Set dst (MaxD src1 src2));
14533
14534 format %{ "fmaxd $dst, $src1, $src2" %}
14535 ins_encode %{
14536 __ fmaxd(as_FloatRegister($dst$$reg),
14537 as_FloatRegister($src1$$reg),
14538 as_FloatRegister($src2$$reg));
14539 %}
14540
14541 ins_pipe(fp_dop_reg_reg_d);
14542 %}
14543
14544 // Math.min(DD)D
14545 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14546 match(Set dst (MinD src1 src2));
14547
14548 format %{ "fmind $dst, $src1, $src2" %}
14549 ins_encode %{
14550 __ fmind(as_FloatRegister($dst$$reg),
14551 as_FloatRegister($src1$$reg),
14552 as_FloatRegister($src2$$reg));
14553 %}
14554
14555 ins_pipe(fp_dop_reg_reg_d);
14556 %}
14557
14558
14559 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14560 match(Set dst (DivF src1 src2));
14561
14562 ins_cost(INSN_COST * 18);
14563 format %{ "fdivs $dst, $src1, $src2" %}
14564
14565 ins_encode %{
14566 __ fdivs(as_FloatRegister($dst$$reg),
14567 as_FloatRegister($src1$$reg),
14568 as_FloatRegister($src2$$reg));
14569 %}
14570
14571 ins_pipe(fp_div_s);
14572 %}
14573
14574 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14575 match(Set dst (DivD src1 src2));
14576
14577 ins_cost(INSN_COST * 32);
14578 format %{ "fdivd $dst, $src1, $src2" %}
14579
14580 ins_encode %{
14581 __ fdivd(as_FloatRegister($dst$$reg),
14582 as_FloatRegister($src1$$reg),
14583 as_FloatRegister($src2$$reg));
14584 %}
14585
14586 ins_pipe(fp_div_d);
14587 %}
14588
14589 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14590 match(Set dst (NegF src));
14591
14592 ins_cost(INSN_COST * 3);
14593 format %{ "fneg $dst, $src" %}
14594
14595 ins_encode %{
14596 __ fnegs(as_FloatRegister($dst$$reg),
14597 as_FloatRegister($src$$reg));
14598 %}
14599
14600 ins_pipe(fp_uop_s);
14601 %}
14602
14603 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14604 match(Set dst (NegD src));
14605
14606 ins_cost(INSN_COST * 3);
14607 format %{ "fnegd $dst, $src" %}
14608
14609 ins_encode %{
14610 __ fnegd(as_FloatRegister($dst$$reg),
14611 as_FloatRegister($src$$reg));
14612 %}
14613
14614 ins_pipe(fp_uop_d);
14615 %}
14616
14617 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14618 %{
14619 match(Set dst (AbsI src));
14620
14621 effect(KILL cr);
14622 ins_cost(INSN_COST * 2);
14623 format %{ "cmpw $src, zr\n\t"
14624 "cnegw $dst, $src, Assembler::LT\t# int abs"
14625 %}
14626
14627 ins_encode %{
14628 __ cmpw(as_Register($src$$reg), zr);
14629 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14630 %}
14631 ins_pipe(pipe_class_default);
14632 %}
14633
14634 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14635 %{
14636 match(Set dst (AbsL src));
14637
14638 effect(KILL cr);
14639 ins_cost(INSN_COST * 2);
14640 format %{ "cmp $src, zr\n\t"
14641 "cneg $dst, $src, Assembler::LT\t# long abs"
14642 %}
14643
14644 ins_encode %{
14645 __ cmp(as_Register($src$$reg), zr);
14646 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14647 %}
14648 ins_pipe(pipe_class_default);
14649 %}
14650
14651 instruct absF_reg(vRegF dst, vRegF src) %{
14652 match(Set dst (AbsF src));
14653
14654 ins_cost(INSN_COST * 3);
14655 format %{ "fabss $dst, $src" %}
14656 ins_encode %{
14657 __ fabss(as_FloatRegister($dst$$reg),
14658 as_FloatRegister($src$$reg));
14659 %}
14660
14661 ins_pipe(fp_uop_s);
14662 %}
14663
14664 instruct absD_reg(vRegD dst, vRegD src) %{
14665 match(Set dst (AbsD src));
14666
14667 ins_cost(INSN_COST * 3);
14668 format %{ "fabsd $dst, $src" %}
14669 ins_encode %{
14670 __ fabsd(as_FloatRegister($dst$$reg),
14671 as_FloatRegister($src$$reg));
14672 %}
14673
14674 ins_pipe(fp_uop_d);
14675 %}
14676
14677 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14678 match(Set dst (AbsF (SubF src1 src2)));
14679
14680 ins_cost(INSN_COST * 3);
14681 format %{ "fabds $dst, $src1, $src2" %}
14682 ins_encode %{
14683 __ fabds(as_FloatRegister($dst$$reg),
14684 as_FloatRegister($src1$$reg),
14685 as_FloatRegister($src2$$reg));
14686 %}
14687
14688 ins_pipe(fp_uop_s);
14689 %}
14690
14691 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14692 match(Set dst (AbsD (SubD src1 src2)));
14693
14694 ins_cost(INSN_COST * 3);
14695 format %{ "fabdd $dst, $src1, $src2" %}
14696 ins_encode %{
14697 __ fabdd(as_FloatRegister($dst$$reg),
14698 as_FloatRegister($src1$$reg),
14699 as_FloatRegister($src2$$reg));
14700 %}
14701
14702 ins_pipe(fp_uop_d);
14703 %}
14704
14705 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14706 match(Set dst (SqrtD src));
14707
14708 ins_cost(INSN_COST * 50);
14709 format %{ "fsqrtd $dst, $src" %}
14710 ins_encode %{
14711 __ fsqrtd(as_FloatRegister($dst$$reg),
14712 as_FloatRegister($src$$reg));
14713 %}
14714
14715 ins_pipe(fp_div_s);
14716 %}
14717
14718 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14719 match(Set dst (SqrtF src));
14720
14721 ins_cost(INSN_COST * 50);
14722 format %{ "fsqrts $dst, $src" %}
14723 ins_encode %{
14724 __ fsqrts(as_FloatRegister($dst$$reg),
14725 as_FloatRegister($src$$reg));
14726 %}
14727
14728 ins_pipe(fp_div_d);
14729 %}
14730
14731 // Math.rint, floor, ceil
14732 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14733 match(Set dst (RoundDoubleMode src rmode));
14734 format %{ "frint $dst, $src, $rmode" %}
14735 ins_encode %{
14736 switch ($rmode$$constant) {
14737 case RoundDoubleModeNode::rmode_rint:
14738 __ frintnd(as_FloatRegister($dst$$reg),
14739 as_FloatRegister($src$$reg));
14740 break;
14741 case RoundDoubleModeNode::rmode_floor:
14742 __ frintmd(as_FloatRegister($dst$$reg),
14743 as_FloatRegister($src$$reg));
14744 break;
14745 case RoundDoubleModeNode::rmode_ceil:
14746 __ frintpd(as_FloatRegister($dst$$reg),
14747 as_FloatRegister($src$$reg));
14748 break;
14749 }
14750 %}
14751 ins_pipe(fp_uop_d);
14752 %}
14753
14754 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14755 match(Set dst (CopySignD src1 (Binary src2 zero)));
14756 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14757 format %{ "CopySignD $dst $src1 $src2" %}
14758 ins_encode %{
14759 FloatRegister dst = as_FloatRegister($dst$$reg),
14760 src1 = as_FloatRegister($src1$$reg),
14761 src2 = as_FloatRegister($src2$$reg),
14762 zero = as_FloatRegister($zero$$reg);
14763 __ fnegd(dst, zero);
14764 __ bsl(dst, __ T8B, src2, src1);
14765 %}
14766 ins_pipe(fp_uop_d);
14767 %}
14768
14769 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14770 match(Set dst (CopySignF src1 src2));
14771 effect(TEMP_DEF dst, USE src1, USE src2);
14772 format %{ "CopySignF $dst $src1 $src2" %}
14773 ins_encode %{
14774 FloatRegister dst = as_FloatRegister($dst$$reg),
14775 src1 = as_FloatRegister($src1$$reg),
14776 src2 = as_FloatRegister($src2$$reg);
14777 __ movi(dst, __ T2S, 0x80, 24);
14778 __ bsl(dst, __ T8B, src2, src1);
14779 %}
14780 ins_pipe(fp_uop_d);
14781 %}
14782
14783 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14784 match(Set dst (SignumD src (Binary zero one)));
14785 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14786 format %{ "signumD $dst, $src" %}
14787 ins_encode %{
14788 FloatRegister src = as_FloatRegister($src$$reg),
14789 dst = as_FloatRegister($dst$$reg),
14790 zero = as_FloatRegister($zero$$reg),
14791 one = as_FloatRegister($one$$reg);
14792 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14793 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14794 // Bit selection instruction gets bit from "one" for each enabled bit in
14795 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14796 // NaN the whole "src" will be copied because "dst" is zero. For all other
14797 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14798 // from "src", and all other bits are copied from 1.0.
14799 __ bsl(dst, __ T8B, one, src);
14800 %}
14801 ins_pipe(fp_uop_d);
14802 %}
14803
14804 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14805 match(Set dst (SignumF src (Binary zero one)));
14806 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14807 format %{ "signumF $dst, $src" %}
14808 ins_encode %{
14809 FloatRegister src = as_FloatRegister($src$$reg),
14810 dst = as_FloatRegister($dst$$reg),
14811 zero = as_FloatRegister($zero$$reg),
14812 one = as_FloatRegister($one$$reg);
14813 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14814 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14815 // Bit selection instruction gets bit from "one" for each enabled bit in
14816 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14817 // NaN the whole "src" will be copied because "dst" is zero. For all other
14818 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14819 // from "src", and all other bits are copied from 1.0.
14820 __ bsl(dst, __ T8B, one, src);
14821 %}
14822 ins_pipe(fp_uop_d);
14823 %}
14824
14825 instruct onspinwait() %{
14826 match(OnSpinWait);
14827 ins_cost(INSN_COST);
14828
14829 format %{ "onspinwait" %}
14830
14831 ins_encode %{
14832 __ spin_wait();
14833 %}
14834 ins_pipe(pipe_class_empty);
14835 %}
14836
14837 // ============================================================================
14838 // Logical Instructions
14839
14840 // Integer Logical Instructions
14841
14842 // And Instructions
14843
14844
14845 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14846 match(Set dst (AndI src1 src2));
14847
14848 format %{ "andw $dst, $src1, $src2\t# int" %}
14849
14850 ins_cost(INSN_COST);
14851 ins_encode %{
14852 __ andw(as_Register($dst$$reg),
14853 as_Register($src1$$reg),
14854 as_Register($src2$$reg));
14855 %}
14856
14857 ins_pipe(ialu_reg_reg);
14858 %}
14859
14860 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14861 match(Set dst (AndI src1 src2));
14862
14863 format %{ "andsw $dst, $src1, $src2\t# int" %}
14864
14865 ins_cost(INSN_COST);
14866 ins_encode %{
14867 __ andw(as_Register($dst$$reg),
14868 as_Register($src1$$reg),
14869 (uint64_t)($src2$$constant));
14870 %}
14871
14872 ins_pipe(ialu_reg_imm);
14873 %}
14874
14875 // Or Instructions
14876
14877 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14878 match(Set dst (OrI src1 src2));
14879
14880 format %{ "orrw $dst, $src1, $src2\t# int" %}
14881
14882 ins_cost(INSN_COST);
14883 ins_encode %{
14884 __ orrw(as_Register($dst$$reg),
14885 as_Register($src1$$reg),
14886 as_Register($src2$$reg));
14887 %}
14888
14889 ins_pipe(ialu_reg_reg);
14890 %}
14891
14892 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14893 match(Set dst (OrI src1 src2));
14894
14895 format %{ "orrw $dst, $src1, $src2\t# int" %}
14896
14897 ins_cost(INSN_COST);
14898 ins_encode %{
14899 __ orrw(as_Register($dst$$reg),
14900 as_Register($src1$$reg),
14901 (uint64_t)($src2$$constant));
14902 %}
14903
14904 ins_pipe(ialu_reg_imm);
14905 %}
14906
14907 // Xor Instructions
14908
14909 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14910 match(Set dst (XorI src1 src2));
14911
14912 format %{ "eorw $dst, $src1, $src2\t# int" %}
14913
14914 ins_cost(INSN_COST);
14915 ins_encode %{
14916 __ eorw(as_Register($dst$$reg),
14917 as_Register($src1$$reg),
14918 as_Register($src2$$reg));
14919 %}
14920
14921 ins_pipe(ialu_reg_reg);
14922 %}
14923
14924 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14925 match(Set dst (XorI src1 src2));
14926
14927 format %{ "eorw $dst, $src1, $src2\t# int" %}
14928
14929 ins_cost(INSN_COST);
14930 ins_encode %{
14931 __ eorw(as_Register($dst$$reg),
14932 as_Register($src1$$reg),
14933 (uint64_t)($src2$$constant));
14934 %}
14935
14936 ins_pipe(ialu_reg_imm);
14937 %}
14938
14939 // Long Logical Instructions
14940 // TODO
14941
14942 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14943 match(Set dst (AndL src1 src2));
14944
14945 format %{ "and $dst, $src1, $src2\t# int" %}
14946
14947 ins_cost(INSN_COST);
14948 ins_encode %{
14949 __ andr(as_Register($dst$$reg),
14950 as_Register($src1$$reg),
14951 as_Register($src2$$reg));
14952 %}
14953
14954 ins_pipe(ialu_reg_reg);
14955 %}
14956
14957 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14958 match(Set dst (AndL src1 src2));
14959
14960 format %{ "and $dst, $src1, $src2\t# int" %}
14961
14962 ins_cost(INSN_COST);
14963 ins_encode %{
14964 __ andr(as_Register($dst$$reg),
14965 as_Register($src1$$reg),
14966 (uint64_t)($src2$$constant));
14967 %}
14968
14969 ins_pipe(ialu_reg_imm);
14970 %}
14971
14972 // Or Instructions
14973
14974 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14975 match(Set dst (OrL src1 src2));
14976
14977 format %{ "orr $dst, $src1, $src2\t# int" %}
14978
14979 ins_cost(INSN_COST);
14980 ins_encode %{
14981 __ orr(as_Register($dst$$reg),
14982 as_Register($src1$$reg),
14983 as_Register($src2$$reg));
14984 %}
14985
14986 ins_pipe(ialu_reg_reg);
14987 %}
14988
14989 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14990 match(Set dst (OrL src1 src2));
14991
14992 format %{ "orr $dst, $src1, $src2\t# int" %}
14993
14994 ins_cost(INSN_COST);
14995 ins_encode %{
14996 __ orr(as_Register($dst$$reg),
14997 as_Register($src1$$reg),
14998 (uint64_t)($src2$$constant));
14999 %}
15000
15001 ins_pipe(ialu_reg_imm);
15002 %}
15003
15004 // Xor Instructions
15005
15006 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
15007 match(Set dst (XorL src1 src2));
15008
15009 format %{ "eor $dst, $src1, $src2\t# int" %}
15010
15011 ins_cost(INSN_COST);
15012 ins_encode %{
15013 __ eor(as_Register($dst$$reg),
15014 as_Register($src1$$reg),
15015 as_Register($src2$$reg));
15016 %}
15017
15018 ins_pipe(ialu_reg_reg);
15019 %}
15020
15021 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
15022 match(Set dst (XorL src1 src2));
15023
15024 ins_cost(INSN_COST);
15025 format %{ "eor $dst, $src1, $src2\t# int" %}
15026
15027 ins_encode %{
15028 __ eor(as_Register($dst$$reg),
15029 as_Register($src1$$reg),
15030 (uint64_t)($src2$$constant));
15031 %}
15032
15033 ins_pipe(ialu_reg_imm);
15034 %}
15035
15036 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
15037 %{
15038 match(Set dst (ConvI2L src));
15039
15040 ins_cost(INSN_COST);
15041 format %{ "sxtw $dst, $src\t# i2l" %}
15042 ins_encode %{
15043 __ sbfm($dst$$Register, $src$$Register, 0, 31);
15044 %}
15045 ins_pipe(ialu_reg_shift);
15046 %}
15047
15048 // this pattern occurs in bigmath arithmetic
15049 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
15050 %{
15051 match(Set dst (AndL (ConvI2L src) mask));
15052
15053 ins_cost(INSN_COST);
15054 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
15055 ins_encode %{
15056 __ ubfm($dst$$Register, $src$$Register, 0, 31);
15057 %}
15058
15059 ins_pipe(ialu_reg_shift);
15060 %}
15061
15062 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
15063 match(Set dst (ConvL2I src));
15064
15065 ins_cost(INSN_COST);
15066 format %{ "movw $dst, $src \t// l2i" %}
15067
15068 ins_encode %{
15069 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
15070 %}
15071
15072 ins_pipe(ialu_reg);
15073 %}
15074
15075 instruct convD2F_reg(vRegF dst, vRegD src) %{
15076 match(Set dst (ConvD2F src));
15077
15078 ins_cost(INSN_COST * 5);
15079 format %{ "fcvtd $dst, $src \t// d2f" %}
15080
15081 ins_encode %{
15082 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15083 %}
15084
15085 ins_pipe(fp_d2f);
15086 %}
15087
15088 instruct convF2D_reg(vRegD dst, vRegF src) %{
15089 match(Set dst (ConvF2D src));
15090
15091 ins_cost(INSN_COST * 5);
15092 format %{ "fcvts $dst, $src \t// f2d" %}
15093
15094 ins_encode %{
15095 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15096 %}
15097
15098 ins_pipe(fp_f2d);
15099 %}
15100
15101 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15102 match(Set dst (ConvF2I src));
15103
15104 ins_cost(INSN_COST * 5);
15105 format %{ "fcvtzsw $dst, $src \t// f2i" %}
15106
15107 ins_encode %{
15108 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15109 %}
15110
15111 ins_pipe(fp_f2i);
15112 %}
15113
15114 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
15115 match(Set dst (ConvF2L src));
15116
15117 ins_cost(INSN_COST * 5);
15118 format %{ "fcvtzs $dst, $src \t// f2l" %}
15119
15120 ins_encode %{
15121 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15122 %}
15123
15124 ins_pipe(fp_f2l);
15125 %}
15126
15127 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
15128 match(Set dst (ConvF2HF src));
15129 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
15130 "smov $dst, $tmp\t# move result from $tmp to $dst"
15131 %}
15132 effect(TEMP tmp);
15133 ins_encode %{
15134 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
15135 %}
15136 ins_pipe(pipe_slow);
15137 %}
15138
15139 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
15140 match(Set dst (ConvHF2F src));
15141 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
15142 "fcvt $dst, $tmp\t# convert half to single precision"
15143 %}
15144 effect(TEMP tmp);
15145 ins_encode %{
15146 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
15147 %}
15148 ins_pipe(pipe_slow);
15149 %}
15150
15151 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
15152 match(Set dst (ConvI2F src));
15153
15154 ins_cost(INSN_COST * 5);
15155 format %{ "scvtfws $dst, $src \t// i2f" %}
15156
15157 ins_encode %{
15158 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15159 %}
15160
15161 ins_pipe(fp_i2f);
15162 %}
15163
15164 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15165 match(Set dst (ConvL2F src));
15166
15167 ins_cost(INSN_COST * 5);
15168 format %{ "scvtfs $dst, $src \t// l2f" %}
15169
15170 ins_encode %{
15171 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15172 %}
15173
15174 ins_pipe(fp_l2f);
15175 %}
15176
15177 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15178 match(Set dst (ConvD2I src));
15179
15180 ins_cost(INSN_COST * 5);
15181 format %{ "fcvtzdw $dst, $src \t// d2i" %}
15182
15183 ins_encode %{
15184 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15185 %}
15186
15187 ins_pipe(fp_d2i);
15188 %}
15189
15190 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15191 match(Set dst (ConvD2L src));
15192
15193 ins_cost(INSN_COST * 5);
15194 format %{ "fcvtzd $dst, $src \t// d2l" %}
15195
15196 ins_encode %{
15197 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15198 %}
15199
15200 ins_pipe(fp_d2l);
15201 %}
15202
15203 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15204 match(Set dst (ConvI2D src));
15205
15206 ins_cost(INSN_COST * 5);
15207 format %{ "scvtfwd $dst, $src \t// i2d" %}
15208
15209 ins_encode %{
15210 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15211 %}
15212
15213 ins_pipe(fp_i2d);
15214 %}
15215
15216 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15217 match(Set dst (ConvL2D src));
15218
15219 ins_cost(INSN_COST * 5);
15220 format %{ "scvtfd $dst, $src \t// l2d" %}
15221
15222 ins_encode %{
15223 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15224 %}
15225
15226 ins_pipe(fp_l2d);
15227 %}
15228
15229 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
15230 %{
15231 match(Set dst (RoundD src));
15232 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15233 format %{ "java_round_double $dst,$src"%}
15234 ins_encode %{
15235 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
15236 as_FloatRegister($ftmp$$reg));
15237 %}
15238 ins_pipe(pipe_slow);
15239 %}
15240
15241 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
15242 %{
15243 match(Set dst (RoundF src));
15244 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15245 format %{ "java_round_float $dst,$src"%}
15246 ins_encode %{
15247 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
15248 as_FloatRegister($ftmp$$reg));
15249 %}
15250 ins_pipe(pipe_slow);
15251 %}
15252
15253 // stack <-> reg and reg <-> reg shuffles with no conversion
15254
15255 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15256
15257 match(Set dst (MoveF2I src));
15258
15259 effect(DEF dst, USE src);
15260
15261 ins_cost(4 * INSN_COST);
15262
15263 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15264
15265 ins_encode %{
15266 __ ldrw($dst$$Register, Address(sp, $src$$disp));
15267 %}
15268
15269 ins_pipe(iload_reg_reg);
15270
15271 %}
15272
15273 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15274
15275 match(Set dst (MoveI2F src));
15276
15277 effect(DEF dst, USE src);
15278
15279 ins_cost(4 * INSN_COST);
15280
15281 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15282
15283 ins_encode %{
15284 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15285 %}
15286
15287 ins_pipe(pipe_class_memory);
15288
15289 %}
15290
15291 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15292
15293 match(Set dst (MoveD2L src));
15294
15295 effect(DEF dst, USE src);
15296
15297 ins_cost(4 * INSN_COST);
15298
15299 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15300
15301 ins_encode %{
15302 __ ldr($dst$$Register, Address(sp, $src$$disp));
15303 %}
15304
15305 ins_pipe(iload_reg_reg);
15306
15307 %}
15308
15309 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15310
15311 match(Set dst (MoveL2D src));
15312
15313 effect(DEF dst, USE src);
15314
15315 ins_cost(4 * INSN_COST);
15316
15317 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15318
15319 ins_encode %{
15320 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15321 %}
15322
15323 ins_pipe(pipe_class_memory);
15324
15325 %}
15326
15327 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15328
15329 match(Set dst (MoveF2I src));
15330
15331 effect(DEF dst, USE src);
15332
15333 ins_cost(INSN_COST);
15334
15335 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15336
15337 ins_encode %{
15338 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15339 %}
15340
15341 ins_pipe(pipe_class_memory);
15342
15343 %}
15344
15345 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15346
15347 match(Set dst (MoveI2F src));
15348
15349 effect(DEF dst, USE src);
15350
15351 ins_cost(INSN_COST);
15352
15353 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15354
15355 ins_encode %{
15356 __ strw($src$$Register, Address(sp, $dst$$disp));
15357 %}
15358
15359 ins_pipe(istore_reg_reg);
15360
15361 %}
15362
15363 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15364
15365 match(Set dst (MoveD2L src));
15366
15367 effect(DEF dst, USE src);
15368
15369 ins_cost(INSN_COST);
15370
15371 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15372
15373 ins_encode %{
15374 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15375 %}
15376
15377 ins_pipe(pipe_class_memory);
15378
15379 %}
15380
15381 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15382
15383 match(Set dst (MoveL2D src));
15384
15385 effect(DEF dst, USE src);
15386
15387 ins_cost(INSN_COST);
15388
15389 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15390
15391 ins_encode %{
15392 __ str($src$$Register, Address(sp, $dst$$disp));
15393 %}
15394
15395 ins_pipe(istore_reg_reg);
15396
15397 %}
15398
15399 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15400
15401 match(Set dst (MoveF2I src));
15402
15403 effect(DEF dst, USE src);
15404
15405 ins_cost(INSN_COST);
15406
15407 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15408
15409 ins_encode %{
15410 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15411 %}
15412
15413 ins_pipe(fp_f2i);
15414
15415 %}
15416
15417 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15418
15419 match(Set dst (MoveI2F src));
15420
15421 effect(DEF dst, USE src);
15422
15423 ins_cost(INSN_COST);
15424
15425 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15426
15427 ins_encode %{
15428 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15429 %}
15430
15431 ins_pipe(fp_i2f);
15432
15433 %}
15434
15435 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15436
15437 match(Set dst (MoveD2L src));
15438
15439 effect(DEF dst, USE src);
15440
15441 ins_cost(INSN_COST);
15442
15443 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15444
15445 ins_encode %{
15446 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15447 %}
15448
15449 ins_pipe(fp_d2l);
15450
15451 %}
15452
15453 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15454
15455 match(Set dst (MoveL2D src));
15456
15457 effect(DEF dst, USE src);
15458
15459 ins_cost(INSN_COST);
15460
15461 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15462
15463 ins_encode %{
15464 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15465 %}
15466
15467 ins_pipe(fp_l2d);
15468
15469 %}
15470
15471 // ============================================================================
15472 // clearing of an array
15473
15474 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
15475 %{
15476 match(Set dummy (ClearArray (Binary cnt base) zero));
15477 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15478
15479 ins_cost(4 * INSN_COST);
15480 format %{ "ClearArray $cnt, $base" %}
15481
15482 ins_encode %{
15483 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15484 if (tpc == nullptr) {
15485 ciEnv::current()->record_failure("CodeCache is full");
15486 return;
15487 }
15488 %}
15489
15490 ins_pipe(pipe_class_memory);
15491 %}
15492
15493 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
15494 %{
15495 predicate(((ClearArrayNode*)n)->word_copy_only());
15496 match(Set dummy (ClearArray (Binary cnt base) val));
15497 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15498
15499 ins_cost(4 * INSN_COST);
15500 format %{ "ClearArray $cnt, $base, $val" %}
15501
15502 ins_encode %{
15503 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
15504 %}
15505
15506 ins_pipe(pipe_class_memory);
15507 %}
15508
15509 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15510 %{
15511 predicate((uint64_t)n->in(2)->get_long()
15512 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
15513 && !((ClearArrayNode*)n)->word_copy_only());
15514 match(Set dummy (ClearArray cnt base));
15515 effect(TEMP temp, USE_KILL base, KILL cr);
15516
15517 ins_cost(4 * INSN_COST);
15518 format %{ "ClearArray $cnt, $base" %}
15519
15520 ins_encode %{
15521 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15522 if (tpc == nullptr) {
15523 ciEnv::current()->record_failure("CodeCache is full");
15524 return;
15525 }
15526 %}
15527
15528 ins_pipe(pipe_class_memory);
15529 %}
15530
15531 // ============================================================================
15532 // Overflow Math Instructions
15533
15534 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15535 %{
15536 match(Set cr (OverflowAddI op1 op2));
15537
15538 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15539 ins_cost(INSN_COST);
15540 ins_encode %{
15541 __ cmnw($op1$$Register, $op2$$Register);
15542 %}
15543
15544 ins_pipe(icmp_reg_reg);
15545 %}
15546
15547 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15548 %{
15549 match(Set cr (OverflowAddI op1 op2));
15550
15551 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15552 ins_cost(INSN_COST);
15553 ins_encode %{
15554 __ cmnw($op1$$Register, $op2$$constant);
15555 %}
15556
15557 ins_pipe(icmp_reg_imm);
15558 %}
15559
15560 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15561 %{
15562 match(Set cr (OverflowAddL op1 op2));
15563
15564 format %{ "cmn $op1, $op2\t# overflow check long" %}
15565 ins_cost(INSN_COST);
15566 ins_encode %{
15567 __ cmn($op1$$Register, $op2$$Register);
15568 %}
15569
15570 ins_pipe(icmp_reg_reg);
15571 %}
15572
15573 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15574 %{
15575 match(Set cr (OverflowAddL op1 op2));
15576
15577 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15578 ins_cost(INSN_COST);
15579 ins_encode %{
15580 __ adds(zr, $op1$$Register, $op2$$constant);
15581 %}
15582
15583 ins_pipe(icmp_reg_imm);
15584 %}
15585
15586 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15587 %{
15588 match(Set cr (OverflowSubI op1 op2));
15589
15590 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15591 ins_cost(INSN_COST);
15592 ins_encode %{
15593 __ cmpw($op1$$Register, $op2$$Register);
15594 %}
15595
15596 ins_pipe(icmp_reg_reg);
15597 %}
15598
15599 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15600 %{
15601 match(Set cr (OverflowSubI op1 op2));
15602
15603 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15604 ins_cost(INSN_COST);
15605 ins_encode %{
15606 __ cmpw($op1$$Register, $op2$$constant);
15607 %}
15608
15609 ins_pipe(icmp_reg_imm);
15610 %}
15611
15612 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15613 %{
15614 match(Set cr (OverflowSubL op1 op2));
15615
15616 format %{ "cmp $op1, $op2\t# overflow check long" %}
15617 ins_cost(INSN_COST);
15618 ins_encode %{
15619 __ cmp($op1$$Register, $op2$$Register);
15620 %}
15621
15622 ins_pipe(icmp_reg_reg);
15623 %}
15624
15625 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15626 %{
15627 match(Set cr (OverflowSubL op1 op2));
15628
15629 format %{ "cmp $op1, $op2\t# overflow check long" %}
15630 ins_cost(INSN_COST);
15631 ins_encode %{
15632 __ subs(zr, $op1$$Register, $op2$$constant);
15633 %}
15634
15635 ins_pipe(icmp_reg_imm);
15636 %}
15637
15638 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15639 %{
15640 match(Set cr (OverflowSubI zero op1));
15641
15642 format %{ "cmpw zr, $op1\t# overflow check int" %}
15643 ins_cost(INSN_COST);
15644 ins_encode %{
15645 __ cmpw(zr, $op1$$Register);
15646 %}
15647
15648 ins_pipe(icmp_reg_imm);
15649 %}
15650
15651 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15652 %{
15653 match(Set cr (OverflowSubL zero op1));
15654
15655 format %{ "cmp zr, $op1\t# overflow check long" %}
15656 ins_cost(INSN_COST);
15657 ins_encode %{
15658 __ cmp(zr, $op1$$Register);
15659 %}
15660
15661 ins_pipe(icmp_reg_imm);
15662 %}
15663
15664 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15665 %{
15666 match(Set cr (OverflowMulI op1 op2));
15667
15668 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15669 "cmp rscratch1, rscratch1, sxtw\n\t"
15670 "movw rscratch1, #0x80000000\n\t"
15671 "cselw rscratch1, rscratch1, zr, NE\n\t"
15672 "cmpw rscratch1, #1" %}
15673 ins_cost(5 * INSN_COST);
15674 ins_encode %{
15675 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15676 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15677 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15678 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15679 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15680 %}
15681
15682 ins_pipe(pipe_slow);
15683 %}
15684
15685 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15686 %{
15687 match(If cmp (OverflowMulI op1 op2));
15688 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15689 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15690 effect(USE labl, KILL cr);
15691
15692 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15693 "cmp rscratch1, rscratch1, sxtw\n\t"
15694 "b$cmp $labl" %}
15695 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15696 ins_encode %{
15697 Label* L = $labl$$label;
15698 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15699 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15700 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15701 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15702 %}
15703
15704 ins_pipe(pipe_serial);
15705 %}
15706
15707 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15708 %{
15709 match(Set cr (OverflowMulL op1 op2));
15710
15711 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15712 "smulh rscratch2, $op1, $op2\n\t"
15713 "cmp rscratch2, rscratch1, ASR #63\n\t"
15714 "movw rscratch1, #0x80000000\n\t"
15715 "cselw rscratch1, rscratch1, zr, NE\n\t"
15716 "cmpw rscratch1, #1" %}
15717 ins_cost(6 * INSN_COST);
15718 ins_encode %{
15719 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15720 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15721 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15722 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15723 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15724 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15725 %}
15726
15727 ins_pipe(pipe_slow);
15728 %}
15729
15730 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15731 %{
15732 match(If cmp (OverflowMulL op1 op2));
15733 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15734 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15735 effect(USE labl, KILL cr);
15736
15737 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15738 "smulh rscratch2, $op1, $op2\n\t"
15739 "cmp rscratch2, rscratch1, ASR #63\n\t"
15740 "b$cmp $labl" %}
15741 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15742 ins_encode %{
15743 Label* L = $labl$$label;
15744 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15745 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15746 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15747 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15748 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15749 %}
15750
15751 ins_pipe(pipe_serial);
15752 %}
15753
15754 // ============================================================================
15755 // Compare Instructions
15756
15757 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15758 %{
15759 match(Set cr (CmpI op1 op2));
15760
15761 effect(DEF cr, USE op1, USE op2);
15762
15763 ins_cost(INSN_COST);
15764 format %{ "cmpw $op1, $op2" %}
15765
15766 ins_encode(aarch64_enc_cmpw(op1, op2));
15767
15768 ins_pipe(icmp_reg_reg);
15769 %}
15770
15771 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15772 %{
15773 match(Set cr (CmpI op1 zero));
15774
15775 effect(DEF cr, USE op1);
15776
15777 ins_cost(INSN_COST);
15778 format %{ "cmpw $op1, 0" %}
15779
15780 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15781
15782 ins_pipe(icmp_reg_imm);
15783 %}
15784
15785 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15786 %{
15787 match(Set cr (CmpI op1 op2));
15788
15789 effect(DEF cr, USE op1);
15790
15791 ins_cost(INSN_COST);
15792 format %{ "cmpw $op1, $op2" %}
15793
15794 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15795
15796 ins_pipe(icmp_reg_imm);
15797 %}
15798
15799 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15800 %{
15801 match(Set cr (CmpI op1 op2));
15802
15803 effect(DEF cr, USE op1);
15804
15805 ins_cost(INSN_COST * 2);
15806 format %{ "cmpw $op1, $op2" %}
15807
15808 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15809
15810 ins_pipe(icmp_reg_imm);
15811 %}
15812
15813 // Unsigned compare Instructions; really, same as signed compare
15814 // except it should only be used to feed an If or a CMovI which takes a
15815 // cmpOpU.
15816
15817 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15818 %{
15819 match(Set cr (CmpU op1 op2));
15820
15821 effect(DEF cr, USE op1, USE op2);
15822
15823 ins_cost(INSN_COST);
15824 format %{ "cmpw $op1, $op2\t# unsigned" %}
15825
15826 ins_encode(aarch64_enc_cmpw(op1, op2));
15827
15828 ins_pipe(icmp_reg_reg);
15829 %}
15830
15831 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15832 %{
15833 match(Set cr (CmpU op1 zero));
15834
15835 effect(DEF cr, USE op1);
15836
15837 ins_cost(INSN_COST);
15838 format %{ "cmpw $op1, #0\t# unsigned" %}
15839
15840 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15841
15842 ins_pipe(icmp_reg_imm);
15843 %}
15844
15845 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15846 %{
15847 match(Set cr (CmpU op1 op2));
15848
15849 effect(DEF cr, USE op1);
15850
15851 ins_cost(INSN_COST);
15852 format %{ "cmpw $op1, $op2\t# unsigned" %}
15853
15854 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15855
15856 ins_pipe(icmp_reg_imm);
15857 %}
15858
15859 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15860 %{
15861 match(Set cr (CmpU op1 op2));
15862
15863 effect(DEF cr, USE op1);
15864
15865 ins_cost(INSN_COST * 2);
15866 format %{ "cmpw $op1, $op2\t# unsigned" %}
15867
15868 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15869
15870 ins_pipe(icmp_reg_imm);
15871 %}
15872
15873 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15874 %{
15875 match(Set cr (CmpL op1 op2));
15876
15877 effect(DEF cr, USE op1, USE op2);
15878
15879 ins_cost(INSN_COST);
15880 format %{ "cmp $op1, $op2" %}
15881
15882 ins_encode(aarch64_enc_cmp(op1, op2));
15883
15884 ins_pipe(icmp_reg_reg);
15885 %}
15886
15887 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15888 %{
15889 match(Set cr (CmpL op1 zero));
15890
15891 effect(DEF cr, USE op1);
15892
15893 ins_cost(INSN_COST);
15894 format %{ "tst $op1" %}
15895
15896 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15897
15898 ins_pipe(icmp_reg_imm);
15899 %}
15900
15901 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15902 %{
15903 match(Set cr (CmpL op1 op2));
15904
15905 effect(DEF cr, USE op1);
15906
15907 ins_cost(INSN_COST);
15908 format %{ "cmp $op1, $op2" %}
15909
15910 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15911
15912 ins_pipe(icmp_reg_imm);
15913 %}
15914
15915 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15916 %{
15917 match(Set cr (CmpL op1 op2));
15918
15919 effect(DEF cr, USE op1);
15920
15921 ins_cost(INSN_COST * 2);
15922 format %{ "cmp $op1, $op2" %}
15923
15924 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15925
15926 ins_pipe(icmp_reg_imm);
15927 %}
15928
15929 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15930 %{
15931 match(Set cr (CmpUL op1 op2));
15932
15933 effect(DEF cr, USE op1, USE op2);
15934
15935 ins_cost(INSN_COST);
15936 format %{ "cmp $op1, $op2" %}
15937
15938 ins_encode(aarch64_enc_cmp(op1, op2));
15939
15940 ins_pipe(icmp_reg_reg);
15941 %}
15942
15943 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15944 %{
15945 match(Set cr (CmpUL op1 zero));
15946
15947 effect(DEF cr, USE op1);
15948
15949 ins_cost(INSN_COST);
15950 format %{ "tst $op1" %}
15951
15952 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15953
15954 ins_pipe(icmp_reg_imm);
15955 %}
15956
15957 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15958 %{
15959 match(Set cr (CmpUL op1 op2));
15960
15961 effect(DEF cr, USE op1);
15962
15963 ins_cost(INSN_COST);
15964 format %{ "cmp $op1, $op2" %}
15965
15966 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15967
15968 ins_pipe(icmp_reg_imm);
15969 %}
15970
15971 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15972 %{
15973 match(Set cr (CmpUL op1 op2));
15974
15975 effect(DEF cr, USE op1);
15976
15977 ins_cost(INSN_COST * 2);
15978 format %{ "cmp $op1, $op2" %}
15979
15980 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15981
15982 ins_pipe(icmp_reg_imm);
15983 %}
15984
15985 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15986 %{
15987 match(Set cr (CmpP op1 op2));
15988
15989 effect(DEF cr, USE op1, USE op2);
15990
15991 ins_cost(INSN_COST);
15992 format %{ "cmp $op1, $op2\t // ptr" %}
15993
15994 ins_encode(aarch64_enc_cmpp(op1, op2));
15995
15996 ins_pipe(icmp_reg_reg);
15997 %}
15998
15999 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
16000 %{
16001 match(Set cr (CmpN op1 op2));
16002
16003 effect(DEF cr, USE op1, USE op2);
16004
16005 ins_cost(INSN_COST);
16006 format %{ "cmp $op1, $op2\t // compressed ptr" %}
16007
16008 ins_encode(aarch64_enc_cmpn(op1, op2));
16009
16010 ins_pipe(icmp_reg_reg);
16011 %}
16012
16013 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
16014 %{
16015 match(Set cr (CmpP op1 zero));
16016
16017 effect(DEF cr, USE op1, USE zero);
16018
16019 ins_cost(INSN_COST);
16020 format %{ "cmp $op1, 0\t // ptr" %}
16021
16022 ins_encode(aarch64_enc_testp(op1));
16023
16024 ins_pipe(icmp_reg_imm);
16025 %}
16026
16027 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
16028 %{
16029 match(Set cr (CmpN op1 zero));
16030
16031 effect(DEF cr, USE op1, USE zero);
16032
16033 ins_cost(INSN_COST);
16034 format %{ "cmp $op1, 0\t // compressed ptr" %}
16035
16036 ins_encode(aarch64_enc_testn(op1));
16037
16038 ins_pipe(icmp_reg_imm);
16039 %}
16040
16041 // FP comparisons
16042 //
16043 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
16044 // using normal cmpOp. See declaration of rFlagsReg for details.
16045
16046 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
16047 %{
16048 match(Set cr (CmpF src1 src2));
16049
16050 ins_cost(3 * INSN_COST);
16051 format %{ "fcmps $src1, $src2" %}
16052
16053 ins_encode %{
16054 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16055 %}
16056
16057 ins_pipe(pipe_class_compare);
16058 %}
16059
16060 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
16061 %{
16062 match(Set cr (CmpF src1 src2));
16063
16064 ins_cost(3 * INSN_COST);
16065 format %{ "fcmps $src1, 0.0" %}
16066
16067 ins_encode %{
16068 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
16069 %}
16070
16071 ins_pipe(pipe_class_compare);
16072 %}
16073 // FROM HERE
16074
16075 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
16076 %{
16077 match(Set cr (CmpD src1 src2));
16078
16079 ins_cost(3 * INSN_COST);
16080 format %{ "fcmpd $src1, $src2" %}
16081
16082 ins_encode %{
16083 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16084 %}
16085
16086 ins_pipe(pipe_class_compare);
16087 %}
16088
16089 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
16090 %{
16091 match(Set cr (CmpD src1 src2));
16092
16093 ins_cost(3 * INSN_COST);
16094 format %{ "fcmpd $src1, 0.0" %}
16095
16096 ins_encode %{
16097 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
16098 %}
16099
16100 ins_pipe(pipe_class_compare);
16101 %}
16102
16103 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
16104 %{
16105 match(Set dst (CmpF3 src1 src2));
16106 effect(KILL cr);
16107
16108 ins_cost(5 * INSN_COST);
16109 format %{ "fcmps $src1, $src2\n\t"
16110 "csinvw($dst, zr, zr, eq\n\t"
16111 "csnegw($dst, $dst, $dst, lt)"
16112 %}
16113
16114 ins_encode %{
16115 Label done;
16116 FloatRegister s1 = as_FloatRegister($src1$$reg);
16117 FloatRegister s2 = as_FloatRegister($src2$$reg);
16118 Register d = as_Register($dst$$reg);
16119 __ fcmps(s1, s2);
16120 // installs 0 if EQ else -1
16121 __ csinvw(d, zr, zr, Assembler::EQ);
16122 // keeps -1 if less or unordered else installs 1
16123 __ csnegw(d, d, d, Assembler::LT);
16124 __ bind(done);
16125 %}
16126
16127 ins_pipe(pipe_class_default);
16128
16129 %}
16130
16131 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
16132 %{
16133 match(Set dst (CmpD3 src1 src2));
16134 effect(KILL cr);
16135
16136 ins_cost(5 * INSN_COST);
16137 format %{ "fcmpd $src1, $src2\n\t"
16138 "csinvw($dst, zr, zr, eq\n\t"
16139 "csnegw($dst, $dst, $dst, lt)"
16140 %}
16141
16142 ins_encode %{
16143 Label done;
16144 FloatRegister s1 = as_FloatRegister($src1$$reg);
16145 FloatRegister s2 = as_FloatRegister($src2$$reg);
16146 Register d = as_Register($dst$$reg);
16147 __ fcmpd(s1, s2);
16148 // installs 0 if EQ else -1
16149 __ csinvw(d, zr, zr, Assembler::EQ);
16150 // keeps -1 if less or unordered else installs 1
16151 __ csnegw(d, d, d, Assembler::LT);
16152 __ bind(done);
16153 %}
16154 ins_pipe(pipe_class_default);
16155
16156 %}
16157
16158 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
16159 %{
16160 match(Set dst (CmpF3 src1 zero));
16161 effect(KILL cr);
16162
16163 ins_cost(5 * INSN_COST);
16164 format %{ "fcmps $src1, 0.0\n\t"
16165 "csinvw($dst, zr, zr, eq\n\t"
16166 "csnegw($dst, $dst, $dst, lt)"
16167 %}
16168
16169 ins_encode %{
16170 Label done;
16171 FloatRegister s1 = as_FloatRegister($src1$$reg);
16172 Register d = as_Register($dst$$reg);
16173 __ fcmps(s1, 0.0);
16174 // installs 0 if EQ else -1
16175 __ csinvw(d, zr, zr, Assembler::EQ);
16176 // keeps -1 if less or unordered else installs 1
16177 __ csnegw(d, d, d, Assembler::LT);
16178 __ bind(done);
16179 %}
16180
16181 ins_pipe(pipe_class_default);
16182
16183 %}
16184
16185 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16186 %{
16187 match(Set dst (CmpD3 src1 zero));
16188 effect(KILL cr);
16189
16190 ins_cost(5 * INSN_COST);
16191 format %{ "fcmpd $src1, 0.0\n\t"
16192 "csinvw($dst, zr, zr, eq\n\t"
16193 "csnegw($dst, $dst, $dst, lt)"
16194 %}
16195
16196 ins_encode %{
16197 Label done;
16198 FloatRegister s1 = as_FloatRegister($src1$$reg);
16199 Register d = as_Register($dst$$reg);
16200 __ fcmpd(s1, 0.0);
16201 // installs 0 if EQ else -1
16202 __ csinvw(d, zr, zr, Assembler::EQ);
16203 // keeps -1 if less or unordered else installs 1
16204 __ csnegw(d, d, d, Assembler::LT);
16205 __ bind(done);
16206 %}
16207 ins_pipe(pipe_class_default);
16208
16209 %}
16210
16211 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16212 %{
16213 match(Set dst (CmpLTMask p q));
16214 effect(KILL cr);
16215
16216 ins_cost(3 * INSN_COST);
16217
16218 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16219 "csetw $dst, lt\n\t"
16220 "subw $dst, zr, $dst"
16221 %}
16222
16223 ins_encode %{
16224 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16225 __ csetw(as_Register($dst$$reg), Assembler::LT);
16226 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16227 %}
16228
16229 ins_pipe(ialu_reg_reg);
16230 %}
16231
16232 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16233 %{
16234 match(Set dst (CmpLTMask src zero));
16235 effect(KILL cr);
16236
16237 ins_cost(INSN_COST);
16238
16239 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16240
16241 ins_encode %{
16242 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16243 %}
16244
16245 ins_pipe(ialu_reg_shift);
16246 %}
16247
16248 // ============================================================================
16249 // Max and Min
16250
16251 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
16252
16253 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
16254 %{
16255 effect(DEF cr, USE src);
16256 ins_cost(INSN_COST);
16257 format %{ "cmpw $src, 0" %}
16258
16259 ins_encode %{
16260 __ cmpw($src$$Register, 0);
16261 %}
16262 ins_pipe(icmp_reg_imm);
16263 %}
16264
16265 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16266 %{
16267 match(Set dst (MinI src1 src2));
16268 ins_cost(INSN_COST * 3);
16269
16270 expand %{
16271 rFlagsReg cr;
16272 compI_reg_reg(cr, src1, src2);
16273 cmovI_reg_reg_lt(dst, src1, src2, cr);
16274 %}
16275 %}
16276
16277 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16278 %{
16279 match(Set dst (MaxI src1 src2));
16280 ins_cost(INSN_COST * 3);
16281
16282 expand %{
16283 rFlagsReg cr;
16284 compI_reg_reg(cr, src1, src2);
16285 cmovI_reg_reg_gt(dst, src1, src2, cr);
16286 %}
16287 %}
16288
16289
16290 // ============================================================================
16291 // Branch Instructions
16292
16293 // Direct Branch.
16294 instruct branch(label lbl)
16295 %{
16296 match(Goto);
16297
16298 effect(USE lbl);
16299
16300 ins_cost(BRANCH_COST);
16301 format %{ "b $lbl" %}
16302
16303 ins_encode(aarch64_enc_b(lbl));
16304
16305 ins_pipe(pipe_branch);
16306 %}
16307
16308 // Conditional Near Branch
16309 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16310 %{
16311 // Same match rule as `branchConFar'.
16312 match(If cmp cr);
16313
16314 effect(USE lbl);
16315
16316 ins_cost(BRANCH_COST);
16317 // If set to 1 this indicates that the current instruction is a
16318 // short variant of a long branch. This avoids using this
16319 // instruction in first-pass matching. It will then only be used in
16320 // the `Shorten_branches' pass.
16321 // ins_short_branch(1);
16322 format %{ "b$cmp $lbl" %}
16323
16324 ins_encode(aarch64_enc_br_con(cmp, lbl));
16325
16326 ins_pipe(pipe_branch_cond);
16327 %}
16328
16329 // Conditional Near Branch Unsigned
16330 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16331 %{
16332 // Same match rule as `branchConFar'.
16333 match(If cmp cr);
16334
16335 effect(USE lbl);
16336
16337 ins_cost(BRANCH_COST);
16338 // If set to 1 this indicates that the current instruction is a
16339 // short variant of a long branch. This avoids using this
16340 // instruction in first-pass matching. It will then only be used in
16341 // the `Shorten_branches' pass.
16342 // ins_short_branch(1);
16343 format %{ "b$cmp $lbl\t# unsigned" %}
16344
16345 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16346
16347 ins_pipe(pipe_branch_cond);
16348 %}
16349
16350 // Make use of CBZ and CBNZ. These instructions, as well as being
16351 // shorter than (cmp; branch), have the additional benefit of not
16352 // killing the flags.
16353
16354 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16355 match(If cmp (CmpI op1 op2));
16356 effect(USE labl);
16357
16358 ins_cost(BRANCH_COST);
16359 format %{ "cbw$cmp $op1, $labl" %}
16360 ins_encode %{
16361 Label* L = $labl$$label;
16362 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16363 if (cond == Assembler::EQ)
16364 __ cbzw($op1$$Register, *L);
16365 else
16366 __ cbnzw($op1$$Register, *L);
16367 %}
16368 ins_pipe(pipe_cmp_branch);
16369 %}
16370
16371 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16372 match(If cmp (CmpL op1 op2));
16373 effect(USE labl);
16374
16375 ins_cost(BRANCH_COST);
16376 format %{ "cb$cmp $op1, $labl" %}
16377 ins_encode %{
16378 Label* L = $labl$$label;
16379 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16380 if (cond == Assembler::EQ)
16381 __ cbz($op1$$Register, *L);
16382 else
16383 __ cbnz($op1$$Register, *L);
16384 %}
16385 ins_pipe(pipe_cmp_branch);
16386 %}
16387
16388 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16389 match(If cmp (CmpP op1 op2));
16390 effect(USE labl);
16391
16392 ins_cost(BRANCH_COST);
16393 format %{ "cb$cmp $op1, $labl" %}
16394 ins_encode %{
16395 Label* L = $labl$$label;
16396 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16397 if (cond == Assembler::EQ)
16398 __ cbz($op1$$Register, *L);
16399 else
16400 __ cbnz($op1$$Register, *L);
16401 %}
16402 ins_pipe(pipe_cmp_branch);
16403 %}
16404
16405 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16406 match(If cmp (CmpN op1 op2));
16407 effect(USE labl);
16408
16409 ins_cost(BRANCH_COST);
16410 format %{ "cbw$cmp $op1, $labl" %}
16411 ins_encode %{
16412 Label* L = $labl$$label;
16413 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16414 if (cond == Assembler::EQ)
16415 __ cbzw($op1$$Register, *L);
16416 else
16417 __ cbnzw($op1$$Register, *L);
16418 %}
16419 ins_pipe(pipe_cmp_branch);
16420 %}
16421
16422 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16423 match(If cmp (CmpP (DecodeN oop) zero));
16424 effect(USE labl);
16425
16426 ins_cost(BRANCH_COST);
16427 format %{ "cb$cmp $oop, $labl" %}
16428 ins_encode %{
16429 Label* L = $labl$$label;
16430 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16431 if (cond == Assembler::EQ)
16432 __ cbzw($oop$$Register, *L);
16433 else
16434 __ cbnzw($oop$$Register, *L);
16435 %}
16436 ins_pipe(pipe_cmp_branch);
16437 %}
16438
16439 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16440 match(If cmp (CmpU op1 op2));
16441 effect(USE labl);
16442
16443 ins_cost(BRANCH_COST);
16444 format %{ "cbw$cmp $op1, $labl" %}
16445 ins_encode %{
16446 Label* L = $labl$$label;
16447 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16448 if (cond == Assembler::EQ || cond == Assembler::LS)
16449 __ cbzw($op1$$Register, *L);
16450 else
16451 __ cbnzw($op1$$Register, *L);
16452 %}
16453 ins_pipe(pipe_cmp_branch);
16454 %}
16455
16456 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16457 match(If cmp (CmpUL op1 op2));
16458 effect(USE labl);
16459
16460 ins_cost(BRANCH_COST);
16461 format %{ "cb$cmp $op1, $labl" %}
16462 ins_encode %{
16463 Label* L = $labl$$label;
16464 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16465 if (cond == Assembler::EQ || cond == Assembler::LS)
16466 __ cbz($op1$$Register, *L);
16467 else
16468 __ cbnz($op1$$Register, *L);
16469 %}
16470 ins_pipe(pipe_cmp_branch);
16471 %}
16472
16473 // Test bit and Branch
16474
16475 // Patterns for short (< 32KiB) variants
16476 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16477 match(If cmp (CmpL op1 op2));
16478 effect(USE labl);
16479
16480 ins_cost(BRANCH_COST);
16481 format %{ "cb$cmp $op1, $labl # long" %}
16482 ins_encode %{
16483 Label* L = $labl$$label;
16484 Assembler::Condition cond =
16485 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16486 __ tbr(cond, $op1$$Register, 63, *L);
16487 %}
16488 ins_pipe(pipe_cmp_branch);
16489 ins_short_branch(1);
16490 %}
16491
16492 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16493 match(If cmp (CmpI op1 op2));
16494 effect(USE labl);
16495
16496 ins_cost(BRANCH_COST);
16497 format %{ "cb$cmp $op1, $labl # int" %}
16498 ins_encode %{
16499 Label* L = $labl$$label;
16500 Assembler::Condition cond =
16501 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16502 __ tbr(cond, $op1$$Register, 31, *L);
16503 %}
16504 ins_pipe(pipe_cmp_branch);
16505 ins_short_branch(1);
16506 %}
16507
16508 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16509 match(If cmp (CmpL (AndL op1 op2) op3));
16510 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16511 effect(USE labl);
16512
16513 ins_cost(BRANCH_COST);
16514 format %{ "tb$cmp $op1, $op2, $labl" %}
16515 ins_encode %{
16516 Label* L = $labl$$label;
16517 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16518 int bit = exact_log2_long($op2$$constant);
16519 __ tbr(cond, $op1$$Register, bit, *L);
16520 %}
16521 ins_pipe(pipe_cmp_branch);
16522 ins_short_branch(1);
16523 %}
16524
16525 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16526 match(If cmp (CmpI (AndI op1 op2) op3));
16527 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16528 effect(USE labl);
16529
16530 ins_cost(BRANCH_COST);
16531 format %{ "tb$cmp $op1, $op2, $labl" %}
16532 ins_encode %{
16533 Label* L = $labl$$label;
16534 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16535 int bit = exact_log2((juint)$op2$$constant);
16536 __ tbr(cond, $op1$$Register, bit, *L);
16537 %}
16538 ins_pipe(pipe_cmp_branch);
16539 ins_short_branch(1);
16540 %}
16541
16542 // And far variants
16543 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16544 match(If cmp (CmpL op1 op2));
16545 effect(USE labl);
16546
16547 ins_cost(BRANCH_COST);
16548 format %{ "cb$cmp $op1, $labl # long" %}
16549 ins_encode %{
16550 Label* L = $labl$$label;
16551 Assembler::Condition cond =
16552 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16553 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16554 %}
16555 ins_pipe(pipe_cmp_branch);
16556 %}
16557
16558 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16559 match(If cmp (CmpI op1 op2));
16560 effect(USE labl);
16561
16562 ins_cost(BRANCH_COST);
16563 format %{ "cb$cmp $op1, $labl # int" %}
16564 ins_encode %{
16565 Label* L = $labl$$label;
16566 Assembler::Condition cond =
16567 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16568 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16569 %}
16570 ins_pipe(pipe_cmp_branch);
16571 %}
16572
16573 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16574 match(If cmp (CmpL (AndL op1 op2) op3));
16575 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16576 effect(USE labl);
16577
16578 ins_cost(BRANCH_COST);
16579 format %{ "tb$cmp $op1, $op2, $labl" %}
16580 ins_encode %{
16581 Label* L = $labl$$label;
16582 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16583 int bit = exact_log2_long($op2$$constant);
16584 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16585 %}
16586 ins_pipe(pipe_cmp_branch);
16587 %}
16588
16589 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16590 match(If cmp (CmpI (AndI op1 op2) op3));
16591 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16592 effect(USE labl);
16593
16594 ins_cost(BRANCH_COST);
16595 format %{ "tb$cmp $op1, $op2, $labl" %}
16596 ins_encode %{
16597 Label* L = $labl$$label;
16598 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16599 int bit = exact_log2((juint)$op2$$constant);
16600 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16601 %}
16602 ins_pipe(pipe_cmp_branch);
16603 %}
16604
16605 // Test bits
16606
16607 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16608 match(Set cr (CmpL (AndL op1 op2) op3));
16609 predicate(Assembler::operand_valid_for_logical_immediate
16610 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16611
16612 ins_cost(INSN_COST);
16613 format %{ "tst $op1, $op2 # long" %}
16614 ins_encode %{
16615 __ tst($op1$$Register, $op2$$constant);
16616 %}
16617 ins_pipe(ialu_reg_reg);
16618 %}
16619
16620 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16621 match(Set cr (CmpI (AndI op1 op2) op3));
16622 predicate(Assembler::operand_valid_for_logical_immediate
16623 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16624
16625 ins_cost(INSN_COST);
16626 format %{ "tst $op1, $op2 # int" %}
16627 ins_encode %{
16628 __ tstw($op1$$Register, $op2$$constant);
16629 %}
16630 ins_pipe(ialu_reg_reg);
16631 %}
16632
16633 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16634 match(Set cr (CmpL (AndL op1 op2) op3));
16635
16636 ins_cost(INSN_COST);
16637 format %{ "tst $op1, $op2 # long" %}
16638 ins_encode %{
16639 __ tst($op1$$Register, $op2$$Register);
16640 %}
16641 ins_pipe(ialu_reg_reg);
16642 %}
16643
16644 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16645 match(Set cr (CmpI (AndI op1 op2) op3));
16646
16647 ins_cost(INSN_COST);
16648 format %{ "tstw $op1, $op2 # int" %}
16649 ins_encode %{
16650 __ tstw($op1$$Register, $op2$$Register);
16651 %}
16652 ins_pipe(ialu_reg_reg);
16653 %}
16654
16655
16656 // Conditional Far Branch
16657 // Conditional Far Branch Unsigned
16658 // TODO: fixme
16659
16660 // counted loop end branch near
16661 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16662 %{
16663 match(CountedLoopEnd cmp cr);
16664
16665 effect(USE lbl);
16666
16667 ins_cost(BRANCH_COST);
16668 // short variant.
16669 // ins_short_branch(1);
16670 format %{ "b$cmp $lbl \t// counted loop end" %}
16671
16672 ins_encode(aarch64_enc_br_con(cmp, lbl));
16673
16674 ins_pipe(pipe_branch);
16675 %}
16676
16677 // counted loop end branch far
16678 // TODO: fixme
16679
16680 // ============================================================================
16681 // inlined locking and unlocking
16682
16683 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16684 %{
16685 match(Set cr (FastLock object box));
16686 effect(TEMP tmp, TEMP tmp2);
16687
16688 // TODO
16689 // identify correct cost
16690 ins_cost(5 * INSN_COST);
16691 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16692
16693 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16694
16695 ins_pipe(pipe_serial);
16696 %}
16697
16698 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16699 %{
16700 match(Set cr (FastUnlock object box));
16701 effect(TEMP tmp, TEMP tmp2);
16702
16703 ins_cost(5 * INSN_COST);
16704 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16705
16706 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16707
16708 ins_pipe(pipe_serial);
16709 %}
16710
16711
16712 // ============================================================================
16713 // Safepoint Instructions
16714
16715 // TODO
16716 // provide a near and far version of this code
16717
16718 instruct safePoint(rFlagsReg cr, iRegP poll)
16719 %{
16720 match(SafePoint poll);
16721 effect(KILL cr);
16722
16723 format %{
16724 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16725 %}
16726 ins_encode %{
16727 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16728 %}
16729 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16730 %}
16731
16732
16733 // ============================================================================
16734 // Procedure Call/Return Instructions
16735
16736 // Call Java Static Instruction
16737
16738 instruct CallStaticJavaDirect(method meth)
16739 %{
16740 match(CallStaticJava);
16741
16742 effect(USE meth);
16743
16744 ins_cost(CALL_COST);
16745
16746 format %{ "call,static $meth \t// ==> " %}
16747
16748 ins_encode(aarch64_enc_java_static_call(meth),
16749 aarch64_enc_call_epilog);
16750
16751 ins_pipe(pipe_class_call);
16752 %}
16753
16754 // TO HERE
16755
16756 // Call Java Dynamic Instruction
16757 instruct CallDynamicJavaDirect(method meth)
16758 %{
16759 match(CallDynamicJava);
16760
16761 effect(USE meth);
16762
16763 ins_cost(CALL_COST);
16764
16765 format %{ "CALL,dynamic $meth \t// ==> " %}
16766
16767 ins_encode(aarch64_enc_java_dynamic_call(meth),
16768 aarch64_enc_call_epilog);
16769
16770 ins_pipe(pipe_class_call);
16771 %}
16772
16773 // Call Runtime Instruction
16774
16775 instruct CallRuntimeDirect(method meth)
16776 %{
16777 match(CallRuntime);
16778
16779 effect(USE meth);
16780
16781 ins_cost(CALL_COST);
16782
16783 format %{ "CALL, runtime $meth" %}
16784
16785 ins_encode( aarch64_enc_java_to_runtime(meth) );
16786
16787 ins_pipe(pipe_class_call);
16788 %}
16789
16790 // Call Runtime Instruction
16791
16792 instruct CallLeafDirect(method meth)
16793 %{
16794 match(CallLeaf);
16795
16796 effect(USE meth);
16797
16798 ins_cost(CALL_COST);
16799
16800 format %{ "CALL, runtime leaf $meth" %}
16801
16802 ins_encode( aarch64_enc_java_to_runtime(meth) );
16803
16804 ins_pipe(pipe_class_call);
16805 %}
16806
16807 // Call Runtime Instruction
16808
16809 // entry point is null, target holds the address to call
16810 instruct CallLeafNoFPIndirect(iRegP target)
16811 %{
16812 predicate(n->as_Call()->entry_point() == nullptr);
16813
16814 match(CallLeafNoFP target);
16815
16816 ins_cost(CALL_COST);
16817
16818 format %{ "CALL, runtime leaf nofp indirect $target" %}
16819
16820 ins_encode %{
16821 __ blr($target$$Register);
16822 %}
16823
16824 ins_pipe(pipe_class_call);
16825 %}
16826
16827 instruct CallLeafNoFPDirect(method meth)
16828 %{
16829 predicate(n->as_Call()->entry_point() != nullptr);
16830
16831 match(CallLeafNoFP);
16832
16833 effect(USE meth);
16834
16835 ins_cost(CALL_COST);
16836
16837 format %{ "CALL, runtime leaf nofp $meth" %}
16838
16839 ins_encode( aarch64_enc_java_to_runtime(meth) );
16840
16841 ins_pipe(pipe_class_call);
16842 %}
16843
16844 // Tail Call; Jump from runtime stub to Java code.
16845 // Also known as an 'interprocedural jump'.
16846 // Target of jump will eventually return to caller.
16847 // TailJump below removes the return address.
16848 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16849 %{
16850 match(TailCall jump_target method_ptr);
16851
16852 ins_cost(CALL_COST);
16853
16854 format %{ "br $jump_target\t# $method_ptr holds method" %}
16855
16856 ins_encode(aarch64_enc_tail_call(jump_target));
16857
16858 ins_pipe(pipe_class_call);
16859 %}
16860
16861 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16862 %{
16863 match(TailJump jump_target ex_oop);
16864
16865 ins_cost(CALL_COST);
16866
16867 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16868
16869 ins_encode(aarch64_enc_tail_jmp(jump_target));
16870
16871 ins_pipe(pipe_class_call);
16872 %}
16873
16874 // Create exception oop: created by stack-crawling runtime code.
16875 // Created exception is now available to this handler, and is setup
16876 // just prior to jumping to this handler. No code emitted.
16877 // TODO check
16878 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16879 instruct CreateException(iRegP_R0 ex_oop)
16880 %{
16881 match(Set ex_oop (CreateEx));
16882
16883 format %{ " -- \t// exception oop; no code emitted" %}
16884
16885 size(0);
16886
16887 ins_encode( /*empty*/ );
16888
16889 ins_pipe(pipe_class_empty);
16890 %}
16891
16892 // Rethrow exception: The exception oop will come in the first
16893 // argument position. Then JUMP (not call) to the rethrow stub code.
16894 instruct RethrowException() %{
16895 match(Rethrow);
16896 ins_cost(CALL_COST);
16897
16898 format %{ "b rethrow_stub" %}
16899
16900 ins_encode( aarch64_enc_rethrow() );
16901
16902 ins_pipe(pipe_class_call);
16903 %}
16904
16905
16906 // Return Instruction
16907 // epilog node loads ret address into lr as part of frame pop
16908 instruct Ret()
16909 %{
16910 match(Return);
16911
16912 format %{ "ret\t// return register" %}
16913
16914 ins_encode( aarch64_enc_ret() );
16915
16916 ins_pipe(pipe_branch);
16917 %}
16918
16919 // Die now.
16920 instruct ShouldNotReachHere() %{
16921 match(Halt);
16922
16923 ins_cost(CALL_COST);
16924 format %{ "ShouldNotReachHere" %}
16925
16926 ins_encode %{
16927 if (is_reachable()) {
16928 __ stop(_halt_reason);
16929 }
16930 %}
16931
16932 ins_pipe(pipe_class_default);
16933 %}
16934
16935 // ============================================================================
16936 // Partial Subtype Check
16937 //
16938 // superklass array for an instance of the superklass. Set a hidden
16939 // internal cache on a hit (cache is checked with exposed code in
16940 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16941 // encoding ALSO sets flags.
16942
16943 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16944 %{
16945 match(Set result (PartialSubtypeCheck sub super));
16946 effect(KILL cr, KILL temp);
16947
16948 ins_cost(1100); // slightly larger than the next version
16949 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16950
16951 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16952
16953 opcode(0x1); // Force zero of result reg on hit
16954
16955 ins_pipe(pipe_class_memory);
16956 %}
16957
16958 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16959 %{
16960 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16961 effect(KILL temp, KILL result);
16962
16963 ins_cost(1100); // slightly larger than the next version
16964 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16965
16966 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16967
16968 opcode(0x0); // Don't zero result reg on hit
16969
16970 ins_pipe(pipe_class_memory);
16971 %}
16972
16973 // Intrisics for String.compareTo()
16974
16975 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16976 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16977 %{
16978 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16979 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16980 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16981
16982 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16983 ins_encode %{
16984 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16985 __ string_compare($str1$$Register, $str2$$Register,
16986 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16987 $tmp1$$Register, $tmp2$$Register,
16988 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16989 %}
16990 ins_pipe(pipe_class_memory);
16991 %}
16992
16993 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16994 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16995 %{
16996 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16997 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16998 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16999
17000 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
17001 ins_encode %{
17002 __ string_compare($str1$$Register, $str2$$Register,
17003 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17004 $tmp1$$Register, $tmp2$$Register,
17005 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
17006 %}
17007 ins_pipe(pipe_class_memory);
17008 %}
17009
17010 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17011 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17012 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
17013 %{
17014 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
17015 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17016 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
17017 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17018
17019 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
17020 ins_encode %{
17021 __ string_compare($str1$$Register, $str2$$Register,
17022 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17023 $tmp1$$Register, $tmp2$$Register,
17024 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
17025 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
17026 %}
17027 ins_pipe(pipe_class_memory);
17028 %}
17029
17030 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17031 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17032 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
17033 %{
17034 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
17035 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17036 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
17037 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17038
17039 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
17040 ins_encode %{
17041 __ string_compare($str1$$Register, $str2$$Register,
17042 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17043 $tmp1$$Register, $tmp2$$Register,
17044 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
17045 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
17046 %}
17047 ins_pipe(pipe_class_memory);
17048 %}
17049
17050 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
17051 // these string_compare variants as NEON register type for convenience so that the prototype of
17052 // string_compare can be shared with all variants.
17053
17054 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17055 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17056 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17057 pRegGov_P1 pgtmp2, rFlagsReg cr)
17058 %{
17059 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
17060 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17061 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17062 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17063
17064 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
17065 ins_encode %{
17066 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17067 __ string_compare($str1$$Register, $str2$$Register,
17068 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17069 $tmp1$$Register, $tmp2$$Register,
17070 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17071 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17072 StrIntrinsicNode::LL);
17073 %}
17074 ins_pipe(pipe_class_memory);
17075 %}
17076
17077 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17078 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17079 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17080 pRegGov_P1 pgtmp2, rFlagsReg cr)
17081 %{
17082 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
17083 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17084 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17085 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17086
17087 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
17088 ins_encode %{
17089 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17090 __ string_compare($str1$$Register, $str2$$Register,
17091 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17092 $tmp1$$Register, $tmp2$$Register,
17093 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17094 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17095 StrIntrinsicNode::LU);
17096 %}
17097 ins_pipe(pipe_class_memory);
17098 %}
17099
17100 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17101 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17102 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17103 pRegGov_P1 pgtmp2, rFlagsReg cr)
17104 %{
17105 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
17106 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17107 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17108 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17109
17110 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
17111 ins_encode %{
17112 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17113 __ string_compare($str1$$Register, $str2$$Register,
17114 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17115 $tmp1$$Register, $tmp2$$Register,
17116 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17117 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17118 StrIntrinsicNode::UL);
17119 %}
17120 ins_pipe(pipe_class_memory);
17121 %}
17122
17123 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17124 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17125 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17126 pRegGov_P1 pgtmp2, rFlagsReg cr)
17127 %{
17128 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
17129 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17130 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17131 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17132
17133 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
17134 ins_encode %{
17135 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17136 __ string_compare($str1$$Register, $str2$$Register,
17137 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17138 $tmp1$$Register, $tmp2$$Register,
17139 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17140 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17141 StrIntrinsicNode::UU);
17142 %}
17143 ins_pipe(pipe_class_memory);
17144 %}
17145
17146 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17147 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17148 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17149 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17150 %{
17151 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17152 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17153 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17154 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
17155 TEMP vtmp0, TEMP vtmp1, KILL cr);
17156 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
17157 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17158
17159 ins_encode %{
17160 __ string_indexof($str1$$Register, $str2$$Register,
17161 $cnt1$$Register, $cnt2$$Register,
17162 $tmp1$$Register, $tmp2$$Register,
17163 $tmp3$$Register, $tmp4$$Register,
17164 $tmp5$$Register, $tmp6$$Register,
17165 -1, $result$$Register, StrIntrinsicNode::UU);
17166 %}
17167 ins_pipe(pipe_class_memory);
17168 %}
17169
17170 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17171 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17172 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17173 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17174 %{
17175 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17176 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17177 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17178 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
17179 TEMP vtmp0, TEMP vtmp1, KILL cr);
17180 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
17181 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17182
17183 ins_encode %{
17184 __ string_indexof($str1$$Register, $str2$$Register,
17185 $cnt1$$Register, $cnt2$$Register,
17186 $tmp1$$Register, $tmp2$$Register,
17187 $tmp3$$Register, $tmp4$$Register,
17188 $tmp5$$Register, $tmp6$$Register,
17189 -1, $result$$Register, StrIntrinsicNode::LL);
17190 %}
17191 ins_pipe(pipe_class_memory);
17192 %}
17193
17194 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17195 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
17196 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17197 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17198 %{
17199 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17200 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17201 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17202 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
17203 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
17204 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
17205 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17206
17207 ins_encode %{
17208 __ string_indexof($str1$$Register, $str2$$Register,
17209 $cnt1$$Register, $cnt2$$Register,
17210 $tmp1$$Register, $tmp2$$Register,
17211 $tmp3$$Register, $tmp4$$Register,
17212 $tmp5$$Register, $tmp6$$Register,
17213 -1, $result$$Register, StrIntrinsicNode::UL);
17214 %}
17215 ins_pipe(pipe_class_memory);
17216 %}
17217
17218 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17219 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17220 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17221 %{
17222 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17223 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17224 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17225 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17226 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
17227 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17228
17229 ins_encode %{
17230 int icnt2 = (int)$int_cnt2$$constant;
17231 __ string_indexof($str1$$Register, $str2$$Register,
17232 $cnt1$$Register, zr,
17233 $tmp1$$Register, $tmp2$$Register,
17234 $tmp3$$Register, $tmp4$$Register, zr, zr,
17235 icnt2, $result$$Register, StrIntrinsicNode::UU);
17236 %}
17237 ins_pipe(pipe_class_memory);
17238 %}
17239
17240 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17241 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17242 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17243 %{
17244 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17245 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17246 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17247 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17248 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
17249 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17250
17251 ins_encode %{
17252 int icnt2 = (int)$int_cnt2$$constant;
17253 __ string_indexof($str1$$Register, $str2$$Register,
17254 $cnt1$$Register, zr,
17255 $tmp1$$Register, $tmp2$$Register,
17256 $tmp3$$Register, $tmp4$$Register, zr, zr,
17257 icnt2, $result$$Register, StrIntrinsicNode::LL);
17258 %}
17259 ins_pipe(pipe_class_memory);
17260 %}
17261
17262 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17263 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17264 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17265 %{
17266 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17267 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17268 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17269 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17270 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
17271 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17272
17273 ins_encode %{
17274 int icnt2 = (int)$int_cnt2$$constant;
17275 __ string_indexof($str1$$Register, $str2$$Register,
17276 $cnt1$$Register, zr,
17277 $tmp1$$Register, $tmp2$$Register,
17278 $tmp3$$Register, $tmp4$$Register, zr, zr,
17279 icnt2, $result$$Register, StrIntrinsicNode::UL);
17280 %}
17281 ins_pipe(pipe_class_memory);
17282 %}
17283
17284 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17285 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17286 iRegINoSp tmp3, rFlagsReg cr)
17287 %{
17288 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17289 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17290 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17291 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17292
17293 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17294
17295 ins_encode %{
17296 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17297 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17298 $tmp3$$Register);
17299 %}
17300 ins_pipe(pipe_class_memory);
17301 %}
17302
17303 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17304 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17305 iRegINoSp tmp3, rFlagsReg cr)
17306 %{
17307 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17308 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17309 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17310 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17311
17312 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17313
17314 ins_encode %{
17315 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17316 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17317 $tmp3$$Register);
17318 %}
17319 ins_pipe(pipe_class_memory);
17320 %}
17321
17322 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17323 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17324 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17325 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17326 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17327 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17328 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17329 ins_encode %{
17330 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17331 $result$$Register, $ztmp1$$FloatRegister,
17332 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17333 $ptmp$$PRegister, true /* isL */);
17334 %}
17335 ins_pipe(pipe_class_memory);
17336 %}
17337
17338 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17339 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17340 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17341 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17342 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17343 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17344 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17345 ins_encode %{
17346 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17347 $result$$Register, $ztmp1$$FloatRegister,
17348 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17349 $ptmp$$PRegister, false /* isL */);
17350 %}
17351 ins_pipe(pipe_class_memory);
17352 %}
17353
17354 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17355 iRegI_R0 result, rFlagsReg cr)
17356 %{
17357 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17358 match(Set result (StrEquals (Binary str1 str2) cnt));
17359 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17360
17361 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17362 ins_encode %{
17363 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17364 __ string_equals($str1$$Register, $str2$$Register,
17365 $result$$Register, $cnt$$Register, 1);
17366 %}
17367 ins_pipe(pipe_class_memory);
17368 %}
17369
17370 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17371 iRegI_R0 result, rFlagsReg cr)
17372 %{
17373 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17374 match(Set result (StrEquals (Binary str1 str2) cnt));
17375 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17376
17377 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17378 ins_encode %{
17379 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17380 __ string_equals($str1$$Register, $str2$$Register,
17381 $result$$Register, $cnt$$Register, 2);
17382 %}
17383 ins_pipe(pipe_class_memory);
17384 %}
17385
17386 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17387 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17388 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17389 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17390 iRegP_R10 tmp, rFlagsReg cr)
17391 %{
17392 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17393 match(Set result (AryEq ary1 ary2));
17394 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17395 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17396 TEMP vtmp6, TEMP vtmp7, KILL cr);
17397
17398 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17399 ins_encode %{
17400 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17401 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17402 $result$$Register, $tmp$$Register, 1);
17403 if (tpc == nullptr) {
17404 ciEnv::current()->record_failure("CodeCache is full");
17405 return;
17406 }
17407 %}
17408 ins_pipe(pipe_class_memory);
17409 %}
17410
17411 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17412 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17413 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17414 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17415 iRegP_R10 tmp, rFlagsReg cr)
17416 %{
17417 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17418 match(Set result (AryEq ary1 ary2));
17419 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17420 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17421 TEMP vtmp6, TEMP vtmp7, KILL cr);
17422
17423 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17424 ins_encode %{
17425 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17426 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17427 $result$$Register, $tmp$$Register, 2);
17428 if (tpc == nullptr) {
17429 ciEnv::current()->record_failure("CodeCache is full");
17430 return;
17431 }
17432 %}
17433 ins_pipe(pipe_class_memory);
17434 %}
17435
17436 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17437 %{
17438 match(Set result (CountPositives ary1 len));
17439 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17440 format %{ "count positives byte[] $ary1,$len -> $result" %}
17441 ins_encode %{
17442 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17443 if (tpc == nullptr) {
17444 ciEnv::current()->record_failure("CodeCache is full");
17445 return;
17446 }
17447 %}
17448 ins_pipe( pipe_slow );
17449 %}
17450
17451 // fast char[] to byte[] compression
17452 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17453 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17454 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17455 iRegI_R0 result, rFlagsReg cr)
17456 %{
17457 match(Set result (StrCompressedCopy src (Binary dst len)));
17458 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17459 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17460
17461 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17462 ins_encode %{
17463 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17464 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17465 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17466 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17467 %}
17468 ins_pipe(pipe_slow);
17469 %}
17470
17471 // fast byte[] to char[] inflation
17472 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17473 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17474 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17475 %{
17476 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17477 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17478 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17479 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17480
17481 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17482 ins_encode %{
17483 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17484 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17485 $vtmp2$$FloatRegister, $tmp$$Register);
17486 if (tpc == nullptr) {
17487 ciEnv::current()->record_failure("CodeCache is full");
17488 return;
17489 }
17490 %}
17491 ins_pipe(pipe_class_memory);
17492 %}
17493
17494 // encode char[] to byte[] in ISO_8859_1
17495 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17496 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17497 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17498 iRegI_R0 result, rFlagsReg cr)
17499 %{
17500 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17501 match(Set result (EncodeISOArray src (Binary dst len)));
17502 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17503 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17504
17505 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17506 ins_encode %{
17507 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17508 $result$$Register, false,
17509 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17510 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17511 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17512 %}
17513 ins_pipe(pipe_class_memory);
17514 %}
17515
17516 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17517 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17518 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17519 iRegI_R0 result, rFlagsReg cr)
17520 %{
17521 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17522 match(Set result (EncodeISOArray src (Binary dst len)));
17523 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17524 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17525
17526 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17527 ins_encode %{
17528 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17529 $result$$Register, true,
17530 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17531 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17532 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17533 %}
17534 ins_pipe(pipe_class_memory);
17535 %}
17536
17537 //----------------------------- CompressBits/ExpandBits ------------------------
17538
17539 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17540 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17541 match(Set dst (CompressBits src mask));
17542 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17543 format %{ "mov $tsrc, $src\n\t"
17544 "mov $tmask, $mask\n\t"
17545 "bext $tdst, $tsrc, $tmask\n\t"
17546 "mov $dst, $tdst"
17547 %}
17548 ins_encode %{
17549 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17550 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17551 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17552 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17553 %}
17554 ins_pipe(pipe_slow);
17555 %}
17556
17557 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17558 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17559 match(Set dst (CompressBits (LoadI mem) mask));
17560 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17561 format %{ "ldrs $tsrc, $mem\n\t"
17562 "ldrs $tmask, $mask\n\t"
17563 "bext $tdst, $tsrc, $tmask\n\t"
17564 "mov $dst, $tdst"
17565 %}
17566 ins_encode %{
17567 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17568 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17569 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17570 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17571 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17572 %}
17573 ins_pipe(pipe_slow);
17574 %}
17575
17576 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17577 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17578 match(Set dst (CompressBits src mask));
17579 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17580 format %{ "mov $tsrc, $src\n\t"
17581 "mov $tmask, $mask\n\t"
17582 "bext $tdst, $tsrc, $tmask\n\t"
17583 "mov $dst, $tdst"
17584 %}
17585 ins_encode %{
17586 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17587 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17588 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17589 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17590 %}
17591 ins_pipe(pipe_slow);
17592 %}
17593
17594 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17595 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17596 match(Set dst (CompressBits (LoadL mem) mask));
17597 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17598 format %{ "ldrd $tsrc, $mem\n\t"
17599 "ldrd $tmask, $mask\n\t"
17600 "bext $tdst, $tsrc, $tmask\n\t"
17601 "mov $dst, $tdst"
17602 %}
17603 ins_encode %{
17604 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17605 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17606 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17607 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17608 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17609 %}
17610 ins_pipe(pipe_slow);
17611 %}
17612
17613 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17614 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17615 match(Set dst (ExpandBits src mask));
17616 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17617 format %{ "mov $tsrc, $src\n\t"
17618 "mov $tmask, $mask\n\t"
17619 "bdep $tdst, $tsrc, $tmask\n\t"
17620 "mov $dst, $tdst"
17621 %}
17622 ins_encode %{
17623 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17624 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17625 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17626 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17627 %}
17628 ins_pipe(pipe_slow);
17629 %}
17630
17631 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17632 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17633 match(Set dst (ExpandBits (LoadI mem) mask));
17634 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17635 format %{ "ldrs $tsrc, $mem\n\t"
17636 "ldrs $tmask, $mask\n\t"
17637 "bdep $tdst, $tsrc, $tmask\n\t"
17638 "mov $dst, $tdst"
17639 %}
17640 ins_encode %{
17641 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17642 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17643 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17644 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17645 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17646 %}
17647 ins_pipe(pipe_slow);
17648 %}
17649
17650 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17651 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17652 match(Set dst (ExpandBits src mask));
17653 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17654 format %{ "mov $tsrc, $src\n\t"
17655 "mov $tmask, $mask\n\t"
17656 "bdep $tdst, $tsrc, $tmask\n\t"
17657 "mov $dst, $tdst"
17658 %}
17659 ins_encode %{
17660 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17661 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17662 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17663 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17664 %}
17665 ins_pipe(pipe_slow);
17666 %}
17667
17668
17669 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17670 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17671 match(Set dst (ExpandBits (LoadL mem) mask));
17672 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17673 format %{ "ldrd $tsrc, $mem\n\t"
17674 "ldrd $tmask, $mask\n\t"
17675 "bdep $tdst, $tsrc, $tmask\n\t"
17676 "mov $dst, $tdst"
17677 %}
17678 ins_encode %{
17679 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17680 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17681 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17682 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17683 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17684 %}
17685 ins_pipe(pipe_slow);
17686 %}
17687
17688 // ============================================================================
17689 // This name is KNOWN by the ADLC and cannot be changed.
17690 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17691 // for this guy.
17692 instruct tlsLoadP(thread_RegP dst)
17693 %{
17694 match(Set dst (ThreadLocal));
17695
17696 ins_cost(0);
17697
17698 format %{ " -- \t// $dst=Thread::current(), empty" %}
17699
17700 size(0);
17701
17702 ins_encode( /*empty*/ );
17703
17704 ins_pipe(pipe_class_empty);
17705 %}
17706
17707 //----------PEEPHOLE RULES-----------------------------------------------------
17708 // These must follow all instruction definitions as they use the names
17709 // defined in the instructions definitions.
17710 //
17711 // peepmatch ( root_instr_name [preceding_instruction]* );
17712 //
17713 // peepconstraint %{
17714 // (instruction_number.operand_name relational_op instruction_number.operand_name
17715 // [, ...] );
17716 // // instruction numbers are zero-based using left to right order in peepmatch
17717 //
17718 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17719 // // provide an instruction_number.operand_name for each operand that appears
17720 // // in the replacement instruction's match rule
17721 //
17722 // ---------VM FLAGS---------------------------------------------------------
17723 //
17724 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17725 //
17726 // Each peephole rule is given an identifying number starting with zero and
17727 // increasing by one in the order seen by the parser. An individual peephole
17728 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17729 // on the command-line.
17730 //
17731 // ---------CURRENT LIMITATIONS----------------------------------------------
17732 //
17733 // Only match adjacent instructions in same basic block
17734 // Only equality constraints
17735 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17736 // Only one replacement instruction
17737 //
17738 // ---------EXAMPLE----------------------------------------------------------
17739 //
17740 // // pertinent parts of existing instructions in architecture description
17741 // instruct movI(iRegINoSp dst, iRegI src)
17742 // %{
17743 // match(Set dst (CopyI src));
17744 // %}
17745 //
17746 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17747 // %{
17748 // match(Set dst (AddI dst src));
17749 // effect(KILL cr);
17750 // %}
17751 //
17752 // // Change (inc mov) to lea
17753 // peephole %{
17754 // // increment preceded by register-register move
17755 // peepmatch ( incI_iReg movI );
17756 // // require that the destination register of the increment
17757 // // match the destination register of the move
17758 // peepconstraint ( 0.dst == 1.dst );
17759 // // construct a replacement instruction that sets
17760 // // the destination to ( move's source register + one )
17761 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17762 // %}
17763 //
17764
17765 // Implementation no longer uses movX instructions since
17766 // machine-independent system no longer uses CopyX nodes.
17767 //
17768 // peephole
17769 // %{
17770 // peepmatch (incI_iReg movI);
17771 // peepconstraint (0.dst == 1.dst);
17772 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17773 // %}
17774
17775 // peephole
17776 // %{
17777 // peepmatch (decI_iReg movI);
17778 // peepconstraint (0.dst == 1.dst);
17779 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17780 // %}
17781
17782 // peephole
17783 // %{
17784 // peepmatch (addI_iReg_imm movI);
17785 // peepconstraint (0.dst == 1.dst);
17786 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17787 // %}
17788
17789 // peephole
17790 // %{
17791 // peepmatch (incL_iReg movL);
17792 // peepconstraint (0.dst == 1.dst);
17793 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17794 // %}
17795
17796 // peephole
17797 // %{
17798 // peepmatch (decL_iReg movL);
17799 // peepconstraint (0.dst == 1.dst);
17800 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17801 // %}
17802
17803 // peephole
17804 // %{
17805 // peepmatch (addL_iReg_imm movL);
17806 // peepconstraint (0.dst == 1.dst);
17807 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17808 // %}
17809
17810 // peephole
17811 // %{
17812 // peepmatch (addP_iReg_imm movP);
17813 // peepconstraint (0.dst == 1.dst);
17814 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17815 // %}
17816
17817 // // Change load of spilled value to only a spill
17818 // instruct storeI(memory mem, iRegI src)
17819 // %{
17820 // match(Set mem (StoreI mem src));
17821 // %}
17822 //
17823 // instruct loadI(iRegINoSp dst, memory mem)
17824 // %{
17825 // match(Set dst (LoadI mem));
17826 // %}
17827 //
17828
17829 //----------SMARTSPILL RULES---------------------------------------------------
17830 // These must follow all instruction definitions as they use the names
17831 // defined in the instructions definitions.
17832
17833 // Local Variables:
17834 // mode: c++
17835 // End: