1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return 0; // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1652 // lea(rscratch1, RuntimeAddress(addr)
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else if (_entry_point == nullptr) {
1658 // See CallLeafNoFPIndirect
1659 return 1 * NativeInstruction::instruction_size;
1660 } else {
1661 return 6 * NativeInstruction::instruction_size;
1662 }
1663 }
1664
1665 //=============================================================================
1666
1667 #ifndef PRODUCT
1668 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1669 st->print("BREAKPOINT");
1670 }
1671 #endif
1672
1673 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1674 __ brk(0);
1675 }
1676
1677 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1678 return MachNode::size(ra_);
1679 }
1680
1681 //=============================================================================
1682
1683 #ifndef PRODUCT
1684 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1685 st->print("nop \t# %d bytes pad for loops and calls", _count);
1686 }
1687 #endif
1688
1689 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1690 for (int i = 0; i < _count; i++) {
1691 __ nop();
1692 }
1693 }
1694
1695 uint MachNopNode::size(PhaseRegAlloc*) const {
1696 return _count * NativeInstruction::instruction_size;
1697 }
1698
1699 //=============================================================================
1700 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1701
1702 int ConstantTable::calculate_table_base_offset() const {
1703 return 0; // absolute addressing, no offset
1704 }
1705
1706 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1707 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1708 ShouldNotReachHere();
1709 }
1710
1711 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1712 // Empty encoding
1713 }
1714
1715 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1716 return 0;
1717 }
1718
1719 #ifndef PRODUCT
1720 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1721 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1722 }
1723 #endif
1724
1725 #ifndef PRODUCT
1726 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1727 Compile* C = ra_->C;
1728
1729 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1730
1731 if (C->output()->need_stack_bang(framesize))
1732 st->print("# stack bang size=%d\n\t", framesize);
1733
1734 if (VM_Version::use_rop_protection()) {
1735 st->print("ldr zr, [lr]\n\t");
1736 st->print("paciaz\n\t");
1737 }
1738 if (framesize < ((1 << 9) + 2 * wordSize)) {
1739 st->print("sub sp, sp, #%d\n\t", framesize);
1740 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1741 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1742 } else {
1743 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1744 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1745 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1746 st->print("sub sp, sp, rscratch1");
1747 }
1748 if (C->stub_function() == nullptr) {
1749 st->print("\n\t");
1750 st->print("ldr rscratch1, [guard]\n\t");
1751 st->print("dmb ishld\n\t");
1752 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1753 st->print("cmp rscratch1, rscratch2\n\t");
1754 st->print("b.eq skip");
1755 st->print("\n\t");
1756 st->print("blr #nmethod_entry_barrier_stub\n\t");
1757 st->print("b skip\n\t");
1758 st->print("guard: int\n\t");
1759 st->print("\n\t");
1760 st->print("skip:\n\t");
1761 }
1762 }
1763 #endif
1764
1765 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1766 Compile* C = ra_->C;
1767
1768
1769 __ verified_entry(C, 0);
1770
1771 if (C->stub_function() == nullptr) {
1772 __ entry_barrier();
1773 }
1774
1775 if (!Compile::current()->output()->in_scratch_emit_size()) {
1776 __ bind(*_verified_entry);
1777 }
1778
1779 if (VerifyStackAtCalls) {
1780 Unimplemented();
1781 }
1782
1783 C->output()->set_frame_complete(__ offset());
1784
1785 if (C->has_mach_constant_base_node()) {
1786 // NOTE: We set the table base offset here because users might be
1787 // emitted before MachConstantBaseNode.
1788 ConstantTable& constant_table = C->output()->constant_table();
1789 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1790 }
1791 }
1792
1793 int MachPrologNode::reloc() const
1794 {
1795 return 0;
1796 }
1797
1798 //=============================================================================
1799
1800 #ifndef PRODUCT
1801 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1802 Compile* C = ra_->C;
1803 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1804
1805 st->print("# pop frame %d\n\t",framesize);
1806
1807 if (framesize == 0) {
1808 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1809 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1810 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1811 st->print("add sp, sp, #%d\n\t", framesize);
1812 } else {
1813 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1814 st->print("add sp, sp, rscratch1\n\t");
1815 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1816 }
1817 if (VM_Version::use_rop_protection()) {
1818 st->print("autiaz\n\t");
1819 st->print("ldr zr, [lr]\n\t");
1820 }
1821
1822 if (do_polling() && C->is_method_compilation()) {
1823 st->print("# test polling word\n\t");
1824 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1825 st->print("cmp sp, rscratch1\n\t");
1826 st->print("bhi #slow_path");
1827 }
1828 }
1829 #endif
1830
1831 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1832 Compile* C = ra_->C;
1833 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1834
1835 __ remove_frame(framesize, C->needs_stack_repair());
1836
1837 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1838 __ reserved_stack_check();
1839 }
1840
1841 if (do_polling() && C->is_method_compilation()) {
1842 Label dummy_label;
1843 Label* code_stub = &dummy_label;
1844 if (!C->output()->in_scratch_emit_size()) {
1845 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1846 C->output()->add_stub(stub);
1847 code_stub = &stub->entry();
1848 }
1849 __ relocate(relocInfo::poll_return_type);
1850 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1851 }
1852 }
1853
1854 int MachEpilogNode::reloc() const {
1855 // Return number of relocatable values contained in this instruction.
1856 return 1; // 1 for polling page.
1857 }
1858
1859 const Pipeline * MachEpilogNode::pipeline() const {
1860 return MachNode::pipeline_class();
1861 }
1862
1863 //=============================================================================
1864
1865 static enum RC rc_class(OptoReg::Name reg) {
1866
1867 if (reg == OptoReg::Bad) {
1868 return rc_bad;
1869 }
1870
1871 // we have 32 int registers * 2 halves
1872 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1873
1874 if (reg < slots_of_int_registers) {
1875 return rc_int;
1876 }
1877
1878 // we have 32 float register * 8 halves
1879 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1880 if (reg < slots_of_int_registers + slots_of_float_registers) {
1881 return rc_float;
1882 }
1883
1884 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1885 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1886 return rc_predicate;
1887 }
1888
1889 // Between predicate regs & stack is the flags.
1890 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1891
1892 return rc_stack;
1893 }
1894
1895 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1896 Compile* C = ra_->C;
1897
1898 // Get registers to move.
1899 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1900 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1901 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1902 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1903
1904 enum RC src_hi_rc = rc_class(src_hi);
1905 enum RC src_lo_rc = rc_class(src_lo);
1906 enum RC dst_hi_rc = rc_class(dst_hi);
1907 enum RC dst_lo_rc = rc_class(dst_lo);
1908
1909 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1910
1911 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1912 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1913 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1914 "expected aligned-adjacent pairs");
1915 }
1916
1917 if (src_lo == dst_lo && src_hi == dst_hi) {
1918 return 0; // Self copy, no move.
1919 }
1920
1921 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1922 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1923 int src_offset = ra_->reg2offset(src_lo);
1924 int dst_offset = ra_->reg2offset(dst_lo);
1925
1926 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1927 uint ireg = ideal_reg();
1928 if (ireg == Op_VecA && masm) {
1929 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1930 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1931 // stack->stack
1932 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1933 sve_vector_reg_size_in_bytes);
1934 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1935 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1936 sve_vector_reg_size_in_bytes);
1937 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1938 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1939 sve_vector_reg_size_in_bytes);
1940 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1941 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1942 as_FloatRegister(Matcher::_regEncode[src_lo]),
1943 as_FloatRegister(Matcher::_regEncode[src_lo]));
1944 } else {
1945 ShouldNotReachHere();
1946 }
1947 } else if (masm) {
1948 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1949 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1950 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1951 // stack->stack
1952 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1953 if (ireg == Op_VecD) {
1954 __ unspill(rscratch1, true, src_offset);
1955 __ spill(rscratch1, true, dst_offset);
1956 } else {
1957 __ spill_copy128(src_offset, dst_offset);
1958 }
1959 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1960 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1961 ireg == Op_VecD ? __ T8B : __ T16B,
1962 as_FloatRegister(Matcher::_regEncode[src_lo]));
1963 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1964 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1965 ireg == Op_VecD ? __ D : __ Q,
1966 ra_->reg2offset(dst_lo));
1967 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1968 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1969 ireg == Op_VecD ? __ D : __ Q,
1970 ra_->reg2offset(src_lo));
1971 } else {
1972 ShouldNotReachHere();
1973 }
1974 }
1975 } else if (masm) {
1976 switch (src_lo_rc) {
1977 case rc_int:
1978 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
1979 if (is64) {
1980 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1981 as_Register(Matcher::_regEncode[src_lo]));
1982 } else {
1983 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1984 as_Register(Matcher::_regEncode[src_lo]));
1985 }
1986 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1987 if (is64) {
1988 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1989 as_Register(Matcher::_regEncode[src_lo]));
1990 } else {
1991 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1992 as_Register(Matcher::_regEncode[src_lo]));
1993 }
1994 } else { // gpr --> stack spill
1995 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1996 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
1997 }
1998 break;
1999 case rc_float:
2000 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2001 if (is64) {
2002 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2003 as_FloatRegister(Matcher::_regEncode[src_lo]));
2004 } else {
2005 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2006 as_FloatRegister(Matcher::_regEncode[src_lo]));
2007 }
2008 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2009 if (is64) {
2010 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2011 as_FloatRegister(Matcher::_regEncode[src_lo]));
2012 } else {
2013 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2014 as_FloatRegister(Matcher::_regEncode[src_lo]));
2015 }
2016 } else { // fpr --> stack spill
2017 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2018 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2019 is64 ? __ D : __ S, dst_offset);
2020 }
2021 break;
2022 case rc_stack:
2023 if (dst_lo_rc == rc_int) { // stack --> gpr load
2024 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2025 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2026 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2027 is64 ? __ D : __ S, src_offset);
2028 } else if (dst_lo_rc == rc_predicate) {
2029 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2030 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2031 } else { // stack --> stack copy
2032 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2033 if (ideal_reg() == Op_RegVectMask) {
2034 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2035 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2036 } else {
2037 __ unspill(rscratch1, is64, src_offset);
2038 __ spill(rscratch1, is64, dst_offset);
2039 }
2040 }
2041 break;
2042 case rc_predicate:
2043 if (dst_lo_rc == rc_predicate) {
2044 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2045 } else if (dst_lo_rc == rc_stack) {
2046 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2047 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2048 } else {
2049 assert(false, "bad src and dst rc_class combination.");
2050 ShouldNotReachHere();
2051 }
2052 break;
2053 default:
2054 assert(false, "bad rc_class for spill");
2055 ShouldNotReachHere();
2056 }
2057 }
2058
2059 if (st) {
2060 st->print("spill ");
2061 if (src_lo_rc == rc_stack) {
2062 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2063 } else {
2064 st->print("%s -> ", Matcher::regName[src_lo]);
2065 }
2066 if (dst_lo_rc == rc_stack) {
2067 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2068 } else {
2069 st->print("%s", Matcher::regName[dst_lo]);
2070 }
2071 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2072 int vsize = 0;
2073 switch (ideal_reg()) {
2074 case Op_VecD:
2075 vsize = 64;
2076 break;
2077 case Op_VecX:
2078 vsize = 128;
2079 break;
2080 case Op_VecA:
2081 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2082 break;
2083 default:
2084 assert(false, "bad register type for spill");
2085 ShouldNotReachHere();
2086 }
2087 st->print("\t# vector spill size = %d", vsize);
2088 } else if (ideal_reg() == Op_RegVectMask) {
2089 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2090 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2091 st->print("\t# predicate spill size = %d", vsize);
2092 } else {
2093 st->print("\t# spill size = %d", is64 ? 64 : 32);
2094 }
2095 }
2096
2097 return 0;
2098
2099 }
2100
2101 #ifndef PRODUCT
2102 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2103 if (!ra_)
2104 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2105 else
2106 implementation(nullptr, ra_, false, st);
2107 }
2108 #endif
2109
2110 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2111 implementation(masm, ra_, false, nullptr);
2112 }
2113
2114 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2115 return MachNode::size(ra_);
2116 }
2117
2118 //=============================================================================
2119
2120 #ifndef PRODUCT
2121 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2122 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2123 int reg = ra_->get_reg_first(this);
2124 st->print("add %s, rsp, #%d]\t# box lock",
2125 Matcher::regName[reg], offset);
2126 }
2127 #endif
2128
2129 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2130 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2131 int reg = ra_->get_encode(this);
2132
2133 // This add will handle any 24-bit signed offset. 24 bits allows an
2134 // 8 megabyte stack frame.
2135 __ add(as_Register(reg), sp, offset);
2136 }
2137
2138 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2139 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2140 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2141
2142 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2143 return NativeInstruction::instruction_size;
2144 } else {
2145 return 2 * NativeInstruction::instruction_size;
2146 }
2147 }
2148
2149 ///=============================================================================
2150 #ifndef PRODUCT
2151 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2152 {
2153 st->print_cr("# MachVEPNode");
2154 if (!_verified) {
2155 st->print_cr("\t load_class");
2156 } else {
2157 st->print_cr("\t unpack_inline_arg");
2158 }
2159 }
2160 #endif
2161
2162 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2163 {
2164 if (!_verified) {
2165 __ ic_check(1);
2166 } else {
2167 // TODO 8284443 Avoid creation of temporary frame
2168 if (ra_->C->stub_function() == nullptr) {
2169 __ verified_entry(ra_->C, 0);
2170 __ entry_barrier();
2171 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2172 __ remove_frame(framesize, false);
2173 }
2174 // Unpack inline type args passed as oop and then jump to
2175 // the verified entry point (skipping the unverified entry).
2176 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2177 // Emit code for verified entry and save increment for stack repair on return
2178 __ verified_entry(ra_->C, sp_inc);
2179 if (Compile::current()->output()->in_scratch_emit_size()) {
2180 Label dummy_verified_entry;
2181 __ b(dummy_verified_entry);
2182 } else {
2183 __ b(*_verified_entry);
2184 }
2185 }
2186 }
2187
2188 //=============================================================================
2189 #ifndef PRODUCT
2190 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2191 {
2192 st->print_cr("# MachUEPNode");
2193 if (UseCompressedClassPointers) {
2194 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2195 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2196 st->print_cr("\tcmpw rscratch1, r10");
2197 } else {
2198 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2199 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2200 st->print_cr("\tcmp rscratch1, r10");
2201 }
2202 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2203 }
2204 #endif
2205
2206 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2207 {
2208 __ ic_check(InteriorEntryAlignment);
2209 }
2210
2211 // REQUIRED EMIT CODE
2212
2213 //=============================================================================
2214
2215 // Emit exception handler code.
2216 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2217 {
2218 // mov rscratch1 #exception_blob_entry_point
2219 // br rscratch1
2220 // Note that the code buffer's insts_mark is always relative to insts.
2221 // That's why we must use the macroassembler to generate a handler.
2222 address base = __ start_a_stub(size_exception_handler());
2223 if (base == nullptr) {
2224 ciEnv::current()->record_failure("CodeCache is full");
2225 return 0; // CodeBuffer::expand failed
2226 }
2227 int offset = __ offset();
2228 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2229 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2230 __ end_a_stub();
2231 return offset;
2232 }
2233
2234 // Emit deopt handler code.
2235 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2236 {
2237 // Note that the code buffer's insts_mark is always relative to insts.
2238 // That's why we must use the macroassembler to generate a handler.
2239 address base = __ start_a_stub(size_deopt_handler());
2240 if (base == nullptr) {
2241 ciEnv::current()->record_failure("CodeCache is full");
2242 return 0; // CodeBuffer::expand failed
2243 }
2244 int offset = __ offset();
2245
2246 __ adr(lr, __ pc());
2247 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2248
2249 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2250 __ end_a_stub();
2251 return offset;
2252 }
2253
2254 // REQUIRED MATCHER CODE
2255
2256 //=============================================================================
2257
2258 bool Matcher::match_rule_supported(int opcode) {
2259 if (!has_match_rule(opcode))
2260 return false;
2261
2262 switch (opcode) {
2263 case Op_OnSpinWait:
2264 return VM_Version::supports_on_spin_wait();
2265 case Op_CacheWB:
2266 case Op_CacheWBPreSync:
2267 case Op_CacheWBPostSync:
2268 if (!VM_Version::supports_data_cache_line_flush()) {
2269 return false;
2270 }
2271 break;
2272 case Op_ExpandBits:
2273 case Op_CompressBits:
2274 if (!VM_Version::supports_svebitperm()) {
2275 return false;
2276 }
2277 break;
2278 case Op_FmaF:
2279 case Op_FmaD:
2280 case Op_FmaVF:
2281 case Op_FmaVD:
2282 if (!UseFMA) {
2283 return false;
2284 }
2285 break;
2286 case Op_FmaHF:
2287 // UseFMA flag also needs to be checked along with FEAT_FP16
2288 if (!UseFMA || !is_feat_fp16_supported()) {
2289 return false;
2290 }
2291 break;
2292 case Op_AddHF:
2293 case Op_SubHF:
2294 case Op_MulHF:
2295 case Op_DivHF:
2296 case Op_MinHF:
2297 case Op_MaxHF:
2298 case Op_SqrtHF:
2299 // Half-precision floating point scalar operations require FEAT_FP16
2300 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2301 // features are supported.
2302 if (!is_feat_fp16_supported()) {
2303 return false;
2304 }
2305 break;
2306 }
2307
2308 return true; // Per default match rules are supported.
2309 }
2310
2311 const RegMask* Matcher::predicate_reg_mask(void) {
2312 return &_PR_REG_mask;
2313 }
2314
2315 bool Matcher::supports_vector_calling_convention(void) {
2316 return EnableVectorSupport;
2317 }
2318
2319 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2320 assert(EnableVectorSupport, "sanity");
2321 int lo = V0_num;
2322 int hi = V0_H_num;
2323 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2324 hi = V0_K_num;
2325 }
2326 return OptoRegPair(hi, lo);
2327 }
2328
2329 // Is this branch offset short enough that a short branch can be used?
2330 //
2331 // NOTE: If the platform does not provide any short branch variants, then
2332 // this method should return false for offset 0.
2333 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2334 // The passed offset is relative to address of the branch.
2335
2336 return (-32768 <= offset && offset < 32768);
2337 }
2338
2339 // Vector width in bytes.
2340 int Matcher::vector_width_in_bytes(BasicType bt) {
2341 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2342 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2343 // Minimum 2 values in vector
2344 if (size < 2*type2aelembytes(bt)) size = 0;
2345 // But never < 4
2346 if (size < 4) size = 0;
2347 return size;
2348 }
2349
2350 // Limits on vector size (number of elements) loaded into vector.
2351 int Matcher::max_vector_size(const BasicType bt) {
2352 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2353 }
2354
2355 int Matcher::min_vector_size(const BasicType bt) {
2356 int max_size = max_vector_size(bt);
2357 // Limit the min vector size to 8 bytes.
2358 int size = 8 / type2aelembytes(bt);
2359 if (bt == T_BYTE) {
2360 // To support vector api shuffle/rearrange.
2361 size = 4;
2362 } else if (bt == T_BOOLEAN) {
2363 // To support vector api load/store mask.
2364 size = 2;
2365 }
2366 if (size < 2) size = 2;
2367 return MIN2(size, max_size);
2368 }
2369
2370 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2371 return Matcher::max_vector_size(bt);
2372 }
2373
2374 // Actual max scalable vector register length.
2375 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2376 return Matcher::max_vector_size(bt);
2377 }
2378
2379 // Vector ideal reg.
2380 uint Matcher::vector_ideal_reg(int len) {
2381 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2382 return Op_VecA;
2383 }
2384 switch(len) {
2385 // For 16-bit/32-bit mask vector, reuse VecD.
2386 case 2:
2387 case 4:
2388 case 8: return Op_VecD;
2389 case 16: return Op_VecX;
2390 }
2391 ShouldNotReachHere();
2392 return 0;
2393 }
2394
2395 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2396 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2397 switch (ideal_reg) {
2398 case Op_VecA: return new vecAOper();
2399 case Op_VecD: return new vecDOper();
2400 case Op_VecX: return new vecXOper();
2401 }
2402 ShouldNotReachHere();
2403 return nullptr;
2404 }
2405
2406 bool Matcher::is_reg2reg_move(MachNode* m) {
2407 return false;
2408 }
2409
2410 bool Matcher::is_generic_vector(MachOper* opnd) {
2411 return opnd->opcode() == VREG;
2412 }
2413
2414 // Return whether or not this register is ever used as an argument.
2415 // This function is used on startup to build the trampoline stubs in
2416 // generateOptoStub. Registers not mentioned will be killed by the VM
2417 // call in the trampoline, and arguments in those registers not be
2418 // available to the callee.
2419 bool Matcher::can_be_java_arg(int reg)
2420 {
2421 return
2422 reg == R0_num || reg == R0_H_num ||
2423 reg == R1_num || reg == R1_H_num ||
2424 reg == R2_num || reg == R2_H_num ||
2425 reg == R3_num || reg == R3_H_num ||
2426 reg == R4_num || reg == R4_H_num ||
2427 reg == R5_num || reg == R5_H_num ||
2428 reg == R6_num || reg == R6_H_num ||
2429 reg == R7_num || reg == R7_H_num ||
2430 reg == V0_num || reg == V0_H_num ||
2431 reg == V1_num || reg == V1_H_num ||
2432 reg == V2_num || reg == V2_H_num ||
2433 reg == V3_num || reg == V3_H_num ||
2434 reg == V4_num || reg == V4_H_num ||
2435 reg == V5_num || reg == V5_H_num ||
2436 reg == V6_num || reg == V6_H_num ||
2437 reg == V7_num || reg == V7_H_num;
2438 }
2439
2440 bool Matcher::is_spillable_arg(int reg)
2441 {
2442 return can_be_java_arg(reg);
2443 }
2444
2445 uint Matcher::int_pressure_limit()
2446 {
2447 // JDK-8183543: When taking the number of available registers as int
2448 // register pressure threshold, the jtreg test:
2449 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2450 // failed due to C2 compilation failure with
2451 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2452 //
2453 // A derived pointer is live at CallNode and then is flagged by RA
2454 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2455 // derived pointers and lastly fail to spill after reaching maximum
2456 // number of iterations. Lowering the default pressure threshold to
2457 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2458 // a high register pressure area of the code so that split_DEF can
2459 // generate DefinitionSpillCopy for the derived pointer.
2460 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2461 if (!PreserveFramePointer) {
2462 // When PreserveFramePointer is off, frame pointer is allocatable,
2463 // but different from other SOC registers, it is excluded from
2464 // fatproj's mask because its save type is No-Save. Decrease 1 to
2465 // ensure high pressure at fatproj when PreserveFramePointer is off.
2466 // See check_pressure_at_fatproj().
2467 default_int_pressure_threshold--;
2468 }
2469 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2470 }
2471
2472 uint Matcher::float_pressure_limit()
2473 {
2474 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2475 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2476 }
2477
2478 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2479 return false;
2480 }
2481
2482 RegMask Matcher::divI_proj_mask() {
2483 ShouldNotReachHere();
2484 return RegMask();
2485 }
2486
2487 // Register for MODI projection of divmodI.
2488 RegMask Matcher::modI_proj_mask() {
2489 ShouldNotReachHere();
2490 return RegMask();
2491 }
2492
2493 // Register for DIVL projection of divmodL.
2494 RegMask Matcher::divL_proj_mask() {
2495 ShouldNotReachHere();
2496 return RegMask();
2497 }
2498
2499 // Register for MODL projection of divmodL.
2500 RegMask Matcher::modL_proj_mask() {
2501 ShouldNotReachHere();
2502 return RegMask();
2503 }
2504
2505 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2506 return FP_REG_mask();
2507 }
2508
2509 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2510 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2511 Node* u = addp->fast_out(i);
2512 if (u->is_LoadStore()) {
2513 // On AArch64, LoadStoreNodes (i.e. compare and swap
2514 // instructions) only take register indirect as an operand, so
2515 // any attempt to use an AddPNode as an input to a LoadStoreNode
2516 // must fail.
2517 return false;
2518 }
2519 if (u->is_Mem()) {
2520 int opsize = u->as_Mem()->memory_size();
2521 assert(opsize > 0, "unexpected memory operand size");
2522 if (u->as_Mem()->memory_size() != (1<<shift)) {
2523 return false;
2524 }
2525 }
2526 }
2527 return true;
2528 }
2529
2530 // Convert BootTest condition to Assembler condition.
2531 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2532 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2533 Assembler::Condition result;
2534 switch(cond) {
2535 case BoolTest::eq:
2536 result = Assembler::EQ; break;
2537 case BoolTest::ne:
2538 result = Assembler::NE; break;
2539 case BoolTest::le:
2540 result = Assembler::LE; break;
2541 case BoolTest::ge:
2542 result = Assembler::GE; break;
2543 case BoolTest::lt:
2544 result = Assembler::LT; break;
2545 case BoolTest::gt:
2546 result = Assembler::GT; break;
2547 case BoolTest::ule:
2548 result = Assembler::LS; break;
2549 case BoolTest::uge:
2550 result = Assembler::HS; break;
2551 case BoolTest::ult:
2552 result = Assembler::LO; break;
2553 case BoolTest::ugt:
2554 result = Assembler::HI; break;
2555 case BoolTest::overflow:
2556 result = Assembler::VS; break;
2557 case BoolTest::no_overflow:
2558 result = Assembler::VC; break;
2559 default:
2560 ShouldNotReachHere();
2561 return Assembler::Condition(-1);
2562 }
2563
2564 // Check conversion
2565 if (cond & BoolTest::unsigned_compare) {
2566 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2567 } else {
2568 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2569 }
2570
2571 return result;
2572 }
2573
2574 // Binary src (Replicate con)
2575 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2576 if (n == nullptr || m == nullptr) {
2577 return false;
2578 }
2579
2580 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2581 return false;
2582 }
2583
2584 Node* imm_node = m->in(1);
2585 if (!imm_node->is_Con()) {
2586 return false;
2587 }
2588
2589 const Type* t = imm_node->bottom_type();
2590 if (!(t->isa_int() || t->isa_long())) {
2591 return false;
2592 }
2593
2594 switch (n->Opcode()) {
2595 case Op_AndV:
2596 case Op_OrV:
2597 case Op_XorV: {
2598 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2599 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2600 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2601 }
2602 case Op_AddVB:
2603 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2604 case Op_AddVS:
2605 case Op_AddVI:
2606 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2607 case Op_AddVL:
2608 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2609 default:
2610 return false;
2611 }
2612 }
2613
2614 // (XorV src (Replicate m1))
2615 // (XorVMask src (MaskAll m1))
2616 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2617 if (n != nullptr && m != nullptr) {
2618 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2619 VectorNode::is_all_ones_vector(m);
2620 }
2621 return false;
2622 }
2623
2624 // Should the matcher clone input 'm' of node 'n'?
2625 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2626 if (is_vshift_con_pattern(n, m) ||
2627 is_vector_bitwise_not_pattern(n, m) ||
2628 is_valid_sve_arith_imm_pattern(n, m) ||
2629 is_encode_and_store_pattern(n, m)) {
2630 mstack.push(m, Visit);
2631 return true;
2632 }
2633 return false;
2634 }
2635
2636 // Should the Matcher clone shifts on addressing modes, expecting them
2637 // to be subsumed into complex addressing expressions or compute them
2638 // into registers?
2639 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2640
2641 // Loads and stores with indirect memory input (e.g., volatile loads and
2642 // stores) do not subsume the input into complex addressing expressions. If
2643 // the addressing expression is input to at least one such load or store, do
2644 // not clone the addressing expression. Query needs_acquiring_load and
2645 // needs_releasing_store as a proxy for indirect memory input, as it is not
2646 // possible to directly query for indirect memory input at this stage.
2647 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2648 Node* n = m->fast_out(i);
2649 if (n->is_Load() && needs_acquiring_load(n)) {
2650 return false;
2651 }
2652 if (n->is_Store() && needs_releasing_store(n)) {
2653 return false;
2654 }
2655 }
2656
2657 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2658 return true;
2659 }
2660
2661 Node *off = m->in(AddPNode::Offset);
2662 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2663 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2664 // Are there other uses besides address expressions?
2665 !is_visited(off)) {
2666 address_visited.set(off->_idx); // Flag as address_visited
2667 mstack.push(off->in(2), Visit);
2668 Node *conv = off->in(1);
2669 if (conv->Opcode() == Op_ConvI2L &&
2670 // Are there other uses besides address expressions?
2671 !is_visited(conv)) {
2672 address_visited.set(conv->_idx); // Flag as address_visited
2673 mstack.push(conv->in(1), Pre_Visit);
2674 } else {
2675 mstack.push(conv, Pre_Visit);
2676 }
2677 address_visited.test_set(m->_idx); // Flag as address_visited
2678 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2679 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2680 return true;
2681 } else if (off->Opcode() == Op_ConvI2L &&
2682 // Are there other uses besides address expressions?
2683 !is_visited(off)) {
2684 address_visited.test_set(m->_idx); // Flag as address_visited
2685 address_visited.set(off->_idx); // Flag as address_visited
2686 mstack.push(off->in(1), Pre_Visit);
2687 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2688 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2689 return true;
2690 }
2691 return false;
2692 }
2693
2694 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2695 { \
2696 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2697 guarantee(DISP == 0, "mode not permitted for volatile"); \
2698 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2699 __ INSN(REG, as_Register(BASE)); \
2700 }
2701
2702
2703 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2704 {
2705 Address::extend scale;
2706
2707 // Hooboy, this is fugly. We need a way to communicate to the
2708 // encoder that the index needs to be sign extended, so we have to
2709 // enumerate all the cases.
2710 switch (opcode) {
2711 case INDINDEXSCALEDI2L:
2712 case INDINDEXSCALEDI2LN:
2713 case INDINDEXI2L:
2714 case INDINDEXI2LN:
2715 scale = Address::sxtw(size);
2716 break;
2717 default:
2718 scale = Address::lsl(size);
2719 }
2720
2721 if (index == -1) {
2722 return Address(base, disp);
2723 } else {
2724 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2725 return Address(base, as_Register(index), scale);
2726 }
2727 }
2728
2729
2730 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2731 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2732 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2733 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2734 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2735
2736 // Used for all non-volatile memory accesses. The use of
2737 // $mem->opcode() to discover whether this pattern uses sign-extended
2738 // offsets is something of a kludge.
2739 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2740 Register reg, int opcode,
2741 Register base, int index, int scale, int disp,
2742 int size_in_memory)
2743 {
2744 Address addr = mem2address(opcode, base, index, scale, disp);
2745 if (addr.getMode() == Address::base_plus_offset) {
2746 /* Fix up any out-of-range offsets. */
2747 assert_different_registers(rscratch1, base);
2748 assert_different_registers(rscratch1, reg);
2749 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2750 }
2751 (masm->*insn)(reg, addr);
2752 }
2753
2754 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2755 FloatRegister reg, int opcode,
2756 Register base, int index, int size, int disp,
2757 int size_in_memory)
2758 {
2759 Address::extend scale;
2760
2761 switch (opcode) {
2762 case INDINDEXSCALEDI2L:
2763 case INDINDEXSCALEDI2LN:
2764 scale = Address::sxtw(size);
2765 break;
2766 default:
2767 scale = Address::lsl(size);
2768 }
2769
2770 if (index == -1) {
2771 // Fix up any out-of-range offsets.
2772 assert_different_registers(rscratch1, base);
2773 Address addr = Address(base, disp);
2774 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2775 (masm->*insn)(reg, addr);
2776 } else {
2777 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2778 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2779 }
2780 }
2781
2782 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2783 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2784 int opcode, Register base, int index, int size, int disp)
2785 {
2786 if (index == -1) {
2787 (masm->*insn)(reg, T, Address(base, disp));
2788 } else {
2789 assert(disp == 0, "unsupported address mode");
2790 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2791 }
2792 }
2793
2794 %}
2795
2796
2797
2798 //----------ENCODING BLOCK-----------------------------------------------------
2799 // This block specifies the encoding classes used by the compiler to
2800 // output byte streams. Encoding classes are parameterized macros
2801 // used by Machine Instruction Nodes in order to generate the bit
2802 // encoding of the instruction. Operands specify their base encoding
2803 // interface with the interface keyword. There are currently
2804 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2805 // COND_INTER. REG_INTER causes an operand to generate a function
2806 // which returns its register number when queried. CONST_INTER causes
2807 // an operand to generate a function which returns the value of the
2808 // constant when queried. MEMORY_INTER causes an operand to generate
2809 // four functions which return the Base Register, the Index Register,
2810 // the Scale Value, and the Offset Value of the operand when queried.
2811 // COND_INTER causes an operand to generate six functions which return
2812 // the encoding code (ie - encoding bits for the instruction)
2813 // associated with each basic boolean condition for a conditional
2814 // instruction.
2815 //
2816 // Instructions specify two basic values for encoding. Again, a
2817 // function is available to check if the constant displacement is an
2818 // oop. They use the ins_encode keyword to specify their encoding
2819 // classes (which must be a sequence of enc_class names, and their
2820 // parameters, specified in the encoding block), and they use the
2821 // opcode keyword to specify, in order, their primary, secondary, and
2822 // tertiary opcode. Only the opcode sections which a particular
2823 // instruction needs for encoding need to be specified.
2824 encode %{
2825 // Build emit functions for each basic byte or larger field in the
2826 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2827 // from C++ code in the enc_class source block. Emit functions will
2828 // live in the main source block for now. In future, we can
2829 // generalize this by adding a syntax that specifies the sizes of
2830 // fields in an order, so that the adlc can build the emit functions
2831 // automagically
2832
2833 // catch all for unimplemented encodings
2834 enc_class enc_unimplemented %{
2835 __ unimplemented("C2 catch all");
2836 %}
2837
2838 // BEGIN Non-volatile memory access
2839
2840 // This encoding class is generated automatically from ad_encode.m4.
2841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2842 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2843 Register dst_reg = as_Register($dst$$reg);
2844 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2845 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2846 %}
2847
2848 // This encoding class is generated automatically from ad_encode.m4.
2849 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2850 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2851 Register dst_reg = as_Register($dst$$reg);
2852 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2853 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2854 %}
2855
2856 // This encoding class is generated automatically from ad_encode.m4.
2857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2858 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2859 Register dst_reg = as_Register($dst$$reg);
2860 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2861 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2862 %}
2863
2864 // This encoding class is generated automatically from ad_encode.m4.
2865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2866 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2867 Register dst_reg = as_Register($dst$$reg);
2868 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2869 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2870 %}
2871
2872 // This encoding class is generated automatically from ad_encode.m4.
2873 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2874 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2875 Register dst_reg = as_Register($dst$$reg);
2876 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2877 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2878 %}
2879
2880 // This encoding class is generated automatically from ad_encode.m4.
2881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2882 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2883 Register dst_reg = as_Register($dst$$reg);
2884 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2885 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2886 %}
2887
2888 // This encoding class is generated automatically from ad_encode.m4.
2889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2890 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2891 Register dst_reg = as_Register($dst$$reg);
2892 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2893 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2894 %}
2895
2896 // This encoding class is generated automatically from ad_encode.m4.
2897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2898 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2899 Register dst_reg = as_Register($dst$$reg);
2900 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2901 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2902 %}
2903
2904 // This encoding class is generated automatically from ad_encode.m4.
2905 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2906 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2907 Register dst_reg = as_Register($dst$$reg);
2908 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2909 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2910 %}
2911
2912 // This encoding class is generated automatically from ad_encode.m4.
2913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2914 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2915 Register dst_reg = as_Register($dst$$reg);
2916 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2917 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2918 %}
2919
2920 // This encoding class is generated automatically from ad_encode.m4.
2921 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2922 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2923 Register dst_reg = as_Register($dst$$reg);
2924 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2925 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2926 %}
2927
2928 // This encoding class is generated automatically from ad_encode.m4.
2929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2930 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2931 Register dst_reg = as_Register($dst$$reg);
2932 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2933 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2934 %}
2935
2936 // This encoding class is generated automatically from ad_encode.m4.
2937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2938 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2939 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2940 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2941 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2942 %}
2943
2944 // This encoding class is generated automatically from ad_encode.m4.
2945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2946 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2947 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2948 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2949 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2950 %}
2951
2952 // This encoding class is generated automatically from ad_encode.m4.
2953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2954 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2955 Register src_reg = as_Register($src$$reg);
2956 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2957 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2958 %}
2959
2960 // This encoding class is generated automatically from ad_encode.m4.
2961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2962 enc_class aarch64_enc_strb0(memory1 mem) %{
2963 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2964 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2965 %}
2966
2967 // This encoding class is generated automatically from ad_encode.m4.
2968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2969 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2970 Register src_reg = as_Register($src$$reg);
2971 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2972 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2973 %}
2974
2975 // This encoding class is generated automatically from ad_encode.m4.
2976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2977 enc_class aarch64_enc_strh0(memory2 mem) %{
2978 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2979 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2980 %}
2981
2982 // This encoding class is generated automatically from ad_encode.m4.
2983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2984 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2985 Register src_reg = as_Register($src$$reg);
2986 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
2987 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2988 %}
2989
2990 // This encoding class is generated automatically from ad_encode.m4.
2991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2992 enc_class aarch64_enc_strw0(memory4 mem) %{
2993 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
2994 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2995 %}
2996
2997 // This encoding class is generated automatically from ad_encode.m4.
2998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2999 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3000 Register src_reg = as_Register($src$$reg);
3001 // we sometimes get asked to store the stack pointer into the
3002 // current thread -- we cannot do that directly on AArch64
3003 if (src_reg == r31_sp) {
3004 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3005 __ mov(rscratch2, sp);
3006 src_reg = rscratch2;
3007 }
3008 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3009 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3010 %}
3011
3012 // This encoding class is generated automatically from ad_encode.m4.
3013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3014 enc_class aarch64_enc_str0(memory8 mem) %{
3015 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3016 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3017 %}
3018
3019 // This encoding class is generated automatically from ad_encode.m4.
3020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3021 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3022 FloatRegister src_reg = as_FloatRegister($src$$reg);
3023 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3024 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3025 %}
3026
3027 // This encoding class is generated automatically from ad_encode.m4.
3028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3029 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3030 FloatRegister src_reg = as_FloatRegister($src$$reg);
3031 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3032 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3033 %}
3034
3035 // This encoding class is generated automatically from ad_encode.m4.
3036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3037 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3038 __ membar(Assembler::StoreStore);
3039 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3040 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3041 %}
3042
3043 // END Non-volatile memory access
3044
3045 // Vector loads and stores
3046 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3047 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3048 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3049 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3050 %}
3051
3052 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3053 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3054 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3055 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3056 %}
3057
3058 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3059 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3060 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3061 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3062 %}
3063
3064 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3065 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3066 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3067 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3068 %}
3069
3070 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3071 FloatRegister src_reg = as_FloatRegister($src$$reg);
3072 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3073 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3074 %}
3075
3076 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3077 FloatRegister src_reg = as_FloatRegister($src$$reg);
3078 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3079 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3080 %}
3081
3082 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3083 FloatRegister src_reg = as_FloatRegister($src$$reg);
3084 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3085 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3086 %}
3087
3088 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3089 FloatRegister src_reg = as_FloatRegister($src$$reg);
3090 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3091 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3092 %}
3093
3094 // volatile loads and stores
3095
3096 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3097 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3098 rscratch1, stlrb);
3099 %}
3100
3101 enc_class aarch64_enc_stlrb0(memory mem) %{
3102 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3103 rscratch1, stlrb);
3104 %}
3105
3106 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3107 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3108 rscratch1, stlrh);
3109 %}
3110
3111 enc_class aarch64_enc_stlrh0(memory mem) %{
3112 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3113 rscratch1, stlrh);
3114 %}
3115
3116 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3117 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3118 rscratch1, stlrw);
3119 %}
3120
3121 enc_class aarch64_enc_stlrw0(memory mem) %{
3122 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3123 rscratch1, stlrw);
3124 %}
3125
3126 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3127 Register dst_reg = as_Register($dst$$reg);
3128 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3129 rscratch1, ldarb);
3130 __ sxtbw(dst_reg, dst_reg);
3131 %}
3132
3133 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3134 Register dst_reg = as_Register($dst$$reg);
3135 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3136 rscratch1, ldarb);
3137 __ sxtb(dst_reg, dst_reg);
3138 %}
3139
3140 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3141 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3142 rscratch1, ldarb);
3143 %}
3144
3145 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3146 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3147 rscratch1, ldarb);
3148 %}
3149
3150 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3151 Register dst_reg = as_Register($dst$$reg);
3152 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3153 rscratch1, ldarh);
3154 __ sxthw(dst_reg, dst_reg);
3155 %}
3156
3157 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3158 Register dst_reg = as_Register($dst$$reg);
3159 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, ldarh);
3161 __ sxth(dst_reg, dst_reg);
3162 %}
3163
3164 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3165 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3166 rscratch1, ldarh);
3167 %}
3168
3169 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3170 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, ldarh);
3172 %}
3173
3174 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3175 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3176 rscratch1, ldarw);
3177 %}
3178
3179 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3180 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3181 rscratch1, ldarw);
3182 %}
3183
3184 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3185 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3186 rscratch1, ldar);
3187 %}
3188
3189 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3190 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3191 rscratch1, ldarw);
3192 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3193 %}
3194
3195 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3196 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3197 rscratch1, ldar);
3198 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3199 %}
3200
3201 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3202 Register src_reg = as_Register($src$$reg);
3203 // we sometimes get asked to store the stack pointer into the
3204 // current thread -- we cannot do that directly on AArch64
3205 if (src_reg == r31_sp) {
3206 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3207 __ mov(rscratch2, sp);
3208 src_reg = rscratch2;
3209 }
3210 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3211 rscratch1, stlr);
3212 %}
3213
3214 enc_class aarch64_enc_stlr0(memory mem) %{
3215 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3216 rscratch1, stlr);
3217 %}
3218
3219 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3220 {
3221 FloatRegister src_reg = as_FloatRegister($src$$reg);
3222 __ fmovs(rscratch2, src_reg);
3223 }
3224 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3225 rscratch1, stlrw);
3226 %}
3227
3228 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3229 {
3230 FloatRegister src_reg = as_FloatRegister($src$$reg);
3231 __ fmovd(rscratch2, src_reg);
3232 }
3233 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3234 rscratch1, stlr);
3235 %}
3236
3237 // synchronized read/update encodings
3238
3239 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3240 Register dst_reg = as_Register($dst$$reg);
3241 Register base = as_Register($mem$$base);
3242 int index = $mem$$index;
3243 int scale = $mem$$scale;
3244 int disp = $mem$$disp;
3245 if (index == -1) {
3246 if (disp != 0) {
3247 __ lea(rscratch1, Address(base, disp));
3248 __ ldaxr(dst_reg, rscratch1);
3249 } else {
3250 // TODO
3251 // should we ever get anything other than this case?
3252 __ ldaxr(dst_reg, base);
3253 }
3254 } else {
3255 Register index_reg = as_Register(index);
3256 if (disp == 0) {
3257 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3258 __ ldaxr(dst_reg, rscratch1);
3259 } else {
3260 __ lea(rscratch1, Address(base, disp));
3261 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3262 __ ldaxr(dst_reg, rscratch1);
3263 }
3264 }
3265 %}
3266
3267 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3268 Register src_reg = as_Register($src$$reg);
3269 Register base = as_Register($mem$$base);
3270 int index = $mem$$index;
3271 int scale = $mem$$scale;
3272 int disp = $mem$$disp;
3273 if (index == -1) {
3274 if (disp != 0) {
3275 __ lea(rscratch2, Address(base, disp));
3276 __ stlxr(rscratch1, src_reg, rscratch2);
3277 } else {
3278 // TODO
3279 // should we ever get anything other than this case?
3280 __ stlxr(rscratch1, src_reg, base);
3281 }
3282 } else {
3283 Register index_reg = as_Register(index);
3284 if (disp == 0) {
3285 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3286 __ stlxr(rscratch1, src_reg, rscratch2);
3287 } else {
3288 __ lea(rscratch2, Address(base, disp));
3289 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3290 __ stlxr(rscratch1, src_reg, rscratch2);
3291 }
3292 }
3293 __ cmpw(rscratch1, zr);
3294 %}
3295
3296 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3297 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3298 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3299 Assembler::xword, /*acquire*/ false, /*release*/ true,
3300 /*weak*/ false, noreg);
3301 %}
3302
3303 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3304 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3305 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3306 Assembler::word, /*acquire*/ false, /*release*/ true,
3307 /*weak*/ false, noreg);
3308 %}
3309
3310 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3311 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3312 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3313 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3314 /*weak*/ false, noreg);
3315 %}
3316
3317 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3318 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3319 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3320 Assembler::byte, /*acquire*/ false, /*release*/ true,
3321 /*weak*/ false, noreg);
3322 %}
3323
3324
3325 // The only difference between aarch64_enc_cmpxchg and
3326 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3327 // CompareAndSwap sequence to serve as a barrier on acquiring a
3328 // lock.
3329 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3330 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3331 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3332 Assembler::xword, /*acquire*/ true, /*release*/ true,
3333 /*weak*/ false, noreg);
3334 %}
3335
3336 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3337 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3338 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3339 Assembler::word, /*acquire*/ true, /*release*/ true,
3340 /*weak*/ false, noreg);
3341 %}
3342
3343 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3344 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3345 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3346 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3347 /*weak*/ false, noreg);
3348 %}
3349
3350 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3351 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3352 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3353 Assembler::byte, /*acquire*/ true, /*release*/ true,
3354 /*weak*/ false, noreg);
3355 %}
3356
3357 // auxiliary used for CompareAndSwapX to set result register
3358 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3359 Register res_reg = as_Register($res$$reg);
3360 __ cset(res_reg, Assembler::EQ);
3361 %}
3362
3363 // prefetch encodings
3364
3365 enc_class aarch64_enc_prefetchw(memory mem) %{
3366 Register base = as_Register($mem$$base);
3367 int index = $mem$$index;
3368 int scale = $mem$$scale;
3369 int disp = $mem$$disp;
3370 if (index == -1) {
3371 // Fix up any out-of-range offsets.
3372 assert_different_registers(rscratch1, base);
3373 Address addr = Address(base, disp);
3374 addr = __ legitimize_address(addr, 8, rscratch1);
3375 __ prfm(addr, PSTL1KEEP);
3376 } else {
3377 Register index_reg = as_Register(index);
3378 if (disp == 0) {
3379 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3380 } else {
3381 __ lea(rscratch1, Address(base, disp));
3382 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3383 }
3384 }
3385 %}
3386
3387 // mov encodings
3388
3389 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3390 uint32_t con = (uint32_t)$src$$constant;
3391 Register dst_reg = as_Register($dst$$reg);
3392 if (con == 0) {
3393 __ movw(dst_reg, zr);
3394 } else {
3395 __ movw(dst_reg, con);
3396 }
3397 %}
3398
3399 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3400 Register dst_reg = as_Register($dst$$reg);
3401 uint64_t con = (uint64_t)$src$$constant;
3402 if (con == 0) {
3403 __ mov(dst_reg, zr);
3404 } else {
3405 __ mov(dst_reg, con);
3406 }
3407 %}
3408
3409 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3410 Register dst_reg = as_Register($dst$$reg);
3411 address con = (address)$src$$constant;
3412 if (con == nullptr || con == (address)1) {
3413 ShouldNotReachHere();
3414 } else {
3415 relocInfo::relocType rtype = $src->constant_reloc();
3416 if (rtype == relocInfo::oop_type) {
3417 __ movoop(dst_reg, (jobject)con);
3418 } else if (rtype == relocInfo::metadata_type) {
3419 __ mov_metadata(dst_reg, (Metadata*)con);
3420 } else {
3421 assert(rtype == relocInfo::none, "unexpected reloc type");
3422 if (! __ is_valid_AArch64_address(con) ||
3423 con < (address)(uintptr_t)os::vm_page_size()) {
3424 __ mov(dst_reg, con);
3425 } else {
3426 uint64_t offset;
3427 __ adrp(dst_reg, con, offset);
3428 __ add(dst_reg, dst_reg, offset);
3429 }
3430 }
3431 }
3432 %}
3433
3434 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3435 Register dst_reg = as_Register($dst$$reg);
3436 __ mov(dst_reg, zr);
3437 %}
3438
3439 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3440 Register dst_reg = as_Register($dst$$reg);
3441 __ mov(dst_reg, (uint64_t)1);
3442 %}
3443
3444 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3445 __ load_byte_map_base($dst$$Register);
3446 %}
3447
3448 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3449 Register dst_reg = as_Register($dst$$reg);
3450 address con = (address)$src$$constant;
3451 if (con == nullptr) {
3452 ShouldNotReachHere();
3453 } else {
3454 relocInfo::relocType rtype = $src->constant_reloc();
3455 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3456 __ set_narrow_oop(dst_reg, (jobject)con);
3457 }
3458 %}
3459
3460 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3461 Register dst_reg = as_Register($dst$$reg);
3462 __ mov(dst_reg, zr);
3463 %}
3464
3465 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3466 Register dst_reg = as_Register($dst$$reg);
3467 address con = (address)$src$$constant;
3468 if (con == nullptr) {
3469 ShouldNotReachHere();
3470 } else {
3471 relocInfo::relocType rtype = $src->constant_reloc();
3472 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3473 __ set_narrow_klass(dst_reg, (Klass *)con);
3474 }
3475 %}
3476
3477 // arithmetic encodings
3478
3479 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3480 Register dst_reg = as_Register($dst$$reg);
3481 Register src_reg = as_Register($src1$$reg);
3482 int32_t con = (int32_t)$src2$$constant;
3483 // add has primary == 0, subtract has primary == 1
3484 if ($primary) { con = -con; }
3485 if (con < 0) {
3486 __ subw(dst_reg, src_reg, -con);
3487 } else {
3488 __ addw(dst_reg, src_reg, con);
3489 }
3490 %}
3491
3492 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3493 Register dst_reg = as_Register($dst$$reg);
3494 Register src_reg = as_Register($src1$$reg);
3495 int32_t con = (int32_t)$src2$$constant;
3496 // add has primary == 0, subtract has primary == 1
3497 if ($primary) { con = -con; }
3498 if (con < 0) {
3499 __ sub(dst_reg, src_reg, -con);
3500 } else {
3501 __ add(dst_reg, src_reg, con);
3502 }
3503 %}
3504
3505 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3506 Register dst_reg = as_Register($dst$$reg);
3507 Register src1_reg = as_Register($src1$$reg);
3508 Register src2_reg = as_Register($src2$$reg);
3509 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3510 %}
3511
3512 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3513 Register dst_reg = as_Register($dst$$reg);
3514 Register src1_reg = as_Register($src1$$reg);
3515 Register src2_reg = as_Register($src2$$reg);
3516 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3517 %}
3518
3519 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3520 Register dst_reg = as_Register($dst$$reg);
3521 Register src1_reg = as_Register($src1$$reg);
3522 Register src2_reg = as_Register($src2$$reg);
3523 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3524 %}
3525
3526 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3527 Register dst_reg = as_Register($dst$$reg);
3528 Register src1_reg = as_Register($src1$$reg);
3529 Register src2_reg = as_Register($src2$$reg);
3530 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3531 %}
3532
3533 // compare instruction encodings
3534
3535 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3536 Register reg1 = as_Register($src1$$reg);
3537 Register reg2 = as_Register($src2$$reg);
3538 __ cmpw(reg1, reg2);
3539 %}
3540
3541 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3542 Register reg = as_Register($src1$$reg);
3543 int32_t val = $src2$$constant;
3544 if (val >= 0) {
3545 __ subsw(zr, reg, val);
3546 } else {
3547 __ addsw(zr, reg, -val);
3548 }
3549 %}
3550
3551 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3552 Register reg1 = as_Register($src1$$reg);
3553 uint32_t val = (uint32_t)$src2$$constant;
3554 __ movw(rscratch1, val);
3555 __ cmpw(reg1, rscratch1);
3556 %}
3557
3558 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3559 Register reg1 = as_Register($src1$$reg);
3560 Register reg2 = as_Register($src2$$reg);
3561 __ cmp(reg1, reg2);
3562 %}
3563
3564 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3565 Register reg = as_Register($src1$$reg);
3566 int64_t val = $src2$$constant;
3567 if (val >= 0) {
3568 __ subs(zr, reg, val);
3569 } else if (val != -val) {
3570 __ adds(zr, reg, -val);
3571 } else {
3572 // aargh, Long.MIN_VALUE is a special case
3573 __ orr(rscratch1, zr, (uint64_t)val);
3574 __ subs(zr, reg, rscratch1);
3575 }
3576 %}
3577
3578 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3579 Register reg1 = as_Register($src1$$reg);
3580 uint64_t val = (uint64_t)$src2$$constant;
3581 __ mov(rscratch1, val);
3582 __ cmp(reg1, rscratch1);
3583 %}
3584
3585 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3586 Register reg1 = as_Register($src1$$reg);
3587 Register reg2 = as_Register($src2$$reg);
3588 __ cmp(reg1, reg2);
3589 %}
3590
3591 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3592 Register reg1 = as_Register($src1$$reg);
3593 Register reg2 = as_Register($src2$$reg);
3594 __ cmpw(reg1, reg2);
3595 %}
3596
3597 enc_class aarch64_enc_testp(iRegP src) %{
3598 Register reg = as_Register($src$$reg);
3599 __ cmp(reg, zr);
3600 %}
3601
3602 enc_class aarch64_enc_testn(iRegN src) %{
3603 Register reg = as_Register($src$$reg);
3604 __ cmpw(reg, zr);
3605 %}
3606
3607 enc_class aarch64_enc_b(label lbl) %{
3608 Label *L = $lbl$$label;
3609 __ b(*L);
3610 %}
3611
3612 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3613 Label *L = $lbl$$label;
3614 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3615 %}
3616
3617 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3618 Label *L = $lbl$$label;
3619 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3620 %}
3621
3622 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3623 %{
3624 Register sub_reg = as_Register($sub$$reg);
3625 Register super_reg = as_Register($super$$reg);
3626 Register temp_reg = as_Register($temp$$reg);
3627 Register result_reg = as_Register($result$$reg);
3628
3629 Label miss;
3630 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3631 nullptr, &miss,
3632 /*set_cond_codes:*/ true);
3633 if ($primary) {
3634 __ mov(result_reg, zr);
3635 }
3636 __ bind(miss);
3637 %}
3638
3639 enc_class aarch64_enc_java_static_call(method meth) %{
3640 address addr = (address)$meth$$method;
3641 address call;
3642 if (!_method) {
3643 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3644 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3645 if (call == nullptr) {
3646 ciEnv::current()->record_failure("CodeCache is full");
3647 return;
3648 }
3649 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3650 // The NOP here is purely to ensure that eliding a call to
3651 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3652 __ nop();
3653 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3654 } else {
3655 int method_index = resolved_method_index(masm);
3656 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3657 : static_call_Relocation::spec(method_index);
3658 call = __ trampoline_call(Address(addr, rspec));
3659 if (call == nullptr) {
3660 ciEnv::current()->record_failure("CodeCache is full");
3661 return;
3662 }
3663 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3664 // Calls of the same statically bound method can share
3665 // a stub to the interpreter.
3666 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3667 } else {
3668 // Emit stub for static call
3669 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3670 if (stub == nullptr) {
3671 ciEnv::current()->record_failure("CodeCache is full");
3672 return;
3673 }
3674 }
3675 }
3676
3677 __ post_call_nop();
3678
3679 // Only non uncommon_trap calls need to reinitialize ptrue.
3680 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3681 __ reinitialize_ptrue();
3682 }
3683 %}
3684
3685 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3686 int method_index = resolved_method_index(masm);
3687 address call = __ ic_call((address)$meth$$method, method_index);
3688 if (call == nullptr) {
3689 ciEnv::current()->record_failure("CodeCache is full");
3690 return;
3691 }
3692 __ post_call_nop();
3693 if (Compile::current()->max_vector_size() > 0) {
3694 __ reinitialize_ptrue();
3695 }
3696 %}
3697
3698 enc_class aarch64_enc_call_epilog() %{
3699 if (VerifyStackAtCalls) {
3700 // Check that stack depth is unchanged: find majik cookie on stack
3701 __ call_Unimplemented();
3702 }
3703 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
3704 // The last return value is not set by the callee but used to pass the null marker to compiled code.
3705 // Search for the corresponding projection, get the register and emit code that initialized it.
3706 uint con = (tf()->range_cc()->cnt() - 1);
3707 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3708 ProjNode* proj = fast_out(i)->as_Proj();
3709 if (proj->_con == con) {
3710 // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
3711 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3712 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3713 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3714 __ cmp(r0, zr);
3715 __ cset(toReg, Assembler::NE);
3716 if (reg->is_stack()) {
3717 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3718 __ str(toReg, Address(sp, st_off));
3719 }
3720 break;
3721 }
3722 }
3723 if (return_value_is_used()) {
3724 // An inline type is returned as fields in multiple registers.
3725 // R0 either contains an oop if the inline type is buffered or a pointer
3726 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3727 // if the lowest bit is set to allow C2 to use the oop after null checking.
3728 // r0 &= (r0 & 1) - 1
3729 __ andr(rscratch1, r0, 0x1);
3730 __ sub(rscratch1, rscratch1, 0x1);
3731 __ andr(r0, r0, rscratch1);
3732 }
3733 }
3734 %}
3735
3736 enc_class aarch64_enc_java_to_runtime(method meth) %{
3737 // some calls to generated routines (arraycopy code) are scheduled
3738 // by C2 as runtime calls. if so we can call them using a br (they
3739 // will be in a reachable segment) otherwise we have to use a blr
3740 // which loads the absolute address into a register.
3741 address entry = (address)$meth$$method;
3742 CodeBlob *cb = CodeCache::find_blob(entry);
3743 if (cb) {
3744 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3745 if (call == nullptr) {
3746 ciEnv::current()->record_failure("CodeCache is full");
3747 return;
3748 }
3749 __ post_call_nop();
3750 } else {
3751 Label retaddr;
3752 // Make the anchor frame walkable
3753 __ adr(rscratch2, retaddr);
3754 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3755 __ lea(rscratch1, RuntimeAddress(entry));
3756 __ blr(rscratch1);
3757 __ bind(retaddr);
3758 __ post_call_nop();
3759 }
3760 if (Compile::current()->max_vector_size() > 0) {
3761 __ reinitialize_ptrue();
3762 }
3763 %}
3764
3765 enc_class aarch64_enc_rethrow() %{
3766 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3767 %}
3768
3769 enc_class aarch64_enc_ret() %{
3770 #ifdef ASSERT
3771 if (Compile::current()->max_vector_size() > 0) {
3772 __ verify_ptrue();
3773 }
3774 #endif
3775 __ ret(lr);
3776 %}
3777
3778 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3779 Register target_reg = as_Register($jump_target$$reg);
3780 __ br(target_reg);
3781 %}
3782
3783 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3784 Register target_reg = as_Register($jump_target$$reg);
3785 // exception oop should be in r0
3786 // ret addr has been popped into lr
3787 // callee expects it in r3
3788 __ mov(r3, lr);
3789 __ br(target_reg);
3790 %}
3791
3792 %}
3793
3794 //----------FRAME--------------------------------------------------------------
3795 // Definition of frame structure and management information.
3796 //
3797 // S T A C K L A Y O U T Allocators stack-slot number
3798 // | (to get allocators register number
3799 // G Owned by | | v add OptoReg::stack0())
3800 // r CALLER | |
3801 // o | +--------+ pad to even-align allocators stack-slot
3802 // w V | pad0 | numbers; owned by CALLER
3803 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3804 // h ^ | in | 5
3805 // | | args | 4 Holes in incoming args owned by SELF
3806 // | | | | 3
3807 // | | +--------+
3808 // V | | old out| Empty on Intel, window on Sparc
3809 // | old |preserve| Must be even aligned.
3810 // | SP-+--------+----> Matcher::_old_SP, even aligned
3811 // | | in | 3 area for Intel ret address
3812 // Owned by |preserve| Empty on Sparc.
3813 // SELF +--------+
3814 // | | pad2 | 2 pad to align old SP
3815 // | +--------+ 1
3816 // | | locks | 0
3817 // | +--------+----> OptoReg::stack0(), even aligned
3818 // | | pad1 | 11 pad to align new SP
3819 // | +--------+
3820 // | | | 10
3821 // | | spills | 9 spills
3822 // V | | 8 (pad0 slot for callee)
3823 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3824 // ^ | out | 7
3825 // | | args | 6 Holes in outgoing args owned by CALLEE
3826 // Owned by +--------+
3827 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3828 // | new |preserve| Must be even-aligned.
3829 // | SP-+--------+----> Matcher::_new_SP, even aligned
3830 // | | |
3831 //
3832 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3833 // known from SELF's arguments and the Java calling convention.
3834 // Region 6-7 is determined per call site.
3835 // Note 2: If the calling convention leaves holes in the incoming argument
3836 // area, those holes are owned by SELF. Holes in the outgoing area
3837 // are owned by the CALLEE. Holes should not be necessary in the
3838 // incoming area, as the Java calling convention is completely under
3839 // the control of the AD file. Doubles can be sorted and packed to
3840 // avoid holes. Holes in the outgoing arguments may be necessary for
3841 // varargs C calling conventions.
3842 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3843 // even aligned with pad0 as needed.
3844 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3845 // (the latter is true on Intel but is it false on AArch64?)
3846 // region 6-11 is even aligned; it may be padded out more so that
3847 // the region from SP to FP meets the minimum stack alignment.
3848 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3849 // alignment. Region 11, pad1, may be dynamically extended so that
3850 // SP meets the minimum alignment.
3851
3852 frame %{
3853 // These three registers define part of the calling convention
3854 // between compiled code and the interpreter.
3855
3856 // Inline Cache Register or Method for I2C.
3857 inline_cache_reg(R12);
3858
3859 // Number of stack slots consumed by locking an object
3860 sync_stack_slots(2);
3861
3862 // Compiled code's Frame Pointer
3863 frame_pointer(R31);
3864
3865 // Interpreter stores its frame pointer in a register which is
3866 // stored to the stack by I2CAdaptors.
3867 // I2CAdaptors convert from interpreted java to compiled java.
3868 interpreter_frame_pointer(R29);
3869
3870 // Stack alignment requirement
3871 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3872
3873 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3874 // for calls to C. Supports the var-args backing area for register parms.
3875 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3876
3877 // The after-PROLOG location of the return address. Location of
3878 // return address specifies a type (REG or STACK) and a number
3879 // representing the register number (i.e. - use a register name) or
3880 // stack slot.
3881 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3882 // Otherwise, it is above the locks and verification slot and alignment word
3883 // TODO this may well be correct but need to check why that - 2 is there
3884 // ppc port uses 0 but we definitely need to allow for fixed_slots
3885 // which folds in the space used for monitors
3886 return_addr(STACK - 2 +
3887 align_up((Compile::current()->in_preserve_stack_slots() +
3888 Compile::current()->fixed_slots()),
3889 stack_alignment_in_slots()));
3890
3891 // Location of compiled Java return values. Same as C for now.
3892 return_value
3893 %{
3894 // TODO do we allow ideal_reg == Op_RegN???
3895 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3896 "only return normal values");
3897
3898 static const int lo[Op_RegL + 1] = { // enum name
3899 0, // Op_Node
3900 0, // Op_Set
3901 R0_num, // Op_RegN
3902 R0_num, // Op_RegI
3903 R0_num, // Op_RegP
3904 V0_num, // Op_RegF
3905 V0_num, // Op_RegD
3906 R0_num // Op_RegL
3907 };
3908
3909 static const int hi[Op_RegL + 1] = { // enum name
3910 0, // Op_Node
3911 0, // Op_Set
3912 OptoReg::Bad, // Op_RegN
3913 OptoReg::Bad, // Op_RegI
3914 R0_H_num, // Op_RegP
3915 OptoReg::Bad, // Op_RegF
3916 V0_H_num, // Op_RegD
3917 R0_H_num // Op_RegL
3918 };
3919
3920 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3921 %}
3922 %}
3923
3924 //----------ATTRIBUTES---------------------------------------------------------
3925 //----------Operand Attributes-------------------------------------------------
3926 op_attrib op_cost(1); // Required cost attribute
3927
3928 //----------Instruction Attributes---------------------------------------------
3929 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3930 ins_attrib ins_size(32); // Required size attribute (in bits)
3931 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3932 // a non-matching short branch variant
3933 // of some long branch?
3934 ins_attrib ins_alignment(4); // Required alignment attribute (must
3935 // be a power of 2) specifies the
3936 // alignment that some part of the
3937 // instruction (not necessarily the
3938 // start) requires. If > 1, a
3939 // compute_padding() function must be
3940 // provided for the instruction
3941
3942 // Whether this node is expanded during code emission into a sequence of
3943 // instructions and the first instruction can perform an implicit null check.
3944 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3945
3946 //----------OPERANDS-----------------------------------------------------------
3947 // Operand definitions must precede instruction definitions for correct parsing
3948 // in the ADLC because operands constitute user defined types which are used in
3949 // instruction definitions.
3950
3951 //----------Simple Operands----------------------------------------------------
3952
3953 // Integer operands 32 bit
3954 // 32 bit immediate
3955 operand immI()
3956 %{
3957 match(ConI);
3958
3959 op_cost(0);
3960 format %{ %}
3961 interface(CONST_INTER);
3962 %}
3963
3964 // 32 bit zero
3965 operand immI0()
3966 %{
3967 predicate(n->get_int() == 0);
3968 match(ConI);
3969
3970 op_cost(0);
3971 format %{ %}
3972 interface(CONST_INTER);
3973 %}
3974
3975 // 32 bit unit increment
3976 operand immI_1()
3977 %{
3978 predicate(n->get_int() == 1);
3979 match(ConI);
3980
3981 op_cost(0);
3982 format %{ %}
3983 interface(CONST_INTER);
3984 %}
3985
3986 // 32 bit unit decrement
3987 operand immI_M1()
3988 %{
3989 predicate(n->get_int() == -1);
3990 match(ConI);
3991
3992 op_cost(0);
3993 format %{ %}
3994 interface(CONST_INTER);
3995 %}
3996
3997 // Shift values for add/sub extension shift
3998 operand immIExt()
3999 %{
4000 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4001 match(ConI);
4002
4003 op_cost(0);
4004 format %{ %}
4005 interface(CONST_INTER);
4006 %}
4007
4008 operand immI_gt_1()
4009 %{
4010 predicate(n->get_int() > 1);
4011 match(ConI);
4012
4013 op_cost(0);
4014 format %{ %}
4015 interface(CONST_INTER);
4016 %}
4017
4018 operand immI_le_4()
4019 %{
4020 predicate(n->get_int() <= 4);
4021 match(ConI);
4022
4023 op_cost(0);
4024 format %{ %}
4025 interface(CONST_INTER);
4026 %}
4027
4028 operand immI_16()
4029 %{
4030 predicate(n->get_int() == 16);
4031 match(ConI);
4032
4033 op_cost(0);
4034 format %{ %}
4035 interface(CONST_INTER);
4036 %}
4037
4038 operand immI_24()
4039 %{
4040 predicate(n->get_int() == 24);
4041 match(ConI);
4042
4043 op_cost(0);
4044 format %{ %}
4045 interface(CONST_INTER);
4046 %}
4047
4048 operand immI_32()
4049 %{
4050 predicate(n->get_int() == 32);
4051 match(ConI);
4052
4053 op_cost(0);
4054 format %{ %}
4055 interface(CONST_INTER);
4056 %}
4057
4058 operand immI_48()
4059 %{
4060 predicate(n->get_int() == 48);
4061 match(ConI);
4062
4063 op_cost(0);
4064 format %{ %}
4065 interface(CONST_INTER);
4066 %}
4067
4068 operand immI_56()
4069 %{
4070 predicate(n->get_int() == 56);
4071 match(ConI);
4072
4073 op_cost(0);
4074 format %{ %}
4075 interface(CONST_INTER);
4076 %}
4077
4078 operand immI_255()
4079 %{
4080 predicate(n->get_int() == 255);
4081 match(ConI);
4082
4083 op_cost(0);
4084 format %{ %}
4085 interface(CONST_INTER);
4086 %}
4087
4088 operand immI_65535()
4089 %{
4090 predicate(n->get_int() == 65535);
4091 match(ConI);
4092
4093 op_cost(0);
4094 format %{ %}
4095 interface(CONST_INTER);
4096 %}
4097
4098 operand immI_positive()
4099 %{
4100 predicate(n->get_int() > 0);
4101 match(ConI);
4102
4103 op_cost(0);
4104 format %{ %}
4105 interface(CONST_INTER);
4106 %}
4107
4108 // BoolTest condition for signed compare
4109 operand immI_cmp_cond()
4110 %{
4111 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4112 match(ConI);
4113
4114 op_cost(0);
4115 format %{ %}
4116 interface(CONST_INTER);
4117 %}
4118
4119 // BoolTest condition for unsigned compare
4120 operand immI_cmpU_cond()
4121 %{
4122 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4123 match(ConI);
4124
4125 op_cost(0);
4126 format %{ %}
4127 interface(CONST_INTER);
4128 %}
4129
4130 operand immL_255()
4131 %{
4132 predicate(n->get_long() == 255L);
4133 match(ConL);
4134
4135 op_cost(0);
4136 format %{ %}
4137 interface(CONST_INTER);
4138 %}
4139
4140 operand immL_65535()
4141 %{
4142 predicate(n->get_long() == 65535L);
4143 match(ConL);
4144
4145 op_cost(0);
4146 format %{ %}
4147 interface(CONST_INTER);
4148 %}
4149
4150 operand immL_4294967295()
4151 %{
4152 predicate(n->get_long() == 4294967295L);
4153 match(ConL);
4154
4155 op_cost(0);
4156 format %{ %}
4157 interface(CONST_INTER);
4158 %}
4159
4160 operand immL_bitmask()
4161 %{
4162 predicate((n->get_long() != 0)
4163 && ((n->get_long() & 0xc000000000000000l) == 0)
4164 && is_power_of_2(n->get_long() + 1));
4165 match(ConL);
4166
4167 op_cost(0);
4168 format %{ %}
4169 interface(CONST_INTER);
4170 %}
4171
4172 operand immI_bitmask()
4173 %{
4174 predicate((n->get_int() != 0)
4175 && ((n->get_int() & 0xc0000000) == 0)
4176 && is_power_of_2(n->get_int() + 1));
4177 match(ConI);
4178
4179 op_cost(0);
4180 format %{ %}
4181 interface(CONST_INTER);
4182 %}
4183
4184 operand immL_positive_bitmaskI()
4185 %{
4186 predicate((n->get_long() != 0)
4187 && ((julong)n->get_long() < 0x80000000ULL)
4188 && is_power_of_2(n->get_long() + 1));
4189 match(ConL);
4190
4191 op_cost(0);
4192 format %{ %}
4193 interface(CONST_INTER);
4194 %}
4195
4196 // Scale values for scaled offset addressing modes (up to long but not quad)
4197 operand immIScale()
4198 %{
4199 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4200 match(ConI);
4201
4202 op_cost(0);
4203 format %{ %}
4204 interface(CONST_INTER);
4205 %}
4206
4207 // 5 bit signed integer
4208 operand immI5()
4209 %{
4210 predicate(Assembler::is_simm(n->get_int(), 5));
4211 match(ConI);
4212
4213 op_cost(0);
4214 format %{ %}
4215 interface(CONST_INTER);
4216 %}
4217
4218 // 7 bit unsigned integer
4219 operand immIU7()
4220 %{
4221 predicate(Assembler::is_uimm(n->get_int(), 7));
4222 match(ConI);
4223
4224 op_cost(0);
4225 format %{ %}
4226 interface(CONST_INTER);
4227 %}
4228
4229 // Offset for scaled or unscaled immediate loads and stores
4230 operand immIOffset()
4231 %{
4232 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4233 match(ConI);
4234
4235 op_cost(0);
4236 format %{ %}
4237 interface(CONST_INTER);
4238 %}
4239
4240 operand immIOffset1()
4241 %{
4242 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4243 match(ConI);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immIOffset2()
4251 %{
4252 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4253 match(ConI);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immIOffset4()
4261 %{
4262 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4263 match(ConI);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immIOffset8()
4271 %{
4272 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4273 match(ConI);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 operand immIOffset16()
4281 %{
4282 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4283 match(ConI);
4284
4285 op_cost(0);
4286 format %{ %}
4287 interface(CONST_INTER);
4288 %}
4289
4290 operand immLOffset()
4291 %{
4292 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4293 match(ConL);
4294
4295 op_cost(0);
4296 format %{ %}
4297 interface(CONST_INTER);
4298 %}
4299
4300 operand immLoffset1()
4301 %{
4302 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4303 match(ConL);
4304
4305 op_cost(0);
4306 format %{ %}
4307 interface(CONST_INTER);
4308 %}
4309
4310 operand immLoffset2()
4311 %{
4312 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4313 match(ConL);
4314
4315 op_cost(0);
4316 format %{ %}
4317 interface(CONST_INTER);
4318 %}
4319
4320 operand immLoffset4()
4321 %{
4322 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4323 match(ConL);
4324
4325 op_cost(0);
4326 format %{ %}
4327 interface(CONST_INTER);
4328 %}
4329
4330 operand immLoffset8()
4331 %{
4332 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4333 match(ConL);
4334
4335 op_cost(0);
4336 format %{ %}
4337 interface(CONST_INTER);
4338 %}
4339
4340 operand immLoffset16()
4341 %{
4342 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4343 match(ConL);
4344
4345 op_cost(0);
4346 format %{ %}
4347 interface(CONST_INTER);
4348 %}
4349
4350 // 5 bit signed long integer
4351 operand immL5()
4352 %{
4353 predicate(Assembler::is_simm(n->get_long(), 5));
4354 match(ConL);
4355
4356 op_cost(0);
4357 format %{ %}
4358 interface(CONST_INTER);
4359 %}
4360
4361 // 7 bit unsigned long integer
4362 operand immLU7()
4363 %{
4364 predicate(Assembler::is_uimm(n->get_long(), 7));
4365 match(ConL);
4366
4367 op_cost(0);
4368 format %{ %}
4369 interface(CONST_INTER);
4370 %}
4371
4372 // 8 bit signed value.
4373 operand immI8()
4374 %{
4375 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4376 match(ConI);
4377
4378 op_cost(0);
4379 format %{ %}
4380 interface(CONST_INTER);
4381 %}
4382
4383 // 8 bit signed value (simm8), or #simm8 LSL 8.
4384 operand immI8_shift8()
4385 %{
4386 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4387 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4388 match(ConI);
4389
4390 op_cost(0);
4391 format %{ %}
4392 interface(CONST_INTER);
4393 %}
4394
4395 // 8 bit signed value (simm8), or #simm8 LSL 8.
4396 operand immL8_shift8()
4397 %{
4398 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4399 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4400 match(ConL);
4401
4402 op_cost(0);
4403 format %{ %}
4404 interface(CONST_INTER);
4405 %}
4406
4407 // 8 bit integer valid for vector add sub immediate
4408 operand immBAddSubV()
4409 %{
4410 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4411 match(ConI);
4412
4413 op_cost(0);
4414 format %{ %}
4415 interface(CONST_INTER);
4416 %}
4417
4418 // 32 bit integer valid for add sub immediate
4419 operand immIAddSub()
4420 %{
4421 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4422 match(ConI);
4423 op_cost(0);
4424 format %{ %}
4425 interface(CONST_INTER);
4426 %}
4427
4428 // 32 bit integer valid for vector add sub immediate
4429 operand immIAddSubV()
4430 %{
4431 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4432 match(ConI);
4433
4434 op_cost(0);
4435 format %{ %}
4436 interface(CONST_INTER);
4437 %}
4438
4439 // 32 bit unsigned integer valid for logical immediate
4440
4441 operand immBLog()
4442 %{
4443 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4444 match(ConI);
4445
4446 op_cost(0);
4447 format %{ %}
4448 interface(CONST_INTER);
4449 %}
4450
4451 operand immSLog()
4452 %{
4453 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4454 match(ConI);
4455
4456 op_cost(0);
4457 format %{ %}
4458 interface(CONST_INTER);
4459 %}
4460
4461 operand immILog()
4462 %{
4463 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4464 match(ConI);
4465
4466 op_cost(0);
4467 format %{ %}
4468 interface(CONST_INTER);
4469 %}
4470
4471 // Integer operands 64 bit
4472 // 64 bit immediate
4473 operand immL()
4474 %{
4475 match(ConL);
4476
4477 op_cost(0);
4478 format %{ %}
4479 interface(CONST_INTER);
4480 %}
4481
4482 // 64 bit zero
4483 operand immL0()
4484 %{
4485 predicate(n->get_long() == 0);
4486 match(ConL);
4487
4488 op_cost(0);
4489 format %{ %}
4490 interface(CONST_INTER);
4491 %}
4492
4493 // 64 bit unit decrement
4494 operand immL_M1()
4495 %{
4496 predicate(n->get_long() == -1);
4497 match(ConL);
4498
4499 op_cost(0);
4500 format %{ %}
4501 interface(CONST_INTER);
4502 %}
4503
4504 // 64 bit integer valid for add sub immediate
4505 operand immLAddSub()
4506 %{
4507 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4508 match(ConL);
4509 op_cost(0);
4510 format %{ %}
4511 interface(CONST_INTER);
4512 %}
4513
4514 // 64 bit integer valid for addv subv immediate
4515 operand immLAddSubV()
4516 %{
4517 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4518 match(ConL);
4519
4520 op_cost(0);
4521 format %{ %}
4522 interface(CONST_INTER);
4523 %}
4524
4525 // 64 bit integer valid for logical immediate
4526 operand immLLog()
4527 %{
4528 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4529 match(ConL);
4530 op_cost(0);
4531 format %{ %}
4532 interface(CONST_INTER);
4533 %}
4534
4535 // Long Immediate: low 32-bit mask
4536 operand immL_32bits()
4537 %{
4538 predicate(n->get_long() == 0xFFFFFFFFL);
4539 match(ConL);
4540 op_cost(0);
4541 format %{ %}
4542 interface(CONST_INTER);
4543 %}
4544
4545 // Pointer operands
4546 // Pointer Immediate
4547 operand immP()
4548 %{
4549 match(ConP);
4550
4551 op_cost(0);
4552 format %{ %}
4553 interface(CONST_INTER);
4554 %}
4555
4556 // nullptr Pointer Immediate
4557 operand immP0()
4558 %{
4559 predicate(n->get_ptr() == 0);
4560 match(ConP);
4561
4562 op_cost(0);
4563 format %{ %}
4564 interface(CONST_INTER);
4565 %}
4566
4567 // Pointer Immediate One
4568 // this is used in object initialization (initial object header)
4569 operand immP_1()
4570 %{
4571 predicate(n->get_ptr() == 1);
4572 match(ConP);
4573
4574 op_cost(0);
4575 format %{ %}
4576 interface(CONST_INTER);
4577 %}
4578
4579 // Card Table Byte Map Base
4580 operand immByteMapBase()
4581 %{
4582 // Get base of card map
4583 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4584 SHENANDOAHGC_ONLY(!BarrierSet::barrier_set()->is_a(BarrierSet::ShenandoahBarrierSet) &&)
4585 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4586 match(ConP);
4587
4588 op_cost(0);
4589 format %{ %}
4590 interface(CONST_INTER);
4591 %}
4592
4593 // Float and Double operands
4594 // Double Immediate
4595 operand immD()
4596 %{
4597 match(ConD);
4598 op_cost(0);
4599 format %{ %}
4600 interface(CONST_INTER);
4601 %}
4602
4603 // Double Immediate: +0.0d
4604 operand immD0()
4605 %{
4606 predicate(jlong_cast(n->getd()) == 0);
4607 match(ConD);
4608
4609 op_cost(0);
4610 format %{ %}
4611 interface(CONST_INTER);
4612 %}
4613
4614 // constant 'double +0.0'.
4615 operand immDPacked()
4616 %{
4617 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4618 match(ConD);
4619 op_cost(0);
4620 format %{ %}
4621 interface(CONST_INTER);
4622 %}
4623
4624 // Float Immediate
4625 operand immF()
4626 %{
4627 match(ConF);
4628 op_cost(0);
4629 format %{ %}
4630 interface(CONST_INTER);
4631 %}
4632
4633 // Float Immediate: +0.0f.
4634 operand immF0()
4635 %{
4636 predicate(jint_cast(n->getf()) == 0);
4637 match(ConF);
4638
4639 op_cost(0);
4640 format %{ %}
4641 interface(CONST_INTER);
4642 %}
4643
4644 // Half Float (FP16) Immediate
4645 operand immH()
4646 %{
4647 match(ConH);
4648 op_cost(0);
4649 format %{ %}
4650 interface(CONST_INTER);
4651 %}
4652
4653 //
4654 operand immFPacked()
4655 %{
4656 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4657 match(ConF);
4658 op_cost(0);
4659 format %{ %}
4660 interface(CONST_INTER);
4661 %}
4662
4663 // Narrow pointer operands
4664 // Narrow Pointer Immediate
4665 operand immN()
4666 %{
4667 match(ConN);
4668
4669 op_cost(0);
4670 format %{ %}
4671 interface(CONST_INTER);
4672 %}
4673
4674 // Narrow nullptr Pointer Immediate
4675 operand immN0()
4676 %{
4677 predicate(n->get_narrowcon() == 0);
4678 match(ConN);
4679
4680 op_cost(0);
4681 format %{ %}
4682 interface(CONST_INTER);
4683 %}
4684
4685 operand immNKlass()
4686 %{
4687 match(ConNKlass);
4688
4689 op_cost(0);
4690 format %{ %}
4691 interface(CONST_INTER);
4692 %}
4693
4694 // Integer 32 bit Register Operands
4695 // Integer 32 bitRegister (excludes SP)
4696 operand iRegI()
4697 %{
4698 constraint(ALLOC_IN_RC(any_reg32));
4699 match(RegI);
4700 match(iRegINoSp);
4701 op_cost(0);
4702 format %{ %}
4703 interface(REG_INTER);
4704 %}
4705
4706 // Integer 32 bit Register not Special
4707 operand iRegINoSp()
4708 %{
4709 constraint(ALLOC_IN_RC(no_special_reg32));
4710 match(RegI);
4711 op_cost(0);
4712 format %{ %}
4713 interface(REG_INTER);
4714 %}
4715
4716 // Integer 64 bit Register Operands
4717 // Integer 64 bit Register (includes SP)
4718 operand iRegL()
4719 %{
4720 constraint(ALLOC_IN_RC(any_reg));
4721 match(RegL);
4722 match(iRegLNoSp);
4723 op_cost(0);
4724 format %{ %}
4725 interface(REG_INTER);
4726 %}
4727
4728 // Integer 64 bit Register not Special
4729 operand iRegLNoSp()
4730 %{
4731 constraint(ALLOC_IN_RC(no_special_reg));
4732 match(RegL);
4733 match(iRegL_R0);
4734 format %{ %}
4735 interface(REG_INTER);
4736 %}
4737
4738 // Pointer Register Operands
4739 // Pointer Register
4740 operand iRegP()
4741 %{
4742 constraint(ALLOC_IN_RC(ptr_reg));
4743 match(RegP);
4744 match(iRegPNoSp);
4745 match(iRegP_R0);
4746 //match(iRegP_R2);
4747 //match(iRegP_R4);
4748 match(iRegP_R5);
4749 match(thread_RegP);
4750 op_cost(0);
4751 format %{ %}
4752 interface(REG_INTER);
4753 %}
4754
4755 // Pointer 64 bit Register not Special
4756 operand iRegPNoSp()
4757 %{
4758 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4759 match(RegP);
4760 // match(iRegP);
4761 // match(iRegP_R0);
4762 // match(iRegP_R2);
4763 // match(iRegP_R4);
4764 // match(iRegP_R5);
4765 // match(thread_RegP);
4766 op_cost(0);
4767 format %{ %}
4768 interface(REG_INTER);
4769 %}
4770
4771 // This operand is not allowed to use rfp even if
4772 // rfp is not used to hold the frame pointer.
4773 operand iRegPNoSpNoRfp()
4774 %{
4775 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4776 match(RegP);
4777 match(iRegPNoSp);
4778 op_cost(0);
4779 format %{ %}
4780 interface(REG_INTER);
4781 %}
4782
4783 // Pointer 64 bit Register R0 only
4784 operand iRegP_R0()
4785 %{
4786 constraint(ALLOC_IN_RC(r0_reg));
4787 match(RegP);
4788 // match(iRegP);
4789 match(iRegPNoSp);
4790 op_cost(0);
4791 format %{ %}
4792 interface(REG_INTER);
4793 %}
4794
4795 // Pointer 64 bit Register R1 only
4796 operand iRegP_R1()
4797 %{
4798 constraint(ALLOC_IN_RC(r1_reg));
4799 match(RegP);
4800 // match(iRegP);
4801 match(iRegPNoSp);
4802 op_cost(0);
4803 format %{ %}
4804 interface(REG_INTER);
4805 %}
4806
4807 // Pointer 64 bit Register R2 only
4808 operand iRegP_R2()
4809 %{
4810 constraint(ALLOC_IN_RC(r2_reg));
4811 match(RegP);
4812 // match(iRegP);
4813 match(iRegPNoSp);
4814 op_cost(0);
4815 format %{ %}
4816 interface(REG_INTER);
4817 %}
4818
4819 // Pointer 64 bit Register R3 only
4820 operand iRegP_R3()
4821 %{
4822 constraint(ALLOC_IN_RC(r3_reg));
4823 match(RegP);
4824 // match(iRegP);
4825 match(iRegPNoSp);
4826 op_cost(0);
4827 format %{ %}
4828 interface(REG_INTER);
4829 %}
4830
4831 // Pointer 64 bit Register R4 only
4832 operand iRegP_R4()
4833 %{
4834 constraint(ALLOC_IN_RC(r4_reg));
4835 match(RegP);
4836 // match(iRegP);
4837 match(iRegPNoSp);
4838 op_cost(0);
4839 format %{ %}
4840 interface(REG_INTER);
4841 %}
4842
4843 // Pointer 64 bit Register R5 only
4844 operand iRegP_R5()
4845 %{
4846 constraint(ALLOC_IN_RC(r5_reg));
4847 match(RegP);
4848 // match(iRegP);
4849 match(iRegPNoSp);
4850 op_cost(0);
4851 format %{ %}
4852 interface(REG_INTER);
4853 %}
4854
4855 // Pointer 64 bit Register R10 only
4856 operand iRegP_R10()
4857 %{
4858 constraint(ALLOC_IN_RC(r10_reg));
4859 match(RegP);
4860 // match(iRegP);
4861 match(iRegPNoSp);
4862 op_cost(0);
4863 format %{ %}
4864 interface(REG_INTER);
4865 %}
4866
4867 // Long 64 bit Register R0 only
4868 operand iRegL_R0()
4869 %{
4870 constraint(ALLOC_IN_RC(r0_reg));
4871 match(RegL);
4872 match(iRegLNoSp);
4873 op_cost(0);
4874 format %{ %}
4875 interface(REG_INTER);
4876 %}
4877
4878 // Long 64 bit Register R11 only
4879 operand iRegL_R11()
4880 %{
4881 constraint(ALLOC_IN_RC(r11_reg));
4882 match(RegL);
4883 match(iRegLNoSp);
4884 op_cost(0);
4885 format %{ %}
4886 interface(REG_INTER);
4887 %}
4888
4889 // Register R0 only
4890 operand iRegI_R0()
4891 %{
4892 constraint(ALLOC_IN_RC(int_r0_reg));
4893 match(RegI);
4894 match(iRegINoSp);
4895 op_cost(0);
4896 format %{ %}
4897 interface(REG_INTER);
4898 %}
4899
4900 // Register R2 only
4901 operand iRegI_R2()
4902 %{
4903 constraint(ALLOC_IN_RC(int_r2_reg));
4904 match(RegI);
4905 match(iRegINoSp);
4906 op_cost(0);
4907 format %{ %}
4908 interface(REG_INTER);
4909 %}
4910
4911 // Register R3 only
4912 operand iRegI_R3()
4913 %{
4914 constraint(ALLOC_IN_RC(int_r3_reg));
4915 match(RegI);
4916 match(iRegINoSp);
4917 op_cost(0);
4918 format %{ %}
4919 interface(REG_INTER);
4920 %}
4921
4922
4923 // Register R4 only
4924 operand iRegI_R4()
4925 %{
4926 constraint(ALLOC_IN_RC(int_r4_reg));
4927 match(RegI);
4928 match(iRegINoSp);
4929 op_cost(0);
4930 format %{ %}
4931 interface(REG_INTER);
4932 %}
4933
4934
4935 // Pointer Register Operands
4936 // Narrow Pointer Register
4937 operand iRegN()
4938 %{
4939 constraint(ALLOC_IN_RC(any_reg32));
4940 match(RegN);
4941 match(iRegNNoSp);
4942 op_cost(0);
4943 format %{ %}
4944 interface(REG_INTER);
4945 %}
4946
4947 // Integer 64 bit Register not Special
4948 operand iRegNNoSp()
4949 %{
4950 constraint(ALLOC_IN_RC(no_special_reg32));
4951 match(RegN);
4952 op_cost(0);
4953 format %{ %}
4954 interface(REG_INTER);
4955 %}
4956
4957 // Float Register
4958 // Float register operands
4959 operand vRegF()
4960 %{
4961 constraint(ALLOC_IN_RC(float_reg));
4962 match(RegF);
4963
4964 op_cost(0);
4965 format %{ %}
4966 interface(REG_INTER);
4967 %}
4968
4969 // Double Register
4970 // Double register operands
4971 operand vRegD()
4972 %{
4973 constraint(ALLOC_IN_RC(double_reg));
4974 match(RegD);
4975
4976 op_cost(0);
4977 format %{ %}
4978 interface(REG_INTER);
4979 %}
4980
4981 // Generic vector class. This will be used for
4982 // all vector operands, including NEON and SVE.
4983 operand vReg()
4984 %{
4985 constraint(ALLOC_IN_RC(dynamic));
4986 match(VecA);
4987 match(VecD);
4988 match(VecX);
4989
4990 op_cost(0);
4991 format %{ %}
4992 interface(REG_INTER);
4993 %}
4994
4995 operand vecA()
4996 %{
4997 constraint(ALLOC_IN_RC(vectora_reg));
4998 match(VecA);
4999
5000 op_cost(0);
5001 format %{ %}
5002 interface(REG_INTER);
5003 %}
5004
5005 operand vecD()
5006 %{
5007 constraint(ALLOC_IN_RC(vectord_reg));
5008 match(VecD);
5009
5010 op_cost(0);
5011 format %{ %}
5012 interface(REG_INTER);
5013 %}
5014
5015 operand vecX()
5016 %{
5017 constraint(ALLOC_IN_RC(vectorx_reg));
5018 match(VecX);
5019
5020 op_cost(0);
5021 format %{ %}
5022 interface(REG_INTER);
5023 %}
5024
5025 operand vRegD_V0()
5026 %{
5027 constraint(ALLOC_IN_RC(v0_reg));
5028 match(RegD);
5029 op_cost(0);
5030 format %{ %}
5031 interface(REG_INTER);
5032 %}
5033
5034 operand vRegD_V1()
5035 %{
5036 constraint(ALLOC_IN_RC(v1_reg));
5037 match(RegD);
5038 op_cost(0);
5039 format %{ %}
5040 interface(REG_INTER);
5041 %}
5042
5043 operand vRegD_V2()
5044 %{
5045 constraint(ALLOC_IN_RC(v2_reg));
5046 match(RegD);
5047 op_cost(0);
5048 format %{ %}
5049 interface(REG_INTER);
5050 %}
5051
5052 operand vRegD_V3()
5053 %{
5054 constraint(ALLOC_IN_RC(v3_reg));
5055 match(RegD);
5056 op_cost(0);
5057 format %{ %}
5058 interface(REG_INTER);
5059 %}
5060
5061 operand vRegD_V4()
5062 %{
5063 constraint(ALLOC_IN_RC(v4_reg));
5064 match(RegD);
5065 op_cost(0);
5066 format %{ %}
5067 interface(REG_INTER);
5068 %}
5069
5070 operand vRegD_V5()
5071 %{
5072 constraint(ALLOC_IN_RC(v5_reg));
5073 match(RegD);
5074 op_cost(0);
5075 format %{ %}
5076 interface(REG_INTER);
5077 %}
5078
5079 operand vRegD_V6()
5080 %{
5081 constraint(ALLOC_IN_RC(v6_reg));
5082 match(RegD);
5083 op_cost(0);
5084 format %{ %}
5085 interface(REG_INTER);
5086 %}
5087
5088 operand vRegD_V7()
5089 %{
5090 constraint(ALLOC_IN_RC(v7_reg));
5091 match(RegD);
5092 op_cost(0);
5093 format %{ %}
5094 interface(REG_INTER);
5095 %}
5096
5097 operand vRegD_V12()
5098 %{
5099 constraint(ALLOC_IN_RC(v12_reg));
5100 match(RegD);
5101 op_cost(0);
5102 format %{ %}
5103 interface(REG_INTER);
5104 %}
5105
5106 operand vRegD_V13()
5107 %{
5108 constraint(ALLOC_IN_RC(v13_reg));
5109 match(RegD);
5110 op_cost(0);
5111 format %{ %}
5112 interface(REG_INTER);
5113 %}
5114
5115 operand pReg()
5116 %{
5117 constraint(ALLOC_IN_RC(pr_reg));
5118 match(RegVectMask);
5119 match(pRegGov);
5120 op_cost(0);
5121 format %{ %}
5122 interface(REG_INTER);
5123 %}
5124
5125 operand pRegGov()
5126 %{
5127 constraint(ALLOC_IN_RC(gov_pr));
5128 match(RegVectMask);
5129 match(pReg);
5130 op_cost(0);
5131 format %{ %}
5132 interface(REG_INTER);
5133 %}
5134
5135 operand pRegGov_P0()
5136 %{
5137 constraint(ALLOC_IN_RC(p0_reg));
5138 match(RegVectMask);
5139 op_cost(0);
5140 format %{ %}
5141 interface(REG_INTER);
5142 %}
5143
5144 operand pRegGov_P1()
5145 %{
5146 constraint(ALLOC_IN_RC(p1_reg));
5147 match(RegVectMask);
5148 op_cost(0);
5149 format %{ %}
5150 interface(REG_INTER);
5151 %}
5152
5153 // Flags register, used as output of signed compare instructions
5154
5155 // note that on AArch64 we also use this register as the output for
5156 // for floating point compare instructions (CmpF CmpD). this ensures
5157 // that ordered inequality tests use GT, GE, LT or LE none of which
5158 // pass through cases where the result is unordered i.e. one or both
5159 // inputs to the compare is a NaN. this means that the ideal code can
5160 // replace e.g. a GT with an LE and not end up capturing the NaN case
5161 // (where the comparison should always fail). EQ and NE tests are
5162 // always generated in ideal code so that unordered folds into the NE
5163 // case, matching the behaviour of AArch64 NE.
5164 //
5165 // This differs from x86 where the outputs of FP compares use a
5166 // special FP flags registers and where compares based on this
5167 // register are distinguished into ordered inequalities (cmpOpUCF) and
5168 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5169 // to explicitly handle the unordered case in branches. x86 also has
5170 // to include extra CMoveX rules to accept a cmpOpUCF input.
5171
5172 operand rFlagsReg()
5173 %{
5174 constraint(ALLOC_IN_RC(int_flags));
5175 match(RegFlags);
5176
5177 op_cost(0);
5178 format %{ "RFLAGS" %}
5179 interface(REG_INTER);
5180 %}
5181
5182 // Flags register, used as output of unsigned compare instructions
5183 operand rFlagsRegU()
5184 %{
5185 constraint(ALLOC_IN_RC(int_flags));
5186 match(RegFlags);
5187
5188 op_cost(0);
5189 format %{ "RFLAGSU" %}
5190 interface(REG_INTER);
5191 %}
5192
5193 // Special Registers
5194
5195 // Method Register
5196 operand inline_cache_RegP(iRegP reg)
5197 %{
5198 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5199 match(reg);
5200 match(iRegPNoSp);
5201 op_cost(0);
5202 format %{ %}
5203 interface(REG_INTER);
5204 %}
5205
5206 // Thread Register
5207 operand thread_RegP(iRegP reg)
5208 %{
5209 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5210 match(reg);
5211 op_cost(0);
5212 format %{ %}
5213 interface(REG_INTER);
5214 %}
5215
5216 //----------Memory Operands----------------------------------------------------
5217
5218 operand indirect(iRegP reg)
5219 %{
5220 constraint(ALLOC_IN_RC(ptr_reg));
5221 match(reg);
5222 op_cost(0);
5223 format %{ "[$reg]" %}
5224 interface(MEMORY_INTER) %{
5225 base($reg);
5226 index(0xffffffff);
5227 scale(0x0);
5228 disp(0x0);
5229 %}
5230 %}
5231
5232 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5233 %{
5234 constraint(ALLOC_IN_RC(ptr_reg));
5235 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5236 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5237 op_cost(0);
5238 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5239 interface(MEMORY_INTER) %{
5240 base($reg);
5241 index($ireg);
5242 scale($scale);
5243 disp(0x0);
5244 %}
5245 %}
5246
5247 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5248 %{
5249 constraint(ALLOC_IN_RC(ptr_reg));
5250 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5251 match(AddP reg (LShiftL lreg scale));
5252 op_cost(0);
5253 format %{ "$reg, $lreg lsl($scale)" %}
5254 interface(MEMORY_INTER) %{
5255 base($reg);
5256 index($lreg);
5257 scale($scale);
5258 disp(0x0);
5259 %}
5260 %}
5261
5262 operand indIndexI2L(iRegP reg, iRegI ireg)
5263 %{
5264 constraint(ALLOC_IN_RC(ptr_reg));
5265 match(AddP reg (ConvI2L ireg));
5266 op_cost(0);
5267 format %{ "$reg, $ireg, 0, I2L" %}
5268 interface(MEMORY_INTER) %{
5269 base($reg);
5270 index($ireg);
5271 scale(0x0);
5272 disp(0x0);
5273 %}
5274 %}
5275
5276 operand indIndex(iRegP reg, iRegL lreg)
5277 %{
5278 constraint(ALLOC_IN_RC(ptr_reg));
5279 match(AddP reg lreg);
5280 op_cost(0);
5281 format %{ "$reg, $lreg" %}
5282 interface(MEMORY_INTER) %{
5283 base($reg);
5284 index($lreg);
5285 scale(0x0);
5286 disp(0x0);
5287 %}
5288 %}
5289
5290 operand indOffI1(iRegP reg, immIOffset1 off)
5291 %{
5292 constraint(ALLOC_IN_RC(ptr_reg));
5293 match(AddP reg off);
5294 op_cost(0);
5295 format %{ "[$reg, $off]" %}
5296 interface(MEMORY_INTER) %{
5297 base($reg);
5298 index(0xffffffff);
5299 scale(0x0);
5300 disp($off);
5301 %}
5302 %}
5303
5304 operand indOffI2(iRegP reg, immIOffset2 off)
5305 %{
5306 constraint(ALLOC_IN_RC(ptr_reg));
5307 match(AddP reg off);
5308 op_cost(0);
5309 format %{ "[$reg, $off]" %}
5310 interface(MEMORY_INTER) %{
5311 base($reg);
5312 index(0xffffffff);
5313 scale(0x0);
5314 disp($off);
5315 %}
5316 %}
5317
5318 operand indOffI4(iRegP reg, immIOffset4 off)
5319 %{
5320 constraint(ALLOC_IN_RC(ptr_reg));
5321 match(AddP reg off);
5322 op_cost(0);
5323 format %{ "[$reg, $off]" %}
5324 interface(MEMORY_INTER) %{
5325 base($reg);
5326 index(0xffffffff);
5327 scale(0x0);
5328 disp($off);
5329 %}
5330 %}
5331
5332 operand indOffI8(iRegP reg, immIOffset8 off)
5333 %{
5334 constraint(ALLOC_IN_RC(ptr_reg));
5335 match(AddP reg off);
5336 op_cost(0);
5337 format %{ "[$reg, $off]" %}
5338 interface(MEMORY_INTER) %{
5339 base($reg);
5340 index(0xffffffff);
5341 scale(0x0);
5342 disp($off);
5343 %}
5344 %}
5345
5346 operand indOffI16(iRegP reg, immIOffset16 off)
5347 %{
5348 constraint(ALLOC_IN_RC(ptr_reg));
5349 match(AddP reg off);
5350 op_cost(0);
5351 format %{ "[$reg, $off]" %}
5352 interface(MEMORY_INTER) %{
5353 base($reg);
5354 index(0xffffffff);
5355 scale(0x0);
5356 disp($off);
5357 %}
5358 %}
5359
5360 operand indOffL1(iRegP reg, immLoffset1 off)
5361 %{
5362 constraint(ALLOC_IN_RC(ptr_reg));
5363 match(AddP reg off);
5364 op_cost(0);
5365 format %{ "[$reg, $off]" %}
5366 interface(MEMORY_INTER) %{
5367 base($reg);
5368 index(0xffffffff);
5369 scale(0x0);
5370 disp($off);
5371 %}
5372 %}
5373
5374 operand indOffL2(iRegP reg, immLoffset2 off)
5375 %{
5376 constraint(ALLOC_IN_RC(ptr_reg));
5377 match(AddP reg off);
5378 op_cost(0);
5379 format %{ "[$reg, $off]" %}
5380 interface(MEMORY_INTER) %{
5381 base($reg);
5382 index(0xffffffff);
5383 scale(0x0);
5384 disp($off);
5385 %}
5386 %}
5387
5388 operand indOffL4(iRegP reg, immLoffset4 off)
5389 %{
5390 constraint(ALLOC_IN_RC(ptr_reg));
5391 match(AddP reg off);
5392 op_cost(0);
5393 format %{ "[$reg, $off]" %}
5394 interface(MEMORY_INTER) %{
5395 base($reg);
5396 index(0xffffffff);
5397 scale(0x0);
5398 disp($off);
5399 %}
5400 %}
5401
5402 operand indOffL8(iRegP reg, immLoffset8 off)
5403 %{
5404 constraint(ALLOC_IN_RC(ptr_reg));
5405 match(AddP reg off);
5406 op_cost(0);
5407 format %{ "[$reg, $off]" %}
5408 interface(MEMORY_INTER) %{
5409 base($reg);
5410 index(0xffffffff);
5411 scale(0x0);
5412 disp($off);
5413 %}
5414 %}
5415
5416 operand indOffL16(iRegP reg, immLoffset16 off)
5417 %{
5418 constraint(ALLOC_IN_RC(ptr_reg));
5419 match(AddP reg off);
5420 op_cost(0);
5421 format %{ "[$reg, $off]" %}
5422 interface(MEMORY_INTER) %{
5423 base($reg);
5424 index(0xffffffff);
5425 scale(0x0);
5426 disp($off);
5427 %}
5428 %}
5429
5430 operand indirectX2P(iRegL reg)
5431 %{
5432 constraint(ALLOC_IN_RC(ptr_reg));
5433 match(CastX2P reg);
5434 op_cost(0);
5435 format %{ "[$reg]\t# long -> ptr" %}
5436 interface(MEMORY_INTER) %{
5437 base($reg);
5438 index(0xffffffff);
5439 scale(0x0);
5440 disp(0x0);
5441 %}
5442 %}
5443
5444 operand indOffX2P(iRegL reg, immLOffset off)
5445 %{
5446 constraint(ALLOC_IN_RC(ptr_reg));
5447 match(AddP (CastX2P reg) off);
5448 op_cost(0);
5449 format %{ "[$reg, $off]\t# long -> ptr" %}
5450 interface(MEMORY_INTER) %{
5451 base($reg);
5452 index(0xffffffff);
5453 scale(0x0);
5454 disp($off);
5455 %}
5456 %}
5457
5458 operand indirectN(iRegN reg)
5459 %{
5460 predicate(CompressedOops::shift() == 0);
5461 constraint(ALLOC_IN_RC(ptr_reg));
5462 match(DecodeN reg);
5463 op_cost(0);
5464 format %{ "[$reg]\t# narrow" %}
5465 interface(MEMORY_INTER) %{
5466 base($reg);
5467 index(0xffffffff);
5468 scale(0x0);
5469 disp(0x0);
5470 %}
5471 %}
5472
5473 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5474 %{
5475 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5476 constraint(ALLOC_IN_RC(ptr_reg));
5477 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5478 op_cost(0);
5479 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5480 interface(MEMORY_INTER) %{
5481 base($reg);
5482 index($ireg);
5483 scale($scale);
5484 disp(0x0);
5485 %}
5486 %}
5487
5488 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5489 %{
5490 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5491 constraint(ALLOC_IN_RC(ptr_reg));
5492 match(AddP (DecodeN reg) (LShiftL lreg scale));
5493 op_cost(0);
5494 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5495 interface(MEMORY_INTER) %{
5496 base($reg);
5497 index($lreg);
5498 scale($scale);
5499 disp(0x0);
5500 %}
5501 %}
5502
5503 operand indIndexI2LN(iRegN reg, iRegI ireg)
5504 %{
5505 predicate(CompressedOops::shift() == 0);
5506 constraint(ALLOC_IN_RC(ptr_reg));
5507 match(AddP (DecodeN reg) (ConvI2L ireg));
5508 op_cost(0);
5509 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5510 interface(MEMORY_INTER) %{
5511 base($reg);
5512 index($ireg);
5513 scale(0x0);
5514 disp(0x0);
5515 %}
5516 %}
5517
5518 operand indIndexN(iRegN reg, iRegL lreg)
5519 %{
5520 predicate(CompressedOops::shift() == 0);
5521 constraint(ALLOC_IN_RC(ptr_reg));
5522 match(AddP (DecodeN reg) lreg);
5523 op_cost(0);
5524 format %{ "$reg, $lreg\t# narrow" %}
5525 interface(MEMORY_INTER) %{
5526 base($reg);
5527 index($lreg);
5528 scale(0x0);
5529 disp(0x0);
5530 %}
5531 %}
5532
5533 operand indOffIN(iRegN reg, immIOffset off)
5534 %{
5535 predicate(CompressedOops::shift() == 0);
5536 constraint(ALLOC_IN_RC(ptr_reg));
5537 match(AddP (DecodeN reg) off);
5538 op_cost(0);
5539 format %{ "[$reg, $off]\t# narrow" %}
5540 interface(MEMORY_INTER) %{
5541 base($reg);
5542 index(0xffffffff);
5543 scale(0x0);
5544 disp($off);
5545 %}
5546 %}
5547
5548 operand indOffLN(iRegN reg, immLOffset off)
5549 %{
5550 predicate(CompressedOops::shift() == 0);
5551 constraint(ALLOC_IN_RC(ptr_reg));
5552 match(AddP (DecodeN reg) off);
5553 op_cost(0);
5554 format %{ "[$reg, $off]\t# narrow" %}
5555 interface(MEMORY_INTER) %{
5556 base($reg);
5557 index(0xffffffff);
5558 scale(0x0);
5559 disp($off);
5560 %}
5561 %}
5562
5563
5564 //----------Special Memory Operands--------------------------------------------
5565 // Stack Slot Operand - This operand is used for loading and storing temporary
5566 // values on the stack where a match requires a value to
5567 // flow through memory.
5568 operand stackSlotP(sRegP reg)
5569 %{
5570 constraint(ALLOC_IN_RC(stack_slots));
5571 op_cost(100);
5572 // No match rule because this operand is only generated in matching
5573 // match(RegP);
5574 format %{ "[$reg]" %}
5575 interface(MEMORY_INTER) %{
5576 base(0x1e); // RSP
5577 index(0x0); // No Index
5578 scale(0x0); // No Scale
5579 disp($reg); // Stack Offset
5580 %}
5581 %}
5582
5583 operand stackSlotI(sRegI reg)
5584 %{
5585 constraint(ALLOC_IN_RC(stack_slots));
5586 // No match rule because this operand is only generated in matching
5587 // match(RegI);
5588 format %{ "[$reg]" %}
5589 interface(MEMORY_INTER) %{
5590 base(0x1e); // RSP
5591 index(0x0); // No Index
5592 scale(0x0); // No Scale
5593 disp($reg); // Stack Offset
5594 %}
5595 %}
5596
5597 operand stackSlotF(sRegF reg)
5598 %{
5599 constraint(ALLOC_IN_RC(stack_slots));
5600 // No match rule because this operand is only generated in matching
5601 // match(RegF);
5602 format %{ "[$reg]" %}
5603 interface(MEMORY_INTER) %{
5604 base(0x1e); // RSP
5605 index(0x0); // No Index
5606 scale(0x0); // No Scale
5607 disp($reg); // Stack Offset
5608 %}
5609 %}
5610
5611 operand stackSlotD(sRegD reg)
5612 %{
5613 constraint(ALLOC_IN_RC(stack_slots));
5614 // No match rule because this operand is only generated in matching
5615 // match(RegD);
5616 format %{ "[$reg]" %}
5617 interface(MEMORY_INTER) %{
5618 base(0x1e); // RSP
5619 index(0x0); // No Index
5620 scale(0x0); // No Scale
5621 disp($reg); // Stack Offset
5622 %}
5623 %}
5624
5625 operand stackSlotL(sRegL reg)
5626 %{
5627 constraint(ALLOC_IN_RC(stack_slots));
5628 // No match rule because this operand is only generated in matching
5629 // match(RegL);
5630 format %{ "[$reg]" %}
5631 interface(MEMORY_INTER) %{
5632 base(0x1e); // RSP
5633 index(0x0); // No Index
5634 scale(0x0); // No Scale
5635 disp($reg); // Stack Offset
5636 %}
5637 %}
5638
5639 // Operands for expressing Control Flow
5640 // NOTE: Label is a predefined operand which should not be redefined in
5641 // the AD file. It is generically handled within the ADLC.
5642
5643 //----------Conditional Branch Operands----------------------------------------
5644 // Comparison Op - This is the operation of the comparison, and is limited to
5645 // the following set of codes:
5646 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5647 //
5648 // Other attributes of the comparison, such as unsignedness, are specified
5649 // by the comparison instruction that sets a condition code flags register.
5650 // That result is represented by a flags operand whose subtype is appropriate
5651 // to the unsignedness (etc.) of the comparison.
5652 //
5653 // Later, the instruction which matches both the Comparison Op (a Bool) and
5654 // the flags (produced by the Cmp) specifies the coding of the comparison op
5655 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5656
5657 // used for signed integral comparisons and fp comparisons
5658
5659 operand cmpOp()
5660 %{
5661 match(Bool);
5662
5663 format %{ "" %}
5664 interface(COND_INTER) %{
5665 equal(0x0, "eq");
5666 not_equal(0x1, "ne");
5667 less(0xb, "lt");
5668 greater_equal(0xa, "ge");
5669 less_equal(0xd, "le");
5670 greater(0xc, "gt");
5671 overflow(0x6, "vs");
5672 no_overflow(0x7, "vc");
5673 %}
5674 %}
5675
5676 // used for unsigned integral comparisons
5677
5678 operand cmpOpU()
5679 %{
5680 match(Bool);
5681
5682 format %{ "" %}
5683 interface(COND_INTER) %{
5684 equal(0x0, "eq");
5685 not_equal(0x1, "ne");
5686 less(0x3, "lo");
5687 greater_equal(0x2, "hs");
5688 less_equal(0x9, "ls");
5689 greater(0x8, "hi");
5690 overflow(0x6, "vs");
5691 no_overflow(0x7, "vc");
5692 %}
5693 %}
5694
5695 // used for certain integral comparisons which can be
5696 // converted to cbxx or tbxx instructions
5697
5698 operand cmpOpEqNe()
5699 %{
5700 match(Bool);
5701 op_cost(0);
5702 predicate(n->as_Bool()->_test._test == BoolTest::ne
5703 || n->as_Bool()->_test._test == BoolTest::eq);
5704
5705 format %{ "" %}
5706 interface(COND_INTER) %{
5707 equal(0x0, "eq");
5708 not_equal(0x1, "ne");
5709 less(0xb, "lt");
5710 greater_equal(0xa, "ge");
5711 less_equal(0xd, "le");
5712 greater(0xc, "gt");
5713 overflow(0x6, "vs");
5714 no_overflow(0x7, "vc");
5715 %}
5716 %}
5717
5718 // used for certain integral comparisons which can be
5719 // converted to cbxx or tbxx instructions
5720
5721 operand cmpOpLtGe()
5722 %{
5723 match(Bool);
5724 op_cost(0);
5725
5726 predicate(n->as_Bool()->_test._test == BoolTest::lt
5727 || n->as_Bool()->_test._test == BoolTest::ge);
5728
5729 format %{ "" %}
5730 interface(COND_INTER) %{
5731 equal(0x0, "eq");
5732 not_equal(0x1, "ne");
5733 less(0xb, "lt");
5734 greater_equal(0xa, "ge");
5735 less_equal(0xd, "le");
5736 greater(0xc, "gt");
5737 overflow(0x6, "vs");
5738 no_overflow(0x7, "vc");
5739 %}
5740 %}
5741
5742 // used for certain unsigned integral comparisons which can be
5743 // converted to cbxx or tbxx instructions
5744
5745 operand cmpOpUEqNeLeGt()
5746 %{
5747 match(Bool);
5748 op_cost(0);
5749
5750 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5751 n->as_Bool()->_test._test == BoolTest::ne ||
5752 n->as_Bool()->_test._test == BoolTest::le ||
5753 n->as_Bool()->_test._test == BoolTest::gt);
5754
5755 format %{ "" %}
5756 interface(COND_INTER) %{
5757 equal(0x0, "eq");
5758 not_equal(0x1, "ne");
5759 less(0x3, "lo");
5760 greater_equal(0x2, "hs");
5761 less_equal(0x9, "ls");
5762 greater(0x8, "hi");
5763 overflow(0x6, "vs");
5764 no_overflow(0x7, "vc");
5765 %}
5766 %}
5767
5768 // Special operand allowing long args to int ops to be truncated for free
5769
5770 operand iRegL2I(iRegL reg) %{
5771
5772 op_cost(0);
5773
5774 match(ConvL2I reg);
5775
5776 format %{ "l2i($reg)" %}
5777
5778 interface(REG_INTER)
5779 %}
5780
5781 operand iRegL2P(iRegL reg) %{
5782
5783 op_cost(0);
5784
5785 match(CastX2P reg);
5786
5787 format %{ "l2p($reg)" %}
5788
5789 interface(REG_INTER)
5790 %}
5791
5792 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5793 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5794 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5795 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5796
5797 //----------OPERAND CLASSES----------------------------------------------------
5798 // Operand Classes are groups of operands that are used as to simplify
5799 // instruction definitions by not requiring the AD writer to specify
5800 // separate instructions for every form of operand when the
5801 // instruction accepts multiple operand types with the same basic
5802 // encoding and format. The classic case of this is memory operands.
5803
5804 // memory is used to define read/write location for load/store
5805 // instruction defs. we can turn a memory op into an Address
5806
5807 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5808 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5809
5810 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5811 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5812
5813 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5814 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5815
5816 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5817 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5818
5819 // All of the memory operands. For the pipeline description.
5820 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5821 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5822 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5823
5824
5825 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5826 // operations. it allows the src to be either an iRegI or a (ConvL2I
5827 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5828 // can be elided because the 32-bit instruction will just employ the
5829 // lower 32 bits anyway.
5830 //
5831 // n.b. this does not elide all L2I conversions. if the truncated
5832 // value is consumed by more than one operation then the ConvL2I
5833 // cannot be bundled into the consuming nodes so an l2i gets planted
5834 // (actually a movw $dst $src) and the downstream instructions consume
5835 // the result of the l2i as an iRegI input. That's a shame since the
5836 // movw is actually redundant but its not too costly.
5837
5838 opclass iRegIorL2I(iRegI, iRegL2I);
5839 opclass iRegPorL2P(iRegP, iRegL2P);
5840
5841 //----------PIPELINE-----------------------------------------------------------
5842 // Rules which define the behavior of the target architectures pipeline.
5843
5844 // For specific pipelines, eg A53, define the stages of that pipeline
5845 //pipe_desc(ISS, EX1, EX2, WR);
5846 #define ISS S0
5847 #define EX1 S1
5848 #define EX2 S2
5849 #define WR S3
5850
5851 // Integer ALU reg operation
5852 pipeline %{
5853
5854 attributes %{
5855 // ARM instructions are of fixed length
5856 fixed_size_instructions; // Fixed size instructions TODO does
5857 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5858 // ARM instructions come in 32-bit word units
5859 instruction_unit_size = 4; // An instruction is 4 bytes long
5860 instruction_fetch_unit_size = 64; // The processor fetches one line
5861 instruction_fetch_units = 1; // of 64 bytes
5862
5863 // List of nop instructions
5864 nops( MachNop );
5865 %}
5866
5867 // We don't use an actual pipeline model so don't care about resources
5868 // or description. we do use pipeline classes to introduce fixed
5869 // latencies
5870
5871 //----------RESOURCES----------------------------------------------------------
5872 // Resources are the functional units available to the machine
5873
5874 resources( INS0, INS1, INS01 = INS0 | INS1,
5875 ALU0, ALU1, ALU = ALU0 | ALU1,
5876 MAC,
5877 DIV,
5878 BRANCH,
5879 LDST,
5880 NEON_FP);
5881
5882 //----------PIPELINE DESCRIPTION-----------------------------------------------
5883 // Pipeline Description specifies the stages in the machine's pipeline
5884
5885 // Define the pipeline as a generic 6 stage pipeline
5886 pipe_desc(S0, S1, S2, S3, S4, S5);
5887
5888 //----------PIPELINE CLASSES---------------------------------------------------
5889 // Pipeline Classes describe the stages in which input and output are
5890 // referenced by the hardware pipeline.
5891
5892 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5893 %{
5894 single_instruction;
5895 src1 : S1(read);
5896 src2 : S2(read);
5897 dst : S5(write);
5898 INS01 : ISS;
5899 NEON_FP : S5;
5900 %}
5901
5902 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5903 %{
5904 single_instruction;
5905 src1 : S1(read);
5906 src2 : S2(read);
5907 dst : S5(write);
5908 INS01 : ISS;
5909 NEON_FP : S5;
5910 %}
5911
5912 pipe_class fp_uop_s(vRegF dst, vRegF src)
5913 %{
5914 single_instruction;
5915 src : S1(read);
5916 dst : S5(write);
5917 INS01 : ISS;
5918 NEON_FP : S5;
5919 %}
5920
5921 pipe_class fp_uop_d(vRegD dst, vRegD src)
5922 %{
5923 single_instruction;
5924 src : S1(read);
5925 dst : S5(write);
5926 INS01 : ISS;
5927 NEON_FP : S5;
5928 %}
5929
5930 pipe_class fp_d2f(vRegF dst, vRegD src)
5931 %{
5932 single_instruction;
5933 src : S1(read);
5934 dst : S5(write);
5935 INS01 : ISS;
5936 NEON_FP : S5;
5937 %}
5938
5939 pipe_class fp_f2d(vRegD dst, vRegF src)
5940 %{
5941 single_instruction;
5942 src : S1(read);
5943 dst : S5(write);
5944 INS01 : ISS;
5945 NEON_FP : S5;
5946 %}
5947
5948 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5949 %{
5950 single_instruction;
5951 src : S1(read);
5952 dst : S5(write);
5953 INS01 : ISS;
5954 NEON_FP : S5;
5955 %}
5956
5957 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5958 %{
5959 single_instruction;
5960 src : S1(read);
5961 dst : S5(write);
5962 INS01 : ISS;
5963 NEON_FP : S5;
5964 %}
5965
5966 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5967 %{
5968 single_instruction;
5969 src : S1(read);
5970 dst : S5(write);
5971 INS01 : ISS;
5972 NEON_FP : S5;
5973 %}
5974
5975 pipe_class fp_l2f(vRegF dst, iRegL src)
5976 %{
5977 single_instruction;
5978 src : S1(read);
5979 dst : S5(write);
5980 INS01 : ISS;
5981 NEON_FP : S5;
5982 %}
5983
5984 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5985 %{
5986 single_instruction;
5987 src : S1(read);
5988 dst : S5(write);
5989 INS01 : ISS;
5990 NEON_FP : S5;
5991 %}
5992
5993 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5994 %{
5995 single_instruction;
5996 src : S1(read);
5997 dst : S5(write);
5998 INS01 : ISS;
5999 NEON_FP : S5;
6000 %}
6001
6002 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6003 %{
6004 single_instruction;
6005 src : S1(read);
6006 dst : S5(write);
6007 INS01 : ISS;
6008 NEON_FP : S5;
6009 %}
6010
6011 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6012 %{
6013 single_instruction;
6014 src : S1(read);
6015 dst : S5(write);
6016 INS01 : ISS;
6017 NEON_FP : S5;
6018 %}
6019
6020 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6021 %{
6022 single_instruction;
6023 src1 : S1(read);
6024 src2 : S2(read);
6025 dst : S5(write);
6026 INS0 : ISS;
6027 NEON_FP : S5;
6028 %}
6029
6030 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6031 %{
6032 single_instruction;
6033 src1 : S1(read);
6034 src2 : S2(read);
6035 dst : S5(write);
6036 INS0 : ISS;
6037 NEON_FP : S5;
6038 %}
6039
6040 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6041 %{
6042 single_instruction;
6043 cr : S1(read);
6044 src1 : S1(read);
6045 src2 : S1(read);
6046 dst : S3(write);
6047 INS01 : ISS;
6048 NEON_FP : S3;
6049 %}
6050
6051 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6052 %{
6053 single_instruction;
6054 cr : S1(read);
6055 src1 : S1(read);
6056 src2 : S1(read);
6057 dst : S3(write);
6058 INS01 : ISS;
6059 NEON_FP : S3;
6060 %}
6061
6062 pipe_class fp_imm_s(vRegF dst)
6063 %{
6064 single_instruction;
6065 dst : S3(write);
6066 INS01 : ISS;
6067 NEON_FP : S3;
6068 %}
6069
6070 pipe_class fp_imm_d(vRegD dst)
6071 %{
6072 single_instruction;
6073 dst : S3(write);
6074 INS01 : ISS;
6075 NEON_FP : S3;
6076 %}
6077
6078 pipe_class fp_load_constant_s(vRegF dst)
6079 %{
6080 single_instruction;
6081 dst : S4(write);
6082 INS01 : ISS;
6083 NEON_FP : S4;
6084 %}
6085
6086 pipe_class fp_load_constant_d(vRegD dst)
6087 %{
6088 single_instruction;
6089 dst : S4(write);
6090 INS01 : ISS;
6091 NEON_FP : S4;
6092 %}
6093
6094 //------- Integer ALU operations --------------------------
6095
6096 // Integer ALU reg-reg operation
6097 // Operands needed in EX1, result generated in EX2
6098 // Eg. ADD x0, x1, x2
6099 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6100 %{
6101 single_instruction;
6102 dst : EX2(write);
6103 src1 : EX1(read);
6104 src2 : EX1(read);
6105 INS01 : ISS; // Dual issue as instruction 0 or 1
6106 ALU : EX2;
6107 %}
6108
6109 // Integer ALU reg-reg operation with constant shift
6110 // Shifted register must be available in LATE_ISS instead of EX1
6111 // Eg. ADD x0, x1, x2, LSL #2
6112 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6113 %{
6114 single_instruction;
6115 dst : EX2(write);
6116 src1 : EX1(read);
6117 src2 : ISS(read);
6118 INS01 : ISS;
6119 ALU : EX2;
6120 %}
6121
6122 // Integer ALU reg operation with constant shift
6123 // Eg. LSL x0, x1, #shift
6124 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6125 %{
6126 single_instruction;
6127 dst : EX2(write);
6128 src1 : ISS(read);
6129 INS01 : ISS;
6130 ALU : EX2;
6131 %}
6132
6133 // Integer ALU reg-reg operation with variable shift
6134 // Both operands must be available in LATE_ISS instead of EX1
6135 // Result is available in EX1 instead of EX2
6136 // Eg. LSLV x0, x1, x2
6137 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6138 %{
6139 single_instruction;
6140 dst : EX1(write);
6141 src1 : ISS(read);
6142 src2 : ISS(read);
6143 INS01 : ISS;
6144 ALU : EX1;
6145 %}
6146
6147 // Integer ALU reg-reg operation with extract
6148 // As for _vshift above, but result generated in EX2
6149 // Eg. EXTR x0, x1, x2, #N
6150 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6151 %{
6152 single_instruction;
6153 dst : EX2(write);
6154 src1 : ISS(read);
6155 src2 : ISS(read);
6156 INS1 : ISS; // Can only dual issue as Instruction 1
6157 ALU : EX1;
6158 %}
6159
6160 // Integer ALU reg operation
6161 // Eg. NEG x0, x1
6162 pipe_class ialu_reg(iRegI dst, iRegI src)
6163 %{
6164 single_instruction;
6165 dst : EX2(write);
6166 src : EX1(read);
6167 INS01 : ISS;
6168 ALU : EX2;
6169 %}
6170
6171 // Integer ALU reg mmediate operation
6172 // Eg. ADD x0, x1, #N
6173 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6174 %{
6175 single_instruction;
6176 dst : EX2(write);
6177 src1 : EX1(read);
6178 INS01 : ISS;
6179 ALU : EX2;
6180 %}
6181
6182 // Integer ALU immediate operation (no source operands)
6183 // Eg. MOV x0, #N
6184 pipe_class ialu_imm(iRegI dst)
6185 %{
6186 single_instruction;
6187 dst : EX1(write);
6188 INS01 : ISS;
6189 ALU : EX1;
6190 %}
6191
6192 //------- Compare operation -------------------------------
6193
6194 // Compare reg-reg
6195 // Eg. CMP x0, x1
6196 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6197 %{
6198 single_instruction;
6199 // fixed_latency(16);
6200 cr : EX2(write);
6201 op1 : EX1(read);
6202 op2 : EX1(read);
6203 INS01 : ISS;
6204 ALU : EX2;
6205 %}
6206
6207 // Compare reg-reg
6208 // Eg. CMP x0, #N
6209 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6210 %{
6211 single_instruction;
6212 // fixed_latency(16);
6213 cr : EX2(write);
6214 op1 : EX1(read);
6215 INS01 : ISS;
6216 ALU : EX2;
6217 %}
6218
6219 //------- Conditional instructions ------------------------
6220
6221 // Conditional no operands
6222 // Eg. CSINC x0, zr, zr, <cond>
6223 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6224 %{
6225 single_instruction;
6226 cr : EX1(read);
6227 dst : EX2(write);
6228 INS01 : ISS;
6229 ALU : EX2;
6230 %}
6231
6232 // Conditional 2 operand
6233 // EG. CSEL X0, X1, X2, <cond>
6234 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6235 %{
6236 single_instruction;
6237 cr : EX1(read);
6238 src1 : EX1(read);
6239 src2 : EX1(read);
6240 dst : EX2(write);
6241 INS01 : ISS;
6242 ALU : EX2;
6243 %}
6244
6245 // Conditional 2 operand
6246 // EG. CSEL X0, X1, X2, <cond>
6247 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6248 %{
6249 single_instruction;
6250 cr : EX1(read);
6251 src : EX1(read);
6252 dst : EX2(write);
6253 INS01 : ISS;
6254 ALU : EX2;
6255 %}
6256
6257 //------- Multiply pipeline operations --------------------
6258
6259 // Multiply reg-reg
6260 // Eg. MUL w0, w1, w2
6261 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6262 %{
6263 single_instruction;
6264 dst : WR(write);
6265 src1 : ISS(read);
6266 src2 : ISS(read);
6267 INS01 : ISS;
6268 MAC : WR;
6269 %}
6270
6271 // Multiply accumulate
6272 // Eg. MADD w0, w1, w2, w3
6273 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6274 %{
6275 single_instruction;
6276 dst : WR(write);
6277 src1 : ISS(read);
6278 src2 : ISS(read);
6279 src3 : ISS(read);
6280 INS01 : ISS;
6281 MAC : WR;
6282 %}
6283
6284 // Eg. MUL w0, w1, w2
6285 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6286 %{
6287 single_instruction;
6288 fixed_latency(3); // Maximum latency for 64 bit mul
6289 dst : WR(write);
6290 src1 : ISS(read);
6291 src2 : ISS(read);
6292 INS01 : ISS;
6293 MAC : WR;
6294 %}
6295
6296 // Multiply accumulate
6297 // Eg. MADD w0, w1, w2, w3
6298 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6299 %{
6300 single_instruction;
6301 fixed_latency(3); // Maximum latency for 64 bit mul
6302 dst : WR(write);
6303 src1 : ISS(read);
6304 src2 : ISS(read);
6305 src3 : ISS(read);
6306 INS01 : ISS;
6307 MAC : WR;
6308 %}
6309
6310 //------- Divide pipeline operations --------------------
6311
6312 // Eg. SDIV w0, w1, w2
6313 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6314 %{
6315 single_instruction;
6316 fixed_latency(8); // Maximum latency for 32 bit divide
6317 dst : WR(write);
6318 src1 : ISS(read);
6319 src2 : ISS(read);
6320 INS0 : ISS; // Can only dual issue as instruction 0
6321 DIV : WR;
6322 %}
6323
6324 // Eg. SDIV x0, x1, x2
6325 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6326 %{
6327 single_instruction;
6328 fixed_latency(16); // Maximum latency for 64 bit divide
6329 dst : WR(write);
6330 src1 : ISS(read);
6331 src2 : ISS(read);
6332 INS0 : ISS; // Can only dual issue as instruction 0
6333 DIV : WR;
6334 %}
6335
6336 //------- Load pipeline operations ------------------------
6337
6338 // Load - prefetch
6339 // Eg. PFRM <mem>
6340 pipe_class iload_prefetch(memory mem)
6341 %{
6342 single_instruction;
6343 mem : ISS(read);
6344 INS01 : ISS;
6345 LDST : WR;
6346 %}
6347
6348 // Load - reg, mem
6349 // Eg. LDR x0, <mem>
6350 pipe_class iload_reg_mem(iRegI dst, memory mem)
6351 %{
6352 single_instruction;
6353 dst : WR(write);
6354 mem : ISS(read);
6355 INS01 : ISS;
6356 LDST : WR;
6357 %}
6358
6359 // Load - reg, reg
6360 // Eg. LDR x0, [sp, x1]
6361 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6362 %{
6363 single_instruction;
6364 dst : WR(write);
6365 src : ISS(read);
6366 INS01 : ISS;
6367 LDST : WR;
6368 %}
6369
6370 //------- Store pipeline operations -----------------------
6371
6372 // Store - zr, mem
6373 // Eg. STR zr, <mem>
6374 pipe_class istore_mem(memory mem)
6375 %{
6376 single_instruction;
6377 mem : ISS(read);
6378 INS01 : ISS;
6379 LDST : WR;
6380 %}
6381
6382 // Store - reg, mem
6383 // Eg. STR x0, <mem>
6384 pipe_class istore_reg_mem(iRegI src, memory mem)
6385 %{
6386 single_instruction;
6387 mem : ISS(read);
6388 src : EX2(read);
6389 INS01 : ISS;
6390 LDST : WR;
6391 %}
6392
6393 // Store - reg, reg
6394 // Eg. STR x0, [sp, x1]
6395 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6396 %{
6397 single_instruction;
6398 dst : ISS(read);
6399 src : EX2(read);
6400 INS01 : ISS;
6401 LDST : WR;
6402 %}
6403
6404 //------- Store pipeline operations -----------------------
6405
6406 // Branch
6407 pipe_class pipe_branch()
6408 %{
6409 single_instruction;
6410 INS01 : ISS;
6411 BRANCH : EX1;
6412 %}
6413
6414 // Conditional branch
6415 pipe_class pipe_branch_cond(rFlagsReg cr)
6416 %{
6417 single_instruction;
6418 cr : EX1(read);
6419 INS01 : ISS;
6420 BRANCH : EX1;
6421 %}
6422
6423 // Compare & Branch
6424 // EG. CBZ/CBNZ
6425 pipe_class pipe_cmp_branch(iRegI op1)
6426 %{
6427 single_instruction;
6428 op1 : EX1(read);
6429 INS01 : ISS;
6430 BRANCH : EX1;
6431 %}
6432
6433 //------- Synchronisation operations ----------------------
6434
6435 // Any operation requiring serialization.
6436 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6437 pipe_class pipe_serial()
6438 %{
6439 single_instruction;
6440 force_serialization;
6441 fixed_latency(16);
6442 INS01 : ISS(2); // Cannot dual issue with any other instruction
6443 LDST : WR;
6444 %}
6445
6446 // Generic big/slow expanded idiom - also serialized
6447 pipe_class pipe_slow()
6448 %{
6449 instruction_count(10);
6450 multiple_bundles;
6451 force_serialization;
6452 fixed_latency(16);
6453 INS01 : ISS(2); // Cannot dual issue with any other instruction
6454 LDST : WR;
6455 %}
6456
6457 // Empty pipeline class
6458 pipe_class pipe_class_empty()
6459 %{
6460 single_instruction;
6461 fixed_latency(0);
6462 %}
6463
6464 // Default pipeline class.
6465 pipe_class pipe_class_default()
6466 %{
6467 single_instruction;
6468 fixed_latency(2);
6469 %}
6470
6471 // Pipeline class for compares.
6472 pipe_class pipe_class_compare()
6473 %{
6474 single_instruction;
6475 fixed_latency(16);
6476 %}
6477
6478 // Pipeline class for memory operations.
6479 pipe_class pipe_class_memory()
6480 %{
6481 single_instruction;
6482 fixed_latency(16);
6483 %}
6484
6485 // Pipeline class for call.
6486 pipe_class pipe_class_call()
6487 %{
6488 single_instruction;
6489 fixed_latency(100);
6490 %}
6491
6492 // Define the class for the Nop node.
6493 define %{
6494 MachNop = pipe_class_empty;
6495 %}
6496
6497 %}
6498 //----------INSTRUCTIONS-------------------------------------------------------
6499 //
6500 // match -- States which machine-independent subtree may be replaced
6501 // by this instruction.
6502 // ins_cost -- The estimated cost of this instruction is used by instruction
6503 // selection to identify a minimum cost tree of machine
6504 // instructions that matches a tree of machine-independent
6505 // instructions.
6506 // format -- A string providing the disassembly for this instruction.
6507 // The value of an instruction's operand may be inserted
6508 // by referring to it with a '$' prefix.
6509 // opcode -- Three instruction opcodes may be provided. These are referred
6510 // to within an encode class as $primary, $secondary, and $tertiary
6511 // rrspectively. The primary opcode is commonly used to
6512 // indicate the type of machine instruction, while secondary
6513 // and tertiary are often used for prefix options or addressing
6514 // modes.
6515 // ins_encode -- A list of encode classes with parameters. The encode class
6516 // name must have been defined in an 'enc_class' specification
6517 // in the encode section of the architecture description.
6518
6519 // ============================================================================
6520 // Memory (Load/Store) Instructions
6521
6522 // Load Instructions
6523
6524 // Load Byte (8 bit signed)
6525 instruct loadB(iRegINoSp dst, memory1 mem)
6526 %{
6527 match(Set dst (LoadB mem));
6528 predicate(!needs_acquiring_load(n));
6529
6530 ins_cost(4 * INSN_COST);
6531 format %{ "ldrsbw $dst, $mem\t# byte" %}
6532
6533 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6534
6535 ins_pipe(iload_reg_mem);
6536 %}
6537
6538 // Load Byte (8 bit signed) into long
6539 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6540 %{
6541 match(Set dst (ConvI2L (LoadB mem)));
6542 predicate(!needs_acquiring_load(n->in(1)));
6543
6544 ins_cost(4 * INSN_COST);
6545 format %{ "ldrsb $dst, $mem\t# byte" %}
6546
6547 ins_encode(aarch64_enc_ldrsb(dst, mem));
6548
6549 ins_pipe(iload_reg_mem);
6550 %}
6551
6552 // Load Byte (8 bit unsigned)
6553 instruct loadUB(iRegINoSp dst, memory1 mem)
6554 %{
6555 match(Set dst (LoadUB mem));
6556 predicate(!needs_acquiring_load(n));
6557
6558 ins_cost(4 * INSN_COST);
6559 format %{ "ldrbw $dst, $mem\t# byte" %}
6560
6561 ins_encode(aarch64_enc_ldrb(dst, mem));
6562
6563 ins_pipe(iload_reg_mem);
6564 %}
6565
6566 // Load Byte (8 bit unsigned) into long
6567 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6568 %{
6569 match(Set dst (ConvI2L (LoadUB mem)));
6570 predicate(!needs_acquiring_load(n->in(1)));
6571
6572 ins_cost(4 * INSN_COST);
6573 format %{ "ldrb $dst, $mem\t# byte" %}
6574
6575 ins_encode(aarch64_enc_ldrb(dst, mem));
6576
6577 ins_pipe(iload_reg_mem);
6578 %}
6579
6580 // Load Short (16 bit signed)
6581 instruct loadS(iRegINoSp dst, memory2 mem)
6582 %{
6583 match(Set dst (LoadS mem));
6584 predicate(!needs_acquiring_load(n));
6585
6586 ins_cost(4 * INSN_COST);
6587 format %{ "ldrshw $dst, $mem\t# short" %}
6588
6589 ins_encode(aarch64_enc_ldrshw(dst, mem));
6590
6591 ins_pipe(iload_reg_mem);
6592 %}
6593
6594 // Load Short (16 bit signed) into long
6595 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6596 %{
6597 match(Set dst (ConvI2L (LoadS mem)));
6598 predicate(!needs_acquiring_load(n->in(1)));
6599
6600 ins_cost(4 * INSN_COST);
6601 format %{ "ldrsh $dst, $mem\t# short" %}
6602
6603 ins_encode(aarch64_enc_ldrsh(dst, mem));
6604
6605 ins_pipe(iload_reg_mem);
6606 %}
6607
6608 // Load Char (16 bit unsigned)
6609 instruct loadUS(iRegINoSp dst, memory2 mem)
6610 %{
6611 match(Set dst (LoadUS mem));
6612 predicate(!needs_acquiring_load(n));
6613
6614 ins_cost(4 * INSN_COST);
6615 format %{ "ldrh $dst, $mem\t# short" %}
6616
6617 ins_encode(aarch64_enc_ldrh(dst, mem));
6618
6619 ins_pipe(iload_reg_mem);
6620 %}
6621
6622 // Load Short/Char (16 bit unsigned) into long
6623 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6624 %{
6625 match(Set dst (ConvI2L (LoadUS mem)));
6626 predicate(!needs_acquiring_load(n->in(1)));
6627
6628 ins_cost(4 * INSN_COST);
6629 format %{ "ldrh $dst, $mem\t# short" %}
6630
6631 ins_encode(aarch64_enc_ldrh(dst, mem));
6632
6633 ins_pipe(iload_reg_mem);
6634 %}
6635
6636 // Load Integer (32 bit signed)
6637 instruct loadI(iRegINoSp dst, memory4 mem)
6638 %{
6639 match(Set dst (LoadI mem));
6640 predicate(!needs_acquiring_load(n));
6641
6642 ins_cost(4 * INSN_COST);
6643 format %{ "ldrw $dst, $mem\t# int" %}
6644
6645 ins_encode(aarch64_enc_ldrw(dst, mem));
6646
6647 ins_pipe(iload_reg_mem);
6648 %}
6649
6650 // Load Integer (32 bit signed) into long
6651 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6652 %{
6653 match(Set dst (ConvI2L (LoadI mem)));
6654 predicate(!needs_acquiring_load(n->in(1)));
6655
6656 ins_cost(4 * INSN_COST);
6657 format %{ "ldrsw $dst, $mem\t# int" %}
6658
6659 ins_encode(aarch64_enc_ldrsw(dst, mem));
6660
6661 ins_pipe(iload_reg_mem);
6662 %}
6663
6664 // Load Integer (32 bit unsigned) into long
6665 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6666 %{
6667 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6668 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6669
6670 ins_cost(4 * INSN_COST);
6671 format %{ "ldrw $dst, $mem\t# int" %}
6672
6673 ins_encode(aarch64_enc_ldrw(dst, mem));
6674
6675 ins_pipe(iload_reg_mem);
6676 %}
6677
6678 // Load Long (64 bit signed)
6679 instruct loadL(iRegLNoSp dst, memory8 mem)
6680 %{
6681 match(Set dst (LoadL mem));
6682 predicate(!needs_acquiring_load(n));
6683
6684 ins_cost(4 * INSN_COST);
6685 format %{ "ldr $dst, $mem\t# int" %}
6686
6687 ins_encode(aarch64_enc_ldr(dst, mem));
6688
6689 ins_pipe(iload_reg_mem);
6690 %}
6691
6692 // Load Range
6693 instruct loadRange(iRegINoSp dst, memory4 mem)
6694 %{
6695 match(Set dst (LoadRange mem));
6696
6697 ins_cost(4 * INSN_COST);
6698 format %{ "ldrw $dst, $mem\t# range" %}
6699
6700 ins_encode(aarch64_enc_ldrw(dst, mem));
6701
6702 ins_pipe(iload_reg_mem);
6703 %}
6704
6705 // Load Pointer
6706 instruct loadP(iRegPNoSp dst, memory8 mem)
6707 %{
6708 match(Set dst (LoadP mem));
6709 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6710
6711 ins_cost(4 * INSN_COST);
6712 format %{ "ldr $dst, $mem\t# ptr" %}
6713
6714 ins_encode(aarch64_enc_ldr(dst, mem));
6715
6716 ins_pipe(iload_reg_mem);
6717 %}
6718
6719 // Load Compressed Pointer
6720 instruct loadN(iRegNNoSp dst, memory4 mem)
6721 %{
6722 match(Set dst (LoadN mem));
6723 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6724
6725 ins_cost(4 * INSN_COST);
6726 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6727
6728 ins_encode(aarch64_enc_ldrw(dst, mem));
6729
6730 ins_pipe(iload_reg_mem);
6731 %}
6732
6733 // Load Klass Pointer
6734 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6735 %{
6736 match(Set dst (LoadKlass mem));
6737 predicate(!needs_acquiring_load(n));
6738
6739 ins_cost(4 * INSN_COST);
6740 format %{ "ldr $dst, $mem\t# class" %}
6741
6742 ins_encode(aarch64_enc_ldr(dst, mem));
6743
6744 ins_pipe(iload_reg_mem);
6745 %}
6746
6747 // Load Narrow Klass Pointer
6748 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6749 %{
6750 match(Set dst (LoadNKlass mem));
6751 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6752
6753 ins_cost(4 * INSN_COST);
6754 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6755
6756 ins_encode(aarch64_enc_ldrw(dst, mem));
6757
6758 ins_pipe(iload_reg_mem);
6759 %}
6760
6761 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6762 %{
6763 match(Set dst (LoadNKlass mem));
6764 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6765
6766 ins_cost(4 * INSN_COST);
6767 format %{
6768 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6769 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6770 %}
6771 ins_encode %{
6772 // inlined aarch64_enc_ldrw
6773 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6774 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6775 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6776 %}
6777 ins_pipe(iload_reg_mem);
6778 %}
6779
6780 // Load Float
6781 instruct loadF(vRegF dst, memory4 mem)
6782 %{
6783 match(Set dst (LoadF mem));
6784 predicate(!needs_acquiring_load(n));
6785
6786 ins_cost(4 * INSN_COST);
6787 format %{ "ldrs $dst, $mem\t# float" %}
6788
6789 ins_encode( aarch64_enc_ldrs(dst, mem) );
6790
6791 ins_pipe(pipe_class_memory);
6792 %}
6793
6794 // Load Double
6795 instruct loadD(vRegD dst, memory8 mem)
6796 %{
6797 match(Set dst (LoadD mem));
6798 predicate(!needs_acquiring_load(n));
6799
6800 ins_cost(4 * INSN_COST);
6801 format %{ "ldrd $dst, $mem\t# double" %}
6802
6803 ins_encode( aarch64_enc_ldrd(dst, mem) );
6804
6805 ins_pipe(pipe_class_memory);
6806 %}
6807
6808
6809 // Load Int Constant
6810 instruct loadConI(iRegINoSp dst, immI src)
6811 %{
6812 match(Set dst src);
6813
6814 ins_cost(INSN_COST);
6815 format %{ "mov $dst, $src\t# int" %}
6816
6817 ins_encode( aarch64_enc_movw_imm(dst, src) );
6818
6819 ins_pipe(ialu_imm);
6820 %}
6821
6822 // Load Long Constant
6823 instruct loadConL(iRegLNoSp dst, immL src)
6824 %{
6825 match(Set dst src);
6826
6827 ins_cost(INSN_COST);
6828 format %{ "mov $dst, $src\t# long" %}
6829
6830 ins_encode( aarch64_enc_mov_imm(dst, src) );
6831
6832 ins_pipe(ialu_imm);
6833 %}
6834
6835 // Load Pointer Constant
6836
6837 instruct loadConP(iRegPNoSp dst, immP con)
6838 %{
6839 match(Set dst con);
6840
6841 ins_cost(INSN_COST * 4);
6842 format %{
6843 "mov $dst, $con\t# ptr"
6844 %}
6845
6846 ins_encode(aarch64_enc_mov_p(dst, con));
6847
6848 ins_pipe(ialu_imm);
6849 %}
6850
6851 // Load Null Pointer Constant
6852
6853 instruct loadConP0(iRegPNoSp dst, immP0 con)
6854 %{
6855 match(Set dst con);
6856
6857 ins_cost(INSN_COST);
6858 format %{ "mov $dst, $con\t# nullptr ptr" %}
6859
6860 ins_encode(aarch64_enc_mov_p0(dst, con));
6861
6862 ins_pipe(ialu_imm);
6863 %}
6864
6865 // Load Pointer Constant One
6866
6867 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6868 %{
6869 match(Set dst con);
6870
6871 ins_cost(INSN_COST);
6872 format %{ "mov $dst, $con\t# nullptr ptr" %}
6873
6874 ins_encode(aarch64_enc_mov_p1(dst, con));
6875
6876 ins_pipe(ialu_imm);
6877 %}
6878
6879 // Load Byte Map Base Constant
6880
6881 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6882 %{
6883 match(Set dst con);
6884
6885 ins_cost(INSN_COST);
6886 format %{ "adr $dst, $con\t# Byte Map Base" %}
6887
6888 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6889
6890 ins_pipe(ialu_imm);
6891 %}
6892
6893 // Load Narrow Pointer Constant
6894
6895 instruct loadConN(iRegNNoSp dst, immN con)
6896 %{
6897 match(Set dst con);
6898
6899 ins_cost(INSN_COST * 4);
6900 format %{ "mov $dst, $con\t# compressed ptr" %}
6901
6902 ins_encode(aarch64_enc_mov_n(dst, con));
6903
6904 ins_pipe(ialu_imm);
6905 %}
6906
6907 // Load Narrow Null Pointer Constant
6908
6909 instruct loadConN0(iRegNNoSp dst, immN0 con)
6910 %{
6911 match(Set dst con);
6912
6913 ins_cost(INSN_COST);
6914 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6915
6916 ins_encode(aarch64_enc_mov_n0(dst, con));
6917
6918 ins_pipe(ialu_imm);
6919 %}
6920
6921 // Load Narrow Klass Constant
6922
6923 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6924 %{
6925 match(Set dst con);
6926
6927 ins_cost(INSN_COST);
6928 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6929
6930 ins_encode(aarch64_enc_mov_nk(dst, con));
6931
6932 ins_pipe(ialu_imm);
6933 %}
6934
6935 // Load Packed Float Constant
6936
6937 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6938 match(Set dst con);
6939 ins_cost(INSN_COST * 4);
6940 format %{ "fmovs $dst, $con"%}
6941 ins_encode %{
6942 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6943 %}
6944
6945 ins_pipe(fp_imm_s);
6946 %}
6947
6948 // Load Float Constant
6949
6950 instruct loadConF(vRegF dst, immF con) %{
6951 match(Set dst con);
6952
6953 ins_cost(INSN_COST * 4);
6954
6955 format %{
6956 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6957 %}
6958
6959 ins_encode %{
6960 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6961 %}
6962
6963 ins_pipe(fp_load_constant_s);
6964 %}
6965
6966 // Load Packed Double Constant
6967
6968 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6969 match(Set dst con);
6970 ins_cost(INSN_COST);
6971 format %{ "fmovd $dst, $con"%}
6972 ins_encode %{
6973 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6974 %}
6975
6976 ins_pipe(fp_imm_d);
6977 %}
6978
6979 // Load Double Constant
6980
6981 instruct loadConD(vRegD dst, immD con) %{
6982 match(Set dst con);
6983
6984 ins_cost(INSN_COST * 5);
6985 format %{
6986 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6987 %}
6988
6989 ins_encode %{
6990 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6991 %}
6992
6993 ins_pipe(fp_load_constant_d);
6994 %}
6995
6996 // Load Half Float Constant
6997 // The "ldr" instruction loads a 32-bit word from the constant pool into a
6998 // 32-bit register but only the bottom half will be populated and the top
6999 // 16 bits are zero.
7000 instruct loadConH(vRegF dst, immH con) %{
7001 match(Set dst con);
7002 format %{
7003 "ldrs $dst, [$constantaddress]\t# load from constant table: half float=$con\n\t"
7004 %}
7005 ins_encode %{
7006 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7007 %}
7008 ins_pipe(fp_load_constant_s);
7009 %}
7010
7011 // Store Instructions
7012
7013 // Store Byte
7014 instruct storeB(iRegIorL2I src, memory1 mem)
7015 %{
7016 match(Set mem (StoreB mem src));
7017 predicate(!needs_releasing_store(n));
7018
7019 ins_cost(INSN_COST);
7020 format %{ "strb $src, $mem\t# byte" %}
7021
7022 ins_encode(aarch64_enc_strb(src, mem));
7023
7024 ins_pipe(istore_reg_mem);
7025 %}
7026
7027
7028 instruct storeimmB0(immI0 zero, memory1 mem)
7029 %{
7030 match(Set mem (StoreB mem zero));
7031 predicate(!needs_releasing_store(n));
7032
7033 ins_cost(INSN_COST);
7034 format %{ "strb rscractch2, $mem\t# byte" %}
7035
7036 ins_encode(aarch64_enc_strb0(mem));
7037
7038 ins_pipe(istore_mem);
7039 %}
7040
7041 // Store Char/Short
7042 instruct storeC(iRegIorL2I src, memory2 mem)
7043 %{
7044 match(Set mem (StoreC mem src));
7045 predicate(!needs_releasing_store(n));
7046
7047 ins_cost(INSN_COST);
7048 format %{ "strh $src, $mem\t# short" %}
7049
7050 ins_encode(aarch64_enc_strh(src, mem));
7051
7052 ins_pipe(istore_reg_mem);
7053 %}
7054
7055 instruct storeimmC0(immI0 zero, memory2 mem)
7056 %{
7057 match(Set mem (StoreC mem zero));
7058 predicate(!needs_releasing_store(n));
7059
7060 ins_cost(INSN_COST);
7061 format %{ "strh zr, $mem\t# short" %}
7062
7063 ins_encode(aarch64_enc_strh0(mem));
7064
7065 ins_pipe(istore_mem);
7066 %}
7067
7068 // Store Integer
7069
7070 instruct storeI(iRegIorL2I src, memory4 mem)
7071 %{
7072 match(Set mem(StoreI mem src));
7073 predicate(!needs_releasing_store(n));
7074
7075 ins_cost(INSN_COST);
7076 format %{ "strw $src, $mem\t# int" %}
7077
7078 ins_encode(aarch64_enc_strw(src, mem));
7079
7080 ins_pipe(istore_reg_mem);
7081 %}
7082
7083 instruct storeimmI0(immI0 zero, memory4 mem)
7084 %{
7085 match(Set mem(StoreI mem zero));
7086 predicate(!needs_releasing_store(n));
7087
7088 ins_cost(INSN_COST);
7089 format %{ "strw zr, $mem\t# int" %}
7090
7091 ins_encode(aarch64_enc_strw0(mem));
7092
7093 ins_pipe(istore_mem);
7094 %}
7095
7096 // Store Long (64 bit signed)
7097 instruct storeL(iRegL src, memory8 mem)
7098 %{
7099 match(Set mem (StoreL mem src));
7100 predicate(!needs_releasing_store(n));
7101
7102 ins_cost(INSN_COST);
7103 format %{ "str $src, $mem\t# int" %}
7104
7105 ins_encode(aarch64_enc_str(src, mem));
7106
7107 ins_pipe(istore_reg_mem);
7108 %}
7109
7110 // Store Long (64 bit signed)
7111 instruct storeimmL0(immL0 zero, memory8 mem)
7112 %{
7113 match(Set mem (StoreL mem zero));
7114 predicate(!needs_releasing_store(n));
7115
7116 ins_cost(INSN_COST);
7117 format %{ "str zr, $mem\t# int" %}
7118
7119 ins_encode(aarch64_enc_str0(mem));
7120
7121 ins_pipe(istore_mem);
7122 %}
7123
7124 // Store Pointer
7125 instruct storeP(iRegP src, memory8 mem)
7126 %{
7127 match(Set mem (StoreP mem src));
7128 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7129
7130 ins_cost(INSN_COST);
7131 format %{ "str $src, $mem\t# ptr" %}
7132
7133 ins_encode(aarch64_enc_str(src, mem));
7134
7135 ins_pipe(istore_reg_mem);
7136 %}
7137
7138 // Store Pointer
7139 instruct storeimmP0(immP0 zero, memory8 mem)
7140 %{
7141 match(Set mem (StoreP mem zero));
7142 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7143
7144 ins_cost(INSN_COST);
7145 format %{ "str zr, $mem\t# ptr" %}
7146
7147 ins_encode(aarch64_enc_str0(mem));
7148
7149 ins_pipe(istore_mem);
7150 %}
7151
7152 // Store Compressed Pointer
7153 instruct storeN(iRegN src, memory4 mem)
7154 %{
7155 match(Set mem (StoreN mem src));
7156 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7157
7158 ins_cost(INSN_COST);
7159 format %{ "strw $src, $mem\t# compressed ptr" %}
7160
7161 ins_encode(aarch64_enc_strw(src, mem));
7162
7163 ins_pipe(istore_reg_mem);
7164 %}
7165
7166 instruct storeImmN0(immN0 zero, memory4 mem)
7167 %{
7168 match(Set mem (StoreN mem zero));
7169 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7170
7171 ins_cost(INSN_COST);
7172 format %{ "strw zr, $mem\t# compressed ptr" %}
7173
7174 ins_encode(aarch64_enc_strw0(mem));
7175
7176 ins_pipe(istore_mem);
7177 %}
7178
7179 // Store Float
7180 instruct storeF(vRegF src, memory4 mem)
7181 %{
7182 match(Set mem (StoreF mem src));
7183 predicate(!needs_releasing_store(n));
7184
7185 ins_cost(INSN_COST);
7186 format %{ "strs $src, $mem\t# float" %}
7187
7188 ins_encode( aarch64_enc_strs(src, mem) );
7189
7190 ins_pipe(pipe_class_memory);
7191 %}
7192
7193 // TODO
7194 // implement storeImmF0 and storeFImmPacked
7195
7196 // Store Double
7197 instruct storeD(vRegD src, memory8 mem)
7198 %{
7199 match(Set mem (StoreD mem src));
7200 predicate(!needs_releasing_store(n));
7201
7202 ins_cost(INSN_COST);
7203 format %{ "strd $src, $mem\t# double" %}
7204
7205 ins_encode( aarch64_enc_strd(src, mem) );
7206
7207 ins_pipe(pipe_class_memory);
7208 %}
7209
7210 // Store Compressed Klass Pointer
7211 instruct storeNKlass(iRegN src, memory4 mem)
7212 %{
7213 predicate(!needs_releasing_store(n));
7214 match(Set mem (StoreNKlass mem src));
7215
7216 ins_cost(INSN_COST);
7217 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7218
7219 ins_encode(aarch64_enc_strw(src, mem));
7220
7221 ins_pipe(istore_reg_mem);
7222 %}
7223
7224 // TODO
7225 // implement storeImmD0 and storeDImmPacked
7226
7227 // prefetch instructions
7228 // Must be safe to execute with invalid address (cannot fault).
7229
7230 instruct prefetchalloc( memory8 mem ) %{
7231 match(PrefetchAllocation mem);
7232
7233 ins_cost(INSN_COST);
7234 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7235
7236 ins_encode( aarch64_enc_prefetchw(mem) );
7237
7238 ins_pipe(iload_prefetch);
7239 %}
7240
7241 // ---------------- volatile loads and stores ----------------
7242
7243 // Load Byte (8 bit signed)
7244 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7245 %{
7246 match(Set dst (LoadB mem));
7247
7248 ins_cost(VOLATILE_REF_COST);
7249 format %{ "ldarsb $dst, $mem\t# byte" %}
7250
7251 ins_encode(aarch64_enc_ldarsb(dst, mem));
7252
7253 ins_pipe(pipe_serial);
7254 %}
7255
7256 // Load Byte (8 bit signed) into long
7257 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7258 %{
7259 match(Set dst (ConvI2L (LoadB mem)));
7260
7261 ins_cost(VOLATILE_REF_COST);
7262 format %{ "ldarsb $dst, $mem\t# byte" %}
7263
7264 ins_encode(aarch64_enc_ldarsb(dst, mem));
7265
7266 ins_pipe(pipe_serial);
7267 %}
7268
7269 // Load Byte (8 bit unsigned)
7270 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7271 %{
7272 match(Set dst (LoadUB mem));
7273
7274 ins_cost(VOLATILE_REF_COST);
7275 format %{ "ldarb $dst, $mem\t# byte" %}
7276
7277 ins_encode(aarch64_enc_ldarb(dst, mem));
7278
7279 ins_pipe(pipe_serial);
7280 %}
7281
7282 // Load Byte (8 bit unsigned) into long
7283 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7284 %{
7285 match(Set dst (ConvI2L (LoadUB mem)));
7286
7287 ins_cost(VOLATILE_REF_COST);
7288 format %{ "ldarb $dst, $mem\t# byte" %}
7289
7290 ins_encode(aarch64_enc_ldarb(dst, mem));
7291
7292 ins_pipe(pipe_serial);
7293 %}
7294
7295 // Load Short (16 bit signed)
7296 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7297 %{
7298 match(Set dst (LoadS mem));
7299
7300 ins_cost(VOLATILE_REF_COST);
7301 format %{ "ldarshw $dst, $mem\t# short" %}
7302
7303 ins_encode(aarch64_enc_ldarshw(dst, mem));
7304
7305 ins_pipe(pipe_serial);
7306 %}
7307
7308 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7309 %{
7310 match(Set dst (LoadUS mem));
7311
7312 ins_cost(VOLATILE_REF_COST);
7313 format %{ "ldarhw $dst, $mem\t# short" %}
7314
7315 ins_encode(aarch64_enc_ldarhw(dst, mem));
7316
7317 ins_pipe(pipe_serial);
7318 %}
7319
7320 // Load Short/Char (16 bit unsigned) into long
7321 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7322 %{
7323 match(Set dst (ConvI2L (LoadUS mem)));
7324
7325 ins_cost(VOLATILE_REF_COST);
7326 format %{ "ldarh $dst, $mem\t# short" %}
7327
7328 ins_encode(aarch64_enc_ldarh(dst, mem));
7329
7330 ins_pipe(pipe_serial);
7331 %}
7332
7333 // Load Short/Char (16 bit signed) into long
7334 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7335 %{
7336 match(Set dst (ConvI2L (LoadS mem)));
7337
7338 ins_cost(VOLATILE_REF_COST);
7339 format %{ "ldarh $dst, $mem\t# short" %}
7340
7341 ins_encode(aarch64_enc_ldarsh(dst, mem));
7342
7343 ins_pipe(pipe_serial);
7344 %}
7345
7346 // Load Integer (32 bit signed)
7347 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7348 %{
7349 match(Set dst (LoadI mem));
7350
7351 ins_cost(VOLATILE_REF_COST);
7352 format %{ "ldarw $dst, $mem\t# int" %}
7353
7354 ins_encode(aarch64_enc_ldarw(dst, mem));
7355
7356 ins_pipe(pipe_serial);
7357 %}
7358
7359 // Load Integer (32 bit unsigned) into long
7360 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7361 %{
7362 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7363
7364 ins_cost(VOLATILE_REF_COST);
7365 format %{ "ldarw $dst, $mem\t# int" %}
7366
7367 ins_encode(aarch64_enc_ldarw(dst, mem));
7368
7369 ins_pipe(pipe_serial);
7370 %}
7371
7372 // Load Long (64 bit signed)
7373 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7374 %{
7375 match(Set dst (LoadL mem));
7376
7377 ins_cost(VOLATILE_REF_COST);
7378 format %{ "ldar $dst, $mem\t# int" %}
7379
7380 ins_encode(aarch64_enc_ldar(dst, mem));
7381
7382 ins_pipe(pipe_serial);
7383 %}
7384
7385 // Load Pointer
7386 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7387 %{
7388 match(Set dst (LoadP mem));
7389 predicate(n->as_Load()->barrier_data() == 0);
7390
7391 ins_cost(VOLATILE_REF_COST);
7392 format %{ "ldar $dst, $mem\t# ptr" %}
7393
7394 ins_encode(aarch64_enc_ldar(dst, mem));
7395
7396 ins_pipe(pipe_serial);
7397 %}
7398
7399 // Load Compressed Pointer
7400 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7401 %{
7402 match(Set dst (LoadN mem));
7403 predicate(n->as_Load()->barrier_data() == 0);
7404
7405 ins_cost(VOLATILE_REF_COST);
7406 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7407
7408 ins_encode(aarch64_enc_ldarw(dst, mem));
7409
7410 ins_pipe(pipe_serial);
7411 %}
7412
7413 // Load Float
7414 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7415 %{
7416 match(Set dst (LoadF mem));
7417
7418 ins_cost(VOLATILE_REF_COST);
7419 format %{ "ldars $dst, $mem\t# float" %}
7420
7421 ins_encode( aarch64_enc_fldars(dst, mem) );
7422
7423 ins_pipe(pipe_serial);
7424 %}
7425
7426 // Load Double
7427 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7428 %{
7429 match(Set dst (LoadD mem));
7430
7431 ins_cost(VOLATILE_REF_COST);
7432 format %{ "ldard $dst, $mem\t# double" %}
7433
7434 ins_encode( aarch64_enc_fldard(dst, mem) );
7435
7436 ins_pipe(pipe_serial);
7437 %}
7438
7439 // Store Byte
7440 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7441 %{
7442 match(Set mem (StoreB mem src));
7443
7444 ins_cost(VOLATILE_REF_COST);
7445 format %{ "stlrb $src, $mem\t# byte" %}
7446
7447 ins_encode(aarch64_enc_stlrb(src, mem));
7448
7449 ins_pipe(pipe_class_memory);
7450 %}
7451
7452 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7453 %{
7454 match(Set mem (StoreB mem zero));
7455
7456 ins_cost(VOLATILE_REF_COST);
7457 format %{ "stlrb zr, $mem\t# byte" %}
7458
7459 ins_encode(aarch64_enc_stlrb0(mem));
7460
7461 ins_pipe(pipe_class_memory);
7462 %}
7463
7464 // Store Char/Short
7465 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7466 %{
7467 match(Set mem (StoreC mem src));
7468
7469 ins_cost(VOLATILE_REF_COST);
7470 format %{ "stlrh $src, $mem\t# short" %}
7471
7472 ins_encode(aarch64_enc_stlrh(src, mem));
7473
7474 ins_pipe(pipe_class_memory);
7475 %}
7476
7477 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7478 %{
7479 match(Set mem (StoreC mem zero));
7480
7481 ins_cost(VOLATILE_REF_COST);
7482 format %{ "stlrh zr, $mem\t# short" %}
7483
7484 ins_encode(aarch64_enc_stlrh0(mem));
7485
7486 ins_pipe(pipe_class_memory);
7487 %}
7488
7489 // Store Integer
7490
7491 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7492 %{
7493 match(Set mem(StoreI mem src));
7494
7495 ins_cost(VOLATILE_REF_COST);
7496 format %{ "stlrw $src, $mem\t# int" %}
7497
7498 ins_encode(aarch64_enc_stlrw(src, mem));
7499
7500 ins_pipe(pipe_class_memory);
7501 %}
7502
7503 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7504 %{
7505 match(Set mem(StoreI mem zero));
7506
7507 ins_cost(VOLATILE_REF_COST);
7508 format %{ "stlrw zr, $mem\t# int" %}
7509
7510 ins_encode(aarch64_enc_stlrw0(mem));
7511
7512 ins_pipe(pipe_class_memory);
7513 %}
7514
7515 // Store Long (64 bit signed)
7516 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7517 %{
7518 match(Set mem (StoreL mem src));
7519
7520 ins_cost(VOLATILE_REF_COST);
7521 format %{ "stlr $src, $mem\t# int" %}
7522
7523 ins_encode(aarch64_enc_stlr(src, mem));
7524
7525 ins_pipe(pipe_class_memory);
7526 %}
7527
7528 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7529 %{
7530 match(Set mem (StoreL mem zero));
7531
7532 ins_cost(VOLATILE_REF_COST);
7533 format %{ "stlr zr, $mem\t# int" %}
7534
7535 ins_encode(aarch64_enc_stlr0(mem));
7536
7537 ins_pipe(pipe_class_memory);
7538 %}
7539
7540 // Store Pointer
7541 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7542 %{
7543 match(Set mem (StoreP mem src));
7544 predicate(n->as_Store()->barrier_data() == 0);
7545
7546 ins_cost(VOLATILE_REF_COST);
7547 format %{ "stlr $src, $mem\t# ptr" %}
7548
7549 ins_encode(aarch64_enc_stlr(src, mem));
7550
7551 ins_pipe(pipe_class_memory);
7552 %}
7553
7554 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7555 %{
7556 match(Set mem (StoreP mem zero));
7557 predicate(n->as_Store()->barrier_data() == 0);
7558
7559 ins_cost(VOLATILE_REF_COST);
7560 format %{ "stlr zr, $mem\t# ptr" %}
7561
7562 ins_encode(aarch64_enc_stlr0(mem));
7563
7564 ins_pipe(pipe_class_memory);
7565 %}
7566
7567 // Store Compressed Pointer
7568 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7569 %{
7570 match(Set mem (StoreN mem src));
7571 predicate(n->as_Store()->barrier_data() == 0);
7572
7573 ins_cost(VOLATILE_REF_COST);
7574 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7575
7576 ins_encode(aarch64_enc_stlrw(src, mem));
7577
7578 ins_pipe(pipe_class_memory);
7579 %}
7580
7581 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7582 %{
7583 match(Set mem (StoreN mem zero));
7584 predicate(n->as_Store()->barrier_data() == 0);
7585
7586 ins_cost(VOLATILE_REF_COST);
7587 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7588
7589 ins_encode(aarch64_enc_stlrw0(mem));
7590
7591 ins_pipe(pipe_class_memory);
7592 %}
7593
7594 // Store Float
7595 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7596 %{
7597 match(Set mem (StoreF mem src));
7598
7599 ins_cost(VOLATILE_REF_COST);
7600 format %{ "stlrs $src, $mem\t# float" %}
7601
7602 ins_encode( aarch64_enc_fstlrs(src, mem) );
7603
7604 ins_pipe(pipe_class_memory);
7605 %}
7606
7607 // TODO
7608 // implement storeImmF0 and storeFImmPacked
7609
7610 // Store Double
7611 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7612 %{
7613 match(Set mem (StoreD mem src));
7614
7615 ins_cost(VOLATILE_REF_COST);
7616 format %{ "stlrd $src, $mem\t# double" %}
7617
7618 ins_encode( aarch64_enc_fstlrd(src, mem) );
7619
7620 ins_pipe(pipe_class_memory);
7621 %}
7622
7623 // ---------------- end of volatile loads and stores ----------------
7624
7625 instruct cacheWB(indirect addr)
7626 %{
7627 predicate(VM_Version::supports_data_cache_line_flush());
7628 match(CacheWB addr);
7629
7630 ins_cost(100);
7631 format %{"cache wb $addr" %}
7632 ins_encode %{
7633 assert($addr->index_position() < 0, "should be");
7634 assert($addr$$disp == 0, "should be");
7635 __ cache_wb(Address($addr$$base$$Register, 0));
7636 %}
7637 ins_pipe(pipe_slow); // XXX
7638 %}
7639
7640 instruct cacheWBPreSync()
7641 %{
7642 predicate(VM_Version::supports_data_cache_line_flush());
7643 match(CacheWBPreSync);
7644
7645 ins_cost(100);
7646 format %{"cache wb presync" %}
7647 ins_encode %{
7648 __ cache_wbsync(true);
7649 %}
7650 ins_pipe(pipe_slow); // XXX
7651 %}
7652
7653 instruct cacheWBPostSync()
7654 %{
7655 predicate(VM_Version::supports_data_cache_line_flush());
7656 match(CacheWBPostSync);
7657
7658 ins_cost(100);
7659 format %{"cache wb postsync" %}
7660 ins_encode %{
7661 __ cache_wbsync(false);
7662 %}
7663 ins_pipe(pipe_slow); // XXX
7664 %}
7665
7666 // ============================================================================
7667 // BSWAP Instructions
7668
7669 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7670 match(Set dst (ReverseBytesI src));
7671
7672 ins_cost(INSN_COST);
7673 format %{ "revw $dst, $src" %}
7674
7675 ins_encode %{
7676 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7677 %}
7678
7679 ins_pipe(ialu_reg);
7680 %}
7681
7682 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7683 match(Set dst (ReverseBytesL src));
7684
7685 ins_cost(INSN_COST);
7686 format %{ "rev $dst, $src" %}
7687
7688 ins_encode %{
7689 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7690 %}
7691
7692 ins_pipe(ialu_reg);
7693 %}
7694
7695 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7696 match(Set dst (ReverseBytesUS src));
7697
7698 ins_cost(INSN_COST);
7699 format %{ "rev16w $dst, $src" %}
7700
7701 ins_encode %{
7702 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7703 %}
7704
7705 ins_pipe(ialu_reg);
7706 %}
7707
7708 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7709 match(Set dst (ReverseBytesS src));
7710
7711 ins_cost(INSN_COST);
7712 format %{ "rev16w $dst, $src\n\t"
7713 "sbfmw $dst, $dst, #0, #15" %}
7714
7715 ins_encode %{
7716 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7717 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7718 %}
7719
7720 ins_pipe(ialu_reg);
7721 %}
7722
7723 // ============================================================================
7724 // Zero Count Instructions
7725
7726 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7727 match(Set dst (CountLeadingZerosI src));
7728
7729 ins_cost(INSN_COST);
7730 format %{ "clzw $dst, $src" %}
7731 ins_encode %{
7732 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7733 %}
7734
7735 ins_pipe(ialu_reg);
7736 %}
7737
7738 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7739 match(Set dst (CountLeadingZerosL src));
7740
7741 ins_cost(INSN_COST);
7742 format %{ "clz $dst, $src" %}
7743 ins_encode %{
7744 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7745 %}
7746
7747 ins_pipe(ialu_reg);
7748 %}
7749
7750 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7751 match(Set dst (CountTrailingZerosI src));
7752
7753 ins_cost(INSN_COST * 2);
7754 format %{ "rbitw $dst, $src\n\t"
7755 "clzw $dst, $dst" %}
7756 ins_encode %{
7757 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7758 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7759 %}
7760
7761 ins_pipe(ialu_reg);
7762 %}
7763
7764 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7765 match(Set dst (CountTrailingZerosL src));
7766
7767 ins_cost(INSN_COST * 2);
7768 format %{ "rbit $dst, $src\n\t"
7769 "clz $dst, $dst" %}
7770 ins_encode %{
7771 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7772 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7773 %}
7774
7775 ins_pipe(ialu_reg);
7776 %}
7777
7778 //---------- Population Count Instructions -------------------------------------
7779 //
7780
7781 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7782 match(Set dst (PopCountI src));
7783 effect(TEMP tmp);
7784 ins_cost(INSN_COST * 13);
7785
7786 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7787 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7788 "addv $tmp, $tmp\t# vector (8B)\n\t"
7789 "mov $dst, $tmp\t# vector (1D)" %}
7790 ins_encode %{
7791 __ fmovs($tmp$$FloatRegister, $src$$Register);
7792 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7793 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7794 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7795 %}
7796
7797 ins_pipe(pipe_class_default);
7798 %}
7799
7800 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7801 match(Set dst (PopCountI (LoadI mem)));
7802 effect(TEMP tmp);
7803 ins_cost(INSN_COST * 13);
7804
7805 format %{ "ldrs $tmp, $mem\n\t"
7806 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7807 "addv $tmp, $tmp\t# vector (8B)\n\t"
7808 "mov $dst, $tmp\t# vector (1D)" %}
7809 ins_encode %{
7810 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7811 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7812 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7813 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7814 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7815 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7816 %}
7817
7818 ins_pipe(pipe_class_default);
7819 %}
7820
7821 // Note: Long.bitCount(long) returns an int.
7822 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7823 match(Set dst (PopCountL src));
7824 effect(TEMP tmp);
7825 ins_cost(INSN_COST * 13);
7826
7827 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7828 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7829 "addv $tmp, $tmp\t# vector (8B)\n\t"
7830 "mov $dst, $tmp\t# vector (1D)" %}
7831 ins_encode %{
7832 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7833 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7834 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7835 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7836 %}
7837
7838 ins_pipe(pipe_class_default);
7839 %}
7840
7841 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7842 match(Set dst (PopCountL (LoadL mem)));
7843 effect(TEMP tmp);
7844 ins_cost(INSN_COST * 13);
7845
7846 format %{ "ldrd $tmp, $mem\n\t"
7847 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7848 "addv $tmp, $tmp\t# vector (8B)\n\t"
7849 "mov $dst, $tmp\t# vector (1D)" %}
7850 ins_encode %{
7851 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7852 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7853 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7854 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7855 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7856 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7857 %}
7858
7859 ins_pipe(pipe_class_default);
7860 %}
7861
7862 // ============================================================================
7863 // VerifyVectorAlignment Instruction
7864
7865 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7866 match(Set addr (VerifyVectorAlignment addr mask));
7867 effect(KILL cr);
7868 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7869 ins_encode %{
7870 Label Lskip;
7871 // check if masked bits of addr are zero
7872 __ tst($addr$$Register, $mask$$constant);
7873 __ br(Assembler::EQ, Lskip);
7874 __ stop("verify_vector_alignment found a misaligned vector memory access");
7875 __ bind(Lskip);
7876 %}
7877 ins_pipe(pipe_slow);
7878 %}
7879
7880 // ============================================================================
7881 // MemBar Instruction
7882
7883 instruct load_fence() %{
7884 match(LoadFence);
7885 ins_cost(VOLATILE_REF_COST);
7886
7887 format %{ "load_fence" %}
7888
7889 ins_encode %{
7890 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7891 %}
7892 ins_pipe(pipe_serial);
7893 %}
7894
7895 instruct unnecessary_membar_acquire() %{
7896 predicate(unnecessary_acquire(n));
7897 match(MemBarAcquire);
7898 ins_cost(0);
7899
7900 format %{ "membar_acquire (elided)" %}
7901
7902 ins_encode %{
7903 __ block_comment("membar_acquire (elided)");
7904 %}
7905
7906 ins_pipe(pipe_class_empty);
7907 %}
7908
7909 instruct membar_acquire() %{
7910 match(MemBarAcquire);
7911 ins_cost(VOLATILE_REF_COST);
7912
7913 format %{ "membar_acquire\n\t"
7914 "dmb ishld" %}
7915
7916 ins_encode %{
7917 __ block_comment("membar_acquire");
7918 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7919 %}
7920
7921 ins_pipe(pipe_serial);
7922 %}
7923
7924
7925 instruct membar_acquire_lock() %{
7926 match(MemBarAcquireLock);
7927 ins_cost(VOLATILE_REF_COST);
7928
7929 format %{ "membar_acquire_lock (elided)" %}
7930
7931 ins_encode %{
7932 __ block_comment("membar_acquire_lock (elided)");
7933 %}
7934
7935 ins_pipe(pipe_serial);
7936 %}
7937
7938 instruct store_fence() %{
7939 match(StoreFence);
7940 ins_cost(VOLATILE_REF_COST);
7941
7942 format %{ "store_fence" %}
7943
7944 ins_encode %{
7945 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7946 %}
7947 ins_pipe(pipe_serial);
7948 %}
7949
7950 instruct unnecessary_membar_release() %{
7951 predicate(unnecessary_release(n));
7952 match(MemBarRelease);
7953 ins_cost(0);
7954
7955 format %{ "membar_release (elided)" %}
7956
7957 ins_encode %{
7958 __ block_comment("membar_release (elided)");
7959 %}
7960 ins_pipe(pipe_serial);
7961 %}
7962
7963 instruct membar_release() %{
7964 match(MemBarRelease);
7965 ins_cost(VOLATILE_REF_COST);
7966
7967 format %{ "membar_release\n\t"
7968 "dmb ishst\n\tdmb ishld" %}
7969
7970 ins_encode %{
7971 __ block_comment("membar_release");
7972 // These will be merged if AlwaysMergeDMB is enabled.
7973 __ membar(Assembler::StoreStore);
7974 __ membar(Assembler::LoadStore);
7975 %}
7976 ins_pipe(pipe_serial);
7977 %}
7978
7979 instruct membar_storestore() %{
7980 match(MemBarStoreStore);
7981 match(StoreStoreFence);
7982 ins_cost(VOLATILE_REF_COST);
7983
7984 format %{ "MEMBAR-store-store" %}
7985
7986 ins_encode %{
7987 __ membar(Assembler::StoreStore);
7988 %}
7989 ins_pipe(pipe_serial);
7990 %}
7991
7992 instruct membar_release_lock() %{
7993 match(MemBarReleaseLock);
7994 ins_cost(VOLATILE_REF_COST);
7995
7996 format %{ "membar_release_lock (elided)" %}
7997
7998 ins_encode %{
7999 __ block_comment("membar_release_lock (elided)");
8000 %}
8001
8002 ins_pipe(pipe_serial);
8003 %}
8004
8005 instruct unnecessary_membar_volatile() %{
8006 predicate(unnecessary_volatile(n));
8007 match(MemBarVolatile);
8008 ins_cost(0);
8009
8010 format %{ "membar_volatile (elided)" %}
8011
8012 ins_encode %{
8013 __ block_comment("membar_volatile (elided)");
8014 %}
8015
8016 ins_pipe(pipe_serial);
8017 %}
8018
8019 instruct membar_volatile() %{
8020 match(MemBarVolatile);
8021 ins_cost(VOLATILE_REF_COST*100);
8022
8023 format %{ "membar_volatile\n\t"
8024 "dmb ish"%}
8025
8026 ins_encode %{
8027 __ block_comment("membar_volatile");
8028 __ membar(Assembler::StoreLoad);
8029 %}
8030
8031 ins_pipe(pipe_serial);
8032 %}
8033
8034 // ============================================================================
8035 // Cast/Convert Instructions
8036
8037 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8038 match(Set dst (CastX2P src));
8039
8040 ins_cost(INSN_COST);
8041 format %{ "mov $dst, $src\t# long -> ptr" %}
8042
8043 ins_encode %{
8044 if ($dst$$reg != $src$$reg) {
8045 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8046 }
8047 %}
8048
8049 ins_pipe(ialu_reg);
8050 %}
8051
8052 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8053 match(Set dst (CastI2N src));
8054
8055 ins_cost(INSN_COST);
8056 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8057
8058 ins_encode %{
8059 if ($dst$$reg != $src$$reg) {
8060 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8061 }
8062 %}
8063
8064 ins_pipe(ialu_reg);
8065 %}
8066
8067 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8068 match(Set dst (CastP2X src));
8069
8070 ins_cost(INSN_COST);
8071 format %{ "mov $dst, $src\t# ptr -> long" %}
8072
8073 ins_encode %{
8074 if ($dst$$reg != $src$$reg) {
8075 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8076 }
8077 %}
8078
8079 ins_pipe(ialu_reg);
8080 %}
8081
8082 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8083 match(Set dst (CastP2X src));
8084
8085 ins_cost(INSN_COST);
8086 format %{ "mov $dst, $src\t# ptr -> long" %}
8087
8088 ins_encode %{
8089 if ($dst$$reg != $src$$reg) {
8090 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8091 }
8092 %}
8093
8094 ins_pipe(ialu_reg);
8095 %}
8096
8097 // Convert oop into int for vectors alignment masking
8098 instruct convP2I(iRegINoSp dst, iRegP src) %{
8099 match(Set dst (ConvL2I (CastP2X src)));
8100
8101 ins_cost(INSN_COST);
8102 format %{ "movw $dst, $src\t# ptr -> int" %}
8103 ins_encode %{
8104 __ movw($dst$$Register, $src$$Register);
8105 %}
8106
8107 ins_pipe(ialu_reg);
8108 %}
8109
8110 // Convert compressed oop into int for vectors alignment masking
8111 // in case of 32bit oops (heap < 4Gb).
8112 instruct convN2I(iRegINoSp dst, iRegN src)
8113 %{
8114 predicate(CompressedOops::shift() == 0);
8115 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8116
8117 ins_cost(INSN_COST);
8118 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8119 ins_encode %{
8120 __ movw($dst$$Register, $src$$Register);
8121 %}
8122
8123 ins_pipe(ialu_reg);
8124 %}
8125
8126
8127 // Convert oop pointer into compressed form
8128 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8129 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8130 match(Set dst (EncodeP src));
8131 effect(KILL cr);
8132 ins_cost(INSN_COST * 3);
8133 format %{ "encode_heap_oop $dst, $src" %}
8134 ins_encode %{
8135 Register s = $src$$Register;
8136 Register d = $dst$$Register;
8137 __ encode_heap_oop(d, s);
8138 %}
8139 ins_pipe(ialu_reg);
8140 %}
8141
8142 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8143 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8144 match(Set dst (EncodeP src));
8145 ins_cost(INSN_COST * 3);
8146 format %{ "encode_heap_oop_not_null $dst, $src" %}
8147 ins_encode %{
8148 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8149 %}
8150 ins_pipe(ialu_reg);
8151 %}
8152
8153 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8154 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8155 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8156 match(Set dst (DecodeN src));
8157 ins_cost(INSN_COST * 3);
8158 format %{ "decode_heap_oop $dst, $src" %}
8159 ins_encode %{
8160 Register s = $src$$Register;
8161 Register d = $dst$$Register;
8162 __ decode_heap_oop(d, s);
8163 %}
8164 ins_pipe(ialu_reg);
8165 %}
8166
8167 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8168 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8169 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8170 match(Set dst (DecodeN src));
8171 ins_cost(INSN_COST * 3);
8172 format %{ "decode_heap_oop_not_null $dst, $src" %}
8173 ins_encode %{
8174 Register s = $src$$Register;
8175 Register d = $dst$$Register;
8176 __ decode_heap_oop_not_null(d, s);
8177 %}
8178 ins_pipe(ialu_reg);
8179 %}
8180
8181 // n.b. AArch64 implementations of encode_klass_not_null and
8182 // decode_klass_not_null do not modify the flags register so, unlike
8183 // Intel, we don't kill CR as a side effect here
8184
8185 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8186 match(Set dst (EncodePKlass src));
8187
8188 ins_cost(INSN_COST * 3);
8189 format %{ "encode_klass_not_null $dst,$src" %}
8190
8191 ins_encode %{
8192 Register src_reg = as_Register($src$$reg);
8193 Register dst_reg = as_Register($dst$$reg);
8194 __ encode_klass_not_null(dst_reg, src_reg);
8195 %}
8196
8197 ins_pipe(ialu_reg);
8198 %}
8199
8200 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8201 match(Set dst (DecodeNKlass src));
8202
8203 ins_cost(INSN_COST * 3);
8204 format %{ "decode_klass_not_null $dst,$src" %}
8205
8206 ins_encode %{
8207 Register src_reg = as_Register($src$$reg);
8208 Register dst_reg = as_Register($dst$$reg);
8209 if (dst_reg != src_reg) {
8210 __ decode_klass_not_null(dst_reg, src_reg);
8211 } else {
8212 __ decode_klass_not_null(dst_reg);
8213 }
8214 %}
8215
8216 ins_pipe(ialu_reg);
8217 %}
8218
8219 instruct checkCastPP(iRegPNoSp dst)
8220 %{
8221 match(Set dst (CheckCastPP dst));
8222
8223 size(0);
8224 format %{ "# checkcastPP of $dst" %}
8225 ins_encode(/* empty encoding */);
8226 ins_pipe(pipe_class_empty);
8227 %}
8228
8229 instruct castPP(iRegPNoSp dst)
8230 %{
8231 match(Set dst (CastPP dst));
8232
8233 size(0);
8234 format %{ "# castPP of $dst" %}
8235 ins_encode(/* empty encoding */);
8236 ins_pipe(pipe_class_empty);
8237 %}
8238
8239 instruct castII(iRegI dst)
8240 %{
8241 predicate(VerifyConstraintCasts == 0);
8242 match(Set dst (CastII dst));
8243
8244 size(0);
8245 format %{ "# castII of $dst" %}
8246 ins_encode(/* empty encoding */);
8247 ins_cost(0);
8248 ins_pipe(pipe_class_empty);
8249 %}
8250
8251 instruct castII_checked(iRegI dst, rFlagsReg cr)
8252 %{
8253 predicate(VerifyConstraintCasts > 0);
8254 match(Set dst (CastII dst));
8255 effect(KILL cr);
8256
8257 format %{ "# castII_checked of $dst" %}
8258 ins_encode %{
8259 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8260 %}
8261 ins_pipe(pipe_slow);
8262 %}
8263
8264 instruct castLL(iRegL dst)
8265 %{
8266 predicate(VerifyConstraintCasts == 0);
8267 match(Set dst (CastLL dst));
8268
8269 size(0);
8270 format %{ "# castLL of $dst" %}
8271 ins_encode(/* empty encoding */);
8272 ins_cost(0);
8273 ins_pipe(pipe_class_empty);
8274 %}
8275
8276 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8277 %{
8278 predicate(VerifyConstraintCasts > 0);
8279 match(Set dst (CastLL dst));
8280 effect(KILL cr);
8281
8282 format %{ "# castLL_checked of $dst" %}
8283 ins_encode %{
8284 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8285 %}
8286 ins_pipe(pipe_slow);
8287 %}
8288
8289 instruct castHH(vRegF dst)
8290 %{
8291 match(Set dst (CastHH dst));
8292 size(0);
8293 format %{ "# castHH of $dst" %}
8294 ins_encode(/* empty encoding */);
8295 ins_cost(0);
8296 ins_pipe(pipe_class_empty);
8297 %}
8298
8299 instruct castFF(vRegF dst)
8300 %{
8301 match(Set dst (CastFF dst));
8302
8303 size(0);
8304 format %{ "# castFF of $dst" %}
8305 ins_encode(/* empty encoding */);
8306 ins_cost(0);
8307 ins_pipe(pipe_class_empty);
8308 %}
8309
8310 instruct castDD(vRegD dst)
8311 %{
8312 match(Set dst (CastDD dst));
8313
8314 size(0);
8315 format %{ "# castDD of $dst" %}
8316 ins_encode(/* empty encoding */);
8317 ins_cost(0);
8318 ins_pipe(pipe_class_empty);
8319 %}
8320
8321 instruct castVV(vReg dst)
8322 %{
8323 match(Set dst (CastVV dst));
8324
8325 size(0);
8326 format %{ "# castVV of $dst" %}
8327 ins_encode(/* empty encoding */);
8328 ins_cost(0);
8329 ins_pipe(pipe_class_empty);
8330 %}
8331
8332 instruct castVVMask(pRegGov dst)
8333 %{
8334 match(Set dst (CastVV dst));
8335
8336 size(0);
8337 format %{ "# castVV of $dst" %}
8338 ins_encode(/* empty encoding */);
8339 ins_cost(0);
8340 ins_pipe(pipe_class_empty);
8341 %}
8342
8343 // ============================================================================
8344 // Atomic operation instructions
8345 //
8346
8347 // standard CompareAndSwapX when we are using barriers
8348 // these have higher priority than the rules selected by a predicate
8349
8350 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8351 // can't match them
8352
8353 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8354
8355 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8356 ins_cost(2 * VOLATILE_REF_COST);
8357
8358 effect(KILL cr);
8359
8360 format %{
8361 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8362 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8363 %}
8364
8365 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8366 aarch64_enc_cset_eq(res));
8367
8368 ins_pipe(pipe_slow);
8369 %}
8370
8371 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8372
8373 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8374 ins_cost(2 * VOLATILE_REF_COST);
8375
8376 effect(KILL cr);
8377
8378 format %{
8379 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8380 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8381 %}
8382
8383 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8384 aarch64_enc_cset_eq(res));
8385
8386 ins_pipe(pipe_slow);
8387 %}
8388
8389 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8390
8391 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8392 ins_cost(2 * VOLATILE_REF_COST);
8393
8394 effect(KILL cr);
8395
8396 format %{
8397 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8398 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8399 %}
8400
8401 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8402 aarch64_enc_cset_eq(res));
8403
8404 ins_pipe(pipe_slow);
8405 %}
8406
8407 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8408
8409 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8410 ins_cost(2 * VOLATILE_REF_COST);
8411
8412 effect(KILL cr);
8413
8414 format %{
8415 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8416 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8417 %}
8418
8419 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8420 aarch64_enc_cset_eq(res));
8421
8422 ins_pipe(pipe_slow);
8423 %}
8424
8425 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8426
8427 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8428 predicate(n->as_LoadStore()->barrier_data() == 0);
8429 ins_cost(2 * VOLATILE_REF_COST);
8430
8431 effect(KILL cr);
8432
8433 format %{
8434 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8435 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8436 %}
8437
8438 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8439 aarch64_enc_cset_eq(res));
8440
8441 ins_pipe(pipe_slow);
8442 %}
8443
8444 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8445
8446 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8447 predicate(n->as_LoadStore()->barrier_data() == 0);
8448 ins_cost(2 * VOLATILE_REF_COST);
8449
8450 effect(KILL cr);
8451
8452 format %{
8453 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8454 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8455 %}
8456
8457 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8458 aarch64_enc_cset_eq(res));
8459
8460 ins_pipe(pipe_slow);
8461 %}
8462
8463 // alternative CompareAndSwapX when we are eliding barriers
8464
8465 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8466
8467 predicate(needs_acquiring_load_exclusive(n));
8468 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8469 ins_cost(VOLATILE_REF_COST);
8470
8471 effect(KILL cr);
8472
8473 format %{
8474 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8475 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8476 %}
8477
8478 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8479 aarch64_enc_cset_eq(res));
8480
8481 ins_pipe(pipe_slow);
8482 %}
8483
8484 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8485
8486 predicate(needs_acquiring_load_exclusive(n));
8487 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8488 ins_cost(VOLATILE_REF_COST);
8489
8490 effect(KILL cr);
8491
8492 format %{
8493 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8494 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8495 %}
8496
8497 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8498 aarch64_enc_cset_eq(res));
8499
8500 ins_pipe(pipe_slow);
8501 %}
8502
8503 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8504
8505 predicate(needs_acquiring_load_exclusive(n));
8506 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8507 ins_cost(VOLATILE_REF_COST);
8508
8509 effect(KILL cr);
8510
8511 format %{
8512 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8513 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8514 %}
8515
8516 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8517 aarch64_enc_cset_eq(res));
8518
8519 ins_pipe(pipe_slow);
8520 %}
8521
8522 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8523
8524 predicate(needs_acquiring_load_exclusive(n));
8525 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8526 ins_cost(VOLATILE_REF_COST);
8527
8528 effect(KILL cr);
8529
8530 format %{
8531 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8532 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8533 %}
8534
8535 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8536 aarch64_enc_cset_eq(res));
8537
8538 ins_pipe(pipe_slow);
8539 %}
8540
8541 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8542
8543 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8544 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8545 ins_cost(VOLATILE_REF_COST);
8546
8547 effect(KILL cr);
8548
8549 format %{
8550 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8551 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8552 %}
8553
8554 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8555 aarch64_enc_cset_eq(res));
8556
8557 ins_pipe(pipe_slow);
8558 %}
8559
8560 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8561
8562 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8563 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8564 ins_cost(VOLATILE_REF_COST);
8565
8566 effect(KILL cr);
8567
8568 format %{
8569 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8570 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8571 %}
8572
8573 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8574 aarch64_enc_cset_eq(res));
8575
8576 ins_pipe(pipe_slow);
8577 %}
8578
8579
8580 // ---------------------------------------------------------------------
8581
8582 // BEGIN This section of the file is automatically generated. Do not edit --------------
8583
8584 // Sundry CAS operations. Note that release is always true,
8585 // regardless of the memory ordering of the CAS. This is because we
8586 // need the volatile case to be sequentially consistent but there is
8587 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8588 // can't check the type of memory ordering here, so we always emit a
8589 // STLXR.
8590
8591 // This section is generated from cas.m4
8592
8593
8594 // This pattern is generated automatically from cas.m4.
8595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8596 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8597 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8598 ins_cost(2 * VOLATILE_REF_COST);
8599 effect(TEMP_DEF res, KILL cr);
8600 format %{
8601 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8602 %}
8603 ins_encode %{
8604 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8605 Assembler::byte, /*acquire*/ false, /*release*/ true,
8606 /*weak*/ false, $res$$Register);
8607 __ sxtbw($res$$Register, $res$$Register);
8608 %}
8609 ins_pipe(pipe_slow);
8610 %}
8611
8612 // This pattern is generated automatically from cas.m4.
8613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8614 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8615 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8616 ins_cost(2 * VOLATILE_REF_COST);
8617 effect(TEMP_DEF res, KILL cr);
8618 format %{
8619 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8620 %}
8621 ins_encode %{
8622 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8623 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8624 /*weak*/ false, $res$$Register);
8625 __ sxthw($res$$Register, $res$$Register);
8626 %}
8627 ins_pipe(pipe_slow);
8628 %}
8629
8630 // This pattern is generated automatically from cas.m4.
8631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8632 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8633 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8634 ins_cost(2 * VOLATILE_REF_COST);
8635 effect(TEMP_DEF res, KILL cr);
8636 format %{
8637 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8638 %}
8639 ins_encode %{
8640 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8641 Assembler::word, /*acquire*/ false, /*release*/ true,
8642 /*weak*/ false, $res$$Register);
8643 %}
8644 ins_pipe(pipe_slow);
8645 %}
8646
8647 // This pattern is generated automatically from cas.m4.
8648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8649 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8650 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8651 ins_cost(2 * VOLATILE_REF_COST);
8652 effect(TEMP_DEF res, KILL cr);
8653 format %{
8654 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8655 %}
8656 ins_encode %{
8657 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8658 Assembler::xword, /*acquire*/ false, /*release*/ true,
8659 /*weak*/ false, $res$$Register);
8660 %}
8661 ins_pipe(pipe_slow);
8662 %}
8663
8664 // This pattern is generated automatically from cas.m4.
8665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8666 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8667 predicate(n->as_LoadStore()->barrier_data() == 0);
8668 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8669 ins_cost(2 * VOLATILE_REF_COST);
8670 effect(TEMP_DEF res, KILL cr);
8671 format %{
8672 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8673 %}
8674 ins_encode %{
8675 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8676 Assembler::word, /*acquire*/ false, /*release*/ true,
8677 /*weak*/ false, $res$$Register);
8678 %}
8679 ins_pipe(pipe_slow);
8680 %}
8681
8682 // This pattern is generated automatically from cas.m4.
8683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8684 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8685 predicate(n->as_LoadStore()->barrier_data() == 0);
8686 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8687 ins_cost(2 * VOLATILE_REF_COST);
8688 effect(TEMP_DEF res, KILL cr);
8689 format %{
8690 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8691 %}
8692 ins_encode %{
8693 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8694 Assembler::xword, /*acquire*/ false, /*release*/ true,
8695 /*weak*/ false, $res$$Register);
8696 %}
8697 ins_pipe(pipe_slow);
8698 %}
8699
8700 // This pattern is generated automatically from cas.m4.
8701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8702 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8703 predicate(needs_acquiring_load_exclusive(n));
8704 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8705 ins_cost(VOLATILE_REF_COST);
8706 effect(TEMP_DEF res, KILL cr);
8707 format %{
8708 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8709 %}
8710 ins_encode %{
8711 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8712 Assembler::byte, /*acquire*/ true, /*release*/ true,
8713 /*weak*/ false, $res$$Register);
8714 __ sxtbw($res$$Register, $res$$Register);
8715 %}
8716 ins_pipe(pipe_slow);
8717 %}
8718
8719 // This pattern is generated automatically from cas.m4.
8720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8721 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8722 predicate(needs_acquiring_load_exclusive(n));
8723 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8724 ins_cost(VOLATILE_REF_COST);
8725 effect(TEMP_DEF res, KILL cr);
8726 format %{
8727 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8728 %}
8729 ins_encode %{
8730 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8731 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8732 /*weak*/ false, $res$$Register);
8733 __ sxthw($res$$Register, $res$$Register);
8734 %}
8735 ins_pipe(pipe_slow);
8736 %}
8737
8738 // This pattern is generated automatically from cas.m4.
8739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8740 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8741 predicate(needs_acquiring_load_exclusive(n));
8742 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8743 ins_cost(VOLATILE_REF_COST);
8744 effect(TEMP_DEF res, KILL cr);
8745 format %{
8746 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8747 %}
8748 ins_encode %{
8749 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8750 Assembler::word, /*acquire*/ true, /*release*/ true,
8751 /*weak*/ false, $res$$Register);
8752 %}
8753 ins_pipe(pipe_slow);
8754 %}
8755
8756 // This pattern is generated automatically from cas.m4.
8757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8758 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8759 predicate(needs_acquiring_load_exclusive(n));
8760 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8761 ins_cost(VOLATILE_REF_COST);
8762 effect(TEMP_DEF res, KILL cr);
8763 format %{
8764 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8765 %}
8766 ins_encode %{
8767 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8768 Assembler::xword, /*acquire*/ true, /*release*/ true,
8769 /*weak*/ false, $res$$Register);
8770 %}
8771 ins_pipe(pipe_slow);
8772 %}
8773
8774 // This pattern is generated automatically from cas.m4.
8775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8776 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8777 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8778 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8779 ins_cost(VOLATILE_REF_COST);
8780 effect(TEMP_DEF res, KILL cr);
8781 format %{
8782 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8783 %}
8784 ins_encode %{
8785 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8786 Assembler::word, /*acquire*/ true, /*release*/ true,
8787 /*weak*/ false, $res$$Register);
8788 %}
8789 ins_pipe(pipe_slow);
8790 %}
8791
8792 // This pattern is generated automatically from cas.m4.
8793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8794 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8795 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8796 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8797 ins_cost(VOLATILE_REF_COST);
8798 effect(TEMP_DEF res, KILL cr);
8799 format %{
8800 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8801 %}
8802 ins_encode %{
8803 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8804 Assembler::xword, /*acquire*/ true, /*release*/ true,
8805 /*weak*/ false, $res$$Register);
8806 %}
8807 ins_pipe(pipe_slow);
8808 %}
8809
8810 // This pattern is generated automatically from cas.m4.
8811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8812 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8813 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8814 ins_cost(2 * VOLATILE_REF_COST);
8815 effect(KILL cr);
8816 format %{
8817 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8818 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8819 %}
8820 ins_encode %{
8821 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8822 Assembler::byte, /*acquire*/ false, /*release*/ true,
8823 /*weak*/ true, noreg);
8824 __ csetw($res$$Register, Assembler::EQ);
8825 %}
8826 ins_pipe(pipe_slow);
8827 %}
8828
8829 // This pattern is generated automatically from cas.m4.
8830 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8831 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8832 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8833 ins_cost(2 * VOLATILE_REF_COST);
8834 effect(KILL cr);
8835 format %{
8836 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8837 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8838 %}
8839 ins_encode %{
8840 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8841 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8842 /*weak*/ true, noreg);
8843 __ csetw($res$$Register, Assembler::EQ);
8844 %}
8845 ins_pipe(pipe_slow);
8846 %}
8847
8848 // This pattern is generated automatically from cas.m4.
8849 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8850 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8851 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8852 ins_cost(2 * VOLATILE_REF_COST);
8853 effect(KILL cr);
8854 format %{
8855 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8856 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8857 %}
8858 ins_encode %{
8859 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8860 Assembler::word, /*acquire*/ false, /*release*/ true,
8861 /*weak*/ true, noreg);
8862 __ csetw($res$$Register, Assembler::EQ);
8863 %}
8864 ins_pipe(pipe_slow);
8865 %}
8866
8867 // This pattern is generated automatically from cas.m4.
8868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8869 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8870 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8871 ins_cost(2 * VOLATILE_REF_COST);
8872 effect(KILL cr);
8873 format %{
8874 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8875 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8876 %}
8877 ins_encode %{
8878 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8879 Assembler::xword, /*acquire*/ false, /*release*/ true,
8880 /*weak*/ true, noreg);
8881 __ csetw($res$$Register, Assembler::EQ);
8882 %}
8883 ins_pipe(pipe_slow);
8884 %}
8885
8886 // This pattern is generated automatically from cas.m4.
8887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8888 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8889 predicate(n->as_LoadStore()->barrier_data() == 0);
8890 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8891 ins_cost(2 * VOLATILE_REF_COST);
8892 effect(KILL cr);
8893 format %{
8894 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8895 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8896 %}
8897 ins_encode %{
8898 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8899 Assembler::word, /*acquire*/ false, /*release*/ true,
8900 /*weak*/ true, noreg);
8901 __ csetw($res$$Register, Assembler::EQ);
8902 %}
8903 ins_pipe(pipe_slow);
8904 %}
8905
8906 // This pattern is generated automatically from cas.m4.
8907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8908 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8909 predicate(n->as_LoadStore()->barrier_data() == 0);
8910 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8911 ins_cost(2 * VOLATILE_REF_COST);
8912 effect(KILL cr);
8913 format %{
8914 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8915 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8916 %}
8917 ins_encode %{
8918 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8919 Assembler::xword, /*acquire*/ false, /*release*/ true,
8920 /*weak*/ true, noreg);
8921 __ csetw($res$$Register, Assembler::EQ);
8922 %}
8923 ins_pipe(pipe_slow);
8924 %}
8925
8926 // This pattern is generated automatically from cas.m4.
8927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8928 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8929 predicate(needs_acquiring_load_exclusive(n));
8930 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8931 ins_cost(VOLATILE_REF_COST);
8932 effect(KILL cr);
8933 format %{
8934 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8935 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8936 %}
8937 ins_encode %{
8938 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8939 Assembler::byte, /*acquire*/ true, /*release*/ true,
8940 /*weak*/ true, noreg);
8941 __ csetw($res$$Register, Assembler::EQ);
8942 %}
8943 ins_pipe(pipe_slow);
8944 %}
8945
8946 // This pattern is generated automatically from cas.m4.
8947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8948 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8949 predicate(needs_acquiring_load_exclusive(n));
8950 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8951 ins_cost(VOLATILE_REF_COST);
8952 effect(KILL cr);
8953 format %{
8954 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8955 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8956 %}
8957 ins_encode %{
8958 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8959 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8960 /*weak*/ true, noreg);
8961 __ csetw($res$$Register, Assembler::EQ);
8962 %}
8963 ins_pipe(pipe_slow);
8964 %}
8965
8966 // This pattern is generated automatically from cas.m4.
8967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8968 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8969 predicate(needs_acquiring_load_exclusive(n));
8970 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8971 ins_cost(VOLATILE_REF_COST);
8972 effect(KILL cr);
8973 format %{
8974 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8975 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8976 %}
8977 ins_encode %{
8978 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8979 Assembler::word, /*acquire*/ true, /*release*/ true,
8980 /*weak*/ true, noreg);
8981 __ csetw($res$$Register, Assembler::EQ);
8982 %}
8983 ins_pipe(pipe_slow);
8984 %}
8985
8986 // This pattern is generated automatically from cas.m4.
8987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8988 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8989 predicate(needs_acquiring_load_exclusive(n));
8990 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8991 ins_cost(VOLATILE_REF_COST);
8992 effect(KILL cr);
8993 format %{
8994 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8995 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8996 %}
8997 ins_encode %{
8998 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8999 Assembler::xword, /*acquire*/ true, /*release*/ true,
9000 /*weak*/ true, noreg);
9001 __ csetw($res$$Register, Assembler::EQ);
9002 %}
9003 ins_pipe(pipe_slow);
9004 %}
9005
9006 // This pattern is generated automatically from cas.m4.
9007 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9008 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9009 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9010 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9011 ins_cost(VOLATILE_REF_COST);
9012 effect(KILL cr);
9013 format %{
9014 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9015 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9016 %}
9017 ins_encode %{
9018 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9019 Assembler::word, /*acquire*/ true, /*release*/ true,
9020 /*weak*/ true, noreg);
9021 __ csetw($res$$Register, Assembler::EQ);
9022 %}
9023 ins_pipe(pipe_slow);
9024 %}
9025
9026 // This pattern is generated automatically from cas.m4.
9027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9028 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9029 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9030 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9031 ins_cost(VOLATILE_REF_COST);
9032 effect(KILL cr);
9033 format %{
9034 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9035 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9036 %}
9037 ins_encode %{
9038 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9039 Assembler::xword, /*acquire*/ true, /*release*/ true,
9040 /*weak*/ true, noreg);
9041 __ csetw($res$$Register, Assembler::EQ);
9042 %}
9043 ins_pipe(pipe_slow);
9044 %}
9045
9046 // END This section of the file is automatically generated. Do not edit --------------
9047 // ---------------------------------------------------------------------
9048
9049 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9050 match(Set prev (GetAndSetI mem newv));
9051 ins_cost(2 * VOLATILE_REF_COST);
9052 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9053 ins_encode %{
9054 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9055 %}
9056 ins_pipe(pipe_serial);
9057 %}
9058
9059 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9060 match(Set prev (GetAndSetL mem newv));
9061 ins_cost(2 * VOLATILE_REF_COST);
9062 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9063 ins_encode %{
9064 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9065 %}
9066 ins_pipe(pipe_serial);
9067 %}
9068
9069 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9070 predicate(n->as_LoadStore()->barrier_data() == 0);
9071 match(Set prev (GetAndSetN mem newv));
9072 ins_cost(2 * VOLATILE_REF_COST);
9073 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9074 ins_encode %{
9075 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9076 %}
9077 ins_pipe(pipe_serial);
9078 %}
9079
9080 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9081 predicate(n->as_LoadStore()->barrier_data() == 0);
9082 match(Set prev (GetAndSetP mem newv));
9083 ins_cost(2 * VOLATILE_REF_COST);
9084 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9085 ins_encode %{
9086 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9087 %}
9088 ins_pipe(pipe_serial);
9089 %}
9090
9091 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9092 predicate(needs_acquiring_load_exclusive(n));
9093 match(Set prev (GetAndSetI mem newv));
9094 ins_cost(VOLATILE_REF_COST);
9095 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9096 ins_encode %{
9097 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9098 %}
9099 ins_pipe(pipe_serial);
9100 %}
9101
9102 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9103 predicate(needs_acquiring_load_exclusive(n));
9104 match(Set prev (GetAndSetL mem newv));
9105 ins_cost(VOLATILE_REF_COST);
9106 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9107 ins_encode %{
9108 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9109 %}
9110 ins_pipe(pipe_serial);
9111 %}
9112
9113 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9114 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9115 match(Set prev (GetAndSetN mem newv));
9116 ins_cost(VOLATILE_REF_COST);
9117 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9118 ins_encode %{
9119 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9120 %}
9121 ins_pipe(pipe_serial);
9122 %}
9123
9124 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9125 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9126 match(Set prev (GetAndSetP mem newv));
9127 ins_cost(VOLATILE_REF_COST);
9128 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9129 ins_encode %{
9130 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9131 %}
9132 ins_pipe(pipe_serial);
9133 %}
9134
9135
9136 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9137 match(Set newval (GetAndAddL mem incr));
9138 ins_cost(2 * VOLATILE_REF_COST + 1);
9139 format %{ "get_and_addL $newval, [$mem], $incr" %}
9140 ins_encode %{
9141 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9142 %}
9143 ins_pipe(pipe_serial);
9144 %}
9145
9146 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9147 predicate(n->as_LoadStore()->result_not_used());
9148 match(Set dummy (GetAndAddL mem incr));
9149 ins_cost(2 * VOLATILE_REF_COST);
9150 format %{ "get_and_addL [$mem], $incr" %}
9151 ins_encode %{
9152 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9153 %}
9154 ins_pipe(pipe_serial);
9155 %}
9156
9157 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9158 match(Set newval (GetAndAddL mem incr));
9159 ins_cost(2 * VOLATILE_REF_COST + 1);
9160 format %{ "get_and_addL $newval, [$mem], $incr" %}
9161 ins_encode %{
9162 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9163 %}
9164 ins_pipe(pipe_serial);
9165 %}
9166
9167 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9168 predicate(n->as_LoadStore()->result_not_used());
9169 match(Set dummy (GetAndAddL mem incr));
9170 ins_cost(2 * VOLATILE_REF_COST);
9171 format %{ "get_and_addL [$mem], $incr" %}
9172 ins_encode %{
9173 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9174 %}
9175 ins_pipe(pipe_serial);
9176 %}
9177
9178 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9179 match(Set newval (GetAndAddI mem incr));
9180 ins_cost(2 * VOLATILE_REF_COST + 1);
9181 format %{ "get_and_addI $newval, [$mem], $incr" %}
9182 ins_encode %{
9183 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9184 %}
9185 ins_pipe(pipe_serial);
9186 %}
9187
9188 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9189 predicate(n->as_LoadStore()->result_not_used());
9190 match(Set dummy (GetAndAddI mem incr));
9191 ins_cost(2 * VOLATILE_REF_COST);
9192 format %{ "get_and_addI [$mem], $incr" %}
9193 ins_encode %{
9194 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9195 %}
9196 ins_pipe(pipe_serial);
9197 %}
9198
9199 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9200 match(Set newval (GetAndAddI mem incr));
9201 ins_cost(2 * VOLATILE_REF_COST + 1);
9202 format %{ "get_and_addI $newval, [$mem], $incr" %}
9203 ins_encode %{
9204 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9205 %}
9206 ins_pipe(pipe_serial);
9207 %}
9208
9209 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9210 predicate(n->as_LoadStore()->result_not_used());
9211 match(Set dummy (GetAndAddI mem incr));
9212 ins_cost(2 * VOLATILE_REF_COST);
9213 format %{ "get_and_addI [$mem], $incr" %}
9214 ins_encode %{
9215 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9216 %}
9217 ins_pipe(pipe_serial);
9218 %}
9219
9220 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9221 predicate(needs_acquiring_load_exclusive(n));
9222 match(Set newval (GetAndAddL mem incr));
9223 ins_cost(VOLATILE_REF_COST + 1);
9224 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9225 ins_encode %{
9226 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9227 %}
9228 ins_pipe(pipe_serial);
9229 %}
9230
9231 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9232 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9233 match(Set dummy (GetAndAddL mem incr));
9234 ins_cost(VOLATILE_REF_COST);
9235 format %{ "get_and_addL_acq [$mem], $incr" %}
9236 ins_encode %{
9237 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9238 %}
9239 ins_pipe(pipe_serial);
9240 %}
9241
9242 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9243 predicate(needs_acquiring_load_exclusive(n));
9244 match(Set newval (GetAndAddL mem incr));
9245 ins_cost(VOLATILE_REF_COST + 1);
9246 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9247 ins_encode %{
9248 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9249 %}
9250 ins_pipe(pipe_serial);
9251 %}
9252
9253 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9254 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9255 match(Set dummy (GetAndAddL mem incr));
9256 ins_cost(VOLATILE_REF_COST);
9257 format %{ "get_and_addL_acq [$mem], $incr" %}
9258 ins_encode %{
9259 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9260 %}
9261 ins_pipe(pipe_serial);
9262 %}
9263
9264 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9265 predicate(needs_acquiring_load_exclusive(n));
9266 match(Set newval (GetAndAddI mem incr));
9267 ins_cost(VOLATILE_REF_COST + 1);
9268 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9269 ins_encode %{
9270 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9271 %}
9272 ins_pipe(pipe_serial);
9273 %}
9274
9275 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9276 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9277 match(Set dummy (GetAndAddI mem incr));
9278 ins_cost(VOLATILE_REF_COST);
9279 format %{ "get_and_addI_acq [$mem], $incr" %}
9280 ins_encode %{
9281 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9282 %}
9283 ins_pipe(pipe_serial);
9284 %}
9285
9286 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9287 predicate(needs_acquiring_load_exclusive(n));
9288 match(Set newval (GetAndAddI mem incr));
9289 ins_cost(VOLATILE_REF_COST + 1);
9290 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9291 ins_encode %{
9292 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9293 %}
9294 ins_pipe(pipe_serial);
9295 %}
9296
9297 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9298 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9299 match(Set dummy (GetAndAddI mem incr));
9300 ins_cost(VOLATILE_REF_COST);
9301 format %{ "get_and_addI_acq [$mem], $incr" %}
9302 ins_encode %{
9303 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9304 %}
9305 ins_pipe(pipe_serial);
9306 %}
9307
9308 // Manifest a CmpU result in an integer register.
9309 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9310 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9311 %{
9312 match(Set dst (CmpU3 src1 src2));
9313 effect(KILL flags);
9314
9315 ins_cost(INSN_COST * 3);
9316 format %{
9317 "cmpw $src1, $src2\n\t"
9318 "csetw $dst, ne\n\t"
9319 "cnegw $dst, lo\t# CmpU3(reg)"
9320 %}
9321 ins_encode %{
9322 __ cmpw($src1$$Register, $src2$$Register);
9323 __ csetw($dst$$Register, Assembler::NE);
9324 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9325 %}
9326
9327 ins_pipe(pipe_class_default);
9328 %}
9329
9330 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9331 %{
9332 match(Set dst (CmpU3 src1 src2));
9333 effect(KILL flags);
9334
9335 ins_cost(INSN_COST * 3);
9336 format %{
9337 "subsw zr, $src1, $src2\n\t"
9338 "csetw $dst, ne\n\t"
9339 "cnegw $dst, lo\t# CmpU3(imm)"
9340 %}
9341 ins_encode %{
9342 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9343 __ csetw($dst$$Register, Assembler::NE);
9344 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9345 %}
9346
9347 ins_pipe(pipe_class_default);
9348 %}
9349
9350 // Manifest a CmpUL result in an integer register.
9351 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9352 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9353 %{
9354 match(Set dst (CmpUL3 src1 src2));
9355 effect(KILL flags);
9356
9357 ins_cost(INSN_COST * 3);
9358 format %{
9359 "cmp $src1, $src2\n\t"
9360 "csetw $dst, ne\n\t"
9361 "cnegw $dst, lo\t# CmpUL3(reg)"
9362 %}
9363 ins_encode %{
9364 __ cmp($src1$$Register, $src2$$Register);
9365 __ csetw($dst$$Register, Assembler::NE);
9366 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9367 %}
9368
9369 ins_pipe(pipe_class_default);
9370 %}
9371
9372 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9373 %{
9374 match(Set dst (CmpUL3 src1 src2));
9375 effect(KILL flags);
9376
9377 ins_cost(INSN_COST * 3);
9378 format %{
9379 "subs zr, $src1, $src2\n\t"
9380 "csetw $dst, ne\n\t"
9381 "cnegw $dst, lo\t# CmpUL3(imm)"
9382 %}
9383 ins_encode %{
9384 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9385 __ csetw($dst$$Register, Assembler::NE);
9386 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9387 %}
9388
9389 ins_pipe(pipe_class_default);
9390 %}
9391
9392 // Manifest a CmpL result in an integer register.
9393 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9394 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9395 %{
9396 match(Set dst (CmpL3 src1 src2));
9397 effect(KILL flags);
9398
9399 ins_cost(INSN_COST * 3);
9400 format %{
9401 "cmp $src1, $src2\n\t"
9402 "csetw $dst, ne\n\t"
9403 "cnegw $dst, lt\t# CmpL3(reg)"
9404 %}
9405 ins_encode %{
9406 __ cmp($src1$$Register, $src2$$Register);
9407 __ csetw($dst$$Register, Assembler::NE);
9408 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9409 %}
9410
9411 ins_pipe(pipe_class_default);
9412 %}
9413
9414 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9415 %{
9416 match(Set dst (CmpL3 src1 src2));
9417 effect(KILL flags);
9418
9419 ins_cost(INSN_COST * 3);
9420 format %{
9421 "subs zr, $src1, $src2\n\t"
9422 "csetw $dst, ne\n\t"
9423 "cnegw $dst, lt\t# CmpL3(imm)"
9424 %}
9425 ins_encode %{
9426 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9427 __ csetw($dst$$Register, Assembler::NE);
9428 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9429 %}
9430
9431 ins_pipe(pipe_class_default);
9432 %}
9433
9434 // ============================================================================
9435 // Conditional Move Instructions
9436
9437 // n.b. we have identical rules for both a signed compare op (cmpOp)
9438 // and an unsigned compare op (cmpOpU). it would be nice if we could
9439 // define an op class which merged both inputs and use it to type the
9440 // argument to a single rule. unfortunatelyt his fails because the
9441 // opclass does not live up to the COND_INTER interface of its
9442 // component operands. When the generic code tries to negate the
9443 // operand it ends up running the generci Machoper::negate method
9444 // which throws a ShouldNotHappen. So, we have to provide two flavours
9445 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9446
9447 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9448 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9449
9450 ins_cost(INSN_COST * 2);
9451 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9452
9453 ins_encode %{
9454 __ cselw(as_Register($dst$$reg),
9455 as_Register($src2$$reg),
9456 as_Register($src1$$reg),
9457 (Assembler::Condition)$cmp$$cmpcode);
9458 %}
9459
9460 ins_pipe(icond_reg_reg);
9461 %}
9462
9463 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9464 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9465
9466 ins_cost(INSN_COST * 2);
9467 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9468
9469 ins_encode %{
9470 __ cselw(as_Register($dst$$reg),
9471 as_Register($src2$$reg),
9472 as_Register($src1$$reg),
9473 (Assembler::Condition)$cmp$$cmpcode);
9474 %}
9475
9476 ins_pipe(icond_reg_reg);
9477 %}
9478
9479 // special cases where one arg is zero
9480
9481 // n.b. this is selected in preference to the rule above because it
9482 // avoids loading constant 0 into a source register
9483
9484 // TODO
9485 // we ought only to be able to cull one of these variants as the ideal
9486 // transforms ought always to order the zero consistently (to left/right?)
9487
9488 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9489 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9490
9491 ins_cost(INSN_COST * 2);
9492 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9493
9494 ins_encode %{
9495 __ cselw(as_Register($dst$$reg),
9496 as_Register($src$$reg),
9497 zr,
9498 (Assembler::Condition)$cmp$$cmpcode);
9499 %}
9500
9501 ins_pipe(icond_reg);
9502 %}
9503
9504 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9505 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9506
9507 ins_cost(INSN_COST * 2);
9508 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9509
9510 ins_encode %{
9511 __ cselw(as_Register($dst$$reg),
9512 as_Register($src$$reg),
9513 zr,
9514 (Assembler::Condition)$cmp$$cmpcode);
9515 %}
9516
9517 ins_pipe(icond_reg);
9518 %}
9519
9520 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9521 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9522
9523 ins_cost(INSN_COST * 2);
9524 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9525
9526 ins_encode %{
9527 __ cselw(as_Register($dst$$reg),
9528 zr,
9529 as_Register($src$$reg),
9530 (Assembler::Condition)$cmp$$cmpcode);
9531 %}
9532
9533 ins_pipe(icond_reg);
9534 %}
9535
9536 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9537 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9538
9539 ins_cost(INSN_COST * 2);
9540 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9541
9542 ins_encode %{
9543 __ cselw(as_Register($dst$$reg),
9544 zr,
9545 as_Register($src$$reg),
9546 (Assembler::Condition)$cmp$$cmpcode);
9547 %}
9548
9549 ins_pipe(icond_reg);
9550 %}
9551
9552 // special case for creating a boolean 0 or 1
9553
9554 // n.b. this is selected in preference to the rule above because it
9555 // avoids loading constants 0 and 1 into a source register
9556
9557 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9558 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9559
9560 ins_cost(INSN_COST * 2);
9561 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9562
9563 ins_encode %{
9564 // equivalently
9565 // cset(as_Register($dst$$reg),
9566 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9567 __ csincw(as_Register($dst$$reg),
9568 zr,
9569 zr,
9570 (Assembler::Condition)$cmp$$cmpcode);
9571 %}
9572
9573 ins_pipe(icond_none);
9574 %}
9575
9576 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9577 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9578
9579 ins_cost(INSN_COST * 2);
9580 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9581
9582 ins_encode %{
9583 // equivalently
9584 // cset(as_Register($dst$$reg),
9585 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9586 __ csincw(as_Register($dst$$reg),
9587 zr,
9588 zr,
9589 (Assembler::Condition)$cmp$$cmpcode);
9590 %}
9591
9592 ins_pipe(icond_none);
9593 %}
9594
9595 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9596 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9597
9598 ins_cost(INSN_COST * 2);
9599 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9600
9601 ins_encode %{
9602 __ csel(as_Register($dst$$reg),
9603 as_Register($src2$$reg),
9604 as_Register($src1$$reg),
9605 (Assembler::Condition)$cmp$$cmpcode);
9606 %}
9607
9608 ins_pipe(icond_reg_reg);
9609 %}
9610
9611 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9612 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9613
9614 ins_cost(INSN_COST * 2);
9615 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9616
9617 ins_encode %{
9618 __ csel(as_Register($dst$$reg),
9619 as_Register($src2$$reg),
9620 as_Register($src1$$reg),
9621 (Assembler::Condition)$cmp$$cmpcode);
9622 %}
9623
9624 ins_pipe(icond_reg_reg);
9625 %}
9626
9627 // special cases where one arg is zero
9628
9629 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9630 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9631
9632 ins_cost(INSN_COST * 2);
9633 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9634
9635 ins_encode %{
9636 __ csel(as_Register($dst$$reg),
9637 zr,
9638 as_Register($src$$reg),
9639 (Assembler::Condition)$cmp$$cmpcode);
9640 %}
9641
9642 ins_pipe(icond_reg);
9643 %}
9644
9645 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9646 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9647
9648 ins_cost(INSN_COST * 2);
9649 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9650
9651 ins_encode %{
9652 __ csel(as_Register($dst$$reg),
9653 zr,
9654 as_Register($src$$reg),
9655 (Assembler::Condition)$cmp$$cmpcode);
9656 %}
9657
9658 ins_pipe(icond_reg);
9659 %}
9660
9661 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9662 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9663
9664 ins_cost(INSN_COST * 2);
9665 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9666
9667 ins_encode %{
9668 __ csel(as_Register($dst$$reg),
9669 as_Register($src$$reg),
9670 zr,
9671 (Assembler::Condition)$cmp$$cmpcode);
9672 %}
9673
9674 ins_pipe(icond_reg);
9675 %}
9676
9677 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9678 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9679
9680 ins_cost(INSN_COST * 2);
9681 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9682
9683 ins_encode %{
9684 __ csel(as_Register($dst$$reg),
9685 as_Register($src$$reg),
9686 zr,
9687 (Assembler::Condition)$cmp$$cmpcode);
9688 %}
9689
9690 ins_pipe(icond_reg);
9691 %}
9692
9693 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9694 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9695
9696 ins_cost(INSN_COST * 2);
9697 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9698
9699 ins_encode %{
9700 __ csel(as_Register($dst$$reg),
9701 as_Register($src2$$reg),
9702 as_Register($src1$$reg),
9703 (Assembler::Condition)$cmp$$cmpcode);
9704 %}
9705
9706 ins_pipe(icond_reg_reg);
9707 %}
9708
9709 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9710 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9711
9712 ins_cost(INSN_COST * 2);
9713 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9714
9715 ins_encode %{
9716 __ csel(as_Register($dst$$reg),
9717 as_Register($src2$$reg),
9718 as_Register($src1$$reg),
9719 (Assembler::Condition)$cmp$$cmpcode);
9720 %}
9721
9722 ins_pipe(icond_reg_reg);
9723 %}
9724
9725 // special cases where one arg is zero
9726
9727 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9728 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9729
9730 ins_cost(INSN_COST * 2);
9731 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9732
9733 ins_encode %{
9734 __ csel(as_Register($dst$$reg),
9735 zr,
9736 as_Register($src$$reg),
9737 (Assembler::Condition)$cmp$$cmpcode);
9738 %}
9739
9740 ins_pipe(icond_reg);
9741 %}
9742
9743 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9744 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9745
9746 ins_cost(INSN_COST * 2);
9747 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9748
9749 ins_encode %{
9750 __ csel(as_Register($dst$$reg),
9751 zr,
9752 as_Register($src$$reg),
9753 (Assembler::Condition)$cmp$$cmpcode);
9754 %}
9755
9756 ins_pipe(icond_reg);
9757 %}
9758
9759 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9760 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9761
9762 ins_cost(INSN_COST * 2);
9763 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9764
9765 ins_encode %{
9766 __ csel(as_Register($dst$$reg),
9767 as_Register($src$$reg),
9768 zr,
9769 (Assembler::Condition)$cmp$$cmpcode);
9770 %}
9771
9772 ins_pipe(icond_reg);
9773 %}
9774
9775 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9776 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9777
9778 ins_cost(INSN_COST * 2);
9779 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9780
9781 ins_encode %{
9782 __ csel(as_Register($dst$$reg),
9783 as_Register($src$$reg),
9784 zr,
9785 (Assembler::Condition)$cmp$$cmpcode);
9786 %}
9787
9788 ins_pipe(icond_reg);
9789 %}
9790
9791 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9792 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9793
9794 ins_cost(INSN_COST * 2);
9795 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9796
9797 ins_encode %{
9798 __ cselw(as_Register($dst$$reg),
9799 as_Register($src2$$reg),
9800 as_Register($src1$$reg),
9801 (Assembler::Condition)$cmp$$cmpcode);
9802 %}
9803
9804 ins_pipe(icond_reg_reg);
9805 %}
9806
9807 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9808 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9809
9810 ins_cost(INSN_COST * 2);
9811 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9812
9813 ins_encode %{
9814 __ cselw(as_Register($dst$$reg),
9815 as_Register($src2$$reg),
9816 as_Register($src1$$reg),
9817 (Assembler::Condition)$cmp$$cmpcode);
9818 %}
9819
9820 ins_pipe(icond_reg_reg);
9821 %}
9822
9823 // special cases where one arg is zero
9824
9825 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9826 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9827
9828 ins_cost(INSN_COST * 2);
9829 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9830
9831 ins_encode %{
9832 __ cselw(as_Register($dst$$reg),
9833 zr,
9834 as_Register($src$$reg),
9835 (Assembler::Condition)$cmp$$cmpcode);
9836 %}
9837
9838 ins_pipe(icond_reg);
9839 %}
9840
9841 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9842 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9843
9844 ins_cost(INSN_COST * 2);
9845 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9846
9847 ins_encode %{
9848 __ cselw(as_Register($dst$$reg),
9849 zr,
9850 as_Register($src$$reg),
9851 (Assembler::Condition)$cmp$$cmpcode);
9852 %}
9853
9854 ins_pipe(icond_reg);
9855 %}
9856
9857 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9858 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9859
9860 ins_cost(INSN_COST * 2);
9861 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9862
9863 ins_encode %{
9864 __ cselw(as_Register($dst$$reg),
9865 as_Register($src$$reg),
9866 zr,
9867 (Assembler::Condition)$cmp$$cmpcode);
9868 %}
9869
9870 ins_pipe(icond_reg);
9871 %}
9872
9873 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9874 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9875
9876 ins_cost(INSN_COST * 2);
9877 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9878
9879 ins_encode %{
9880 __ cselw(as_Register($dst$$reg),
9881 as_Register($src$$reg),
9882 zr,
9883 (Assembler::Condition)$cmp$$cmpcode);
9884 %}
9885
9886 ins_pipe(icond_reg);
9887 %}
9888
9889 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9890 %{
9891 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9892
9893 ins_cost(INSN_COST * 3);
9894
9895 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9896 ins_encode %{
9897 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9898 __ fcsels(as_FloatRegister($dst$$reg),
9899 as_FloatRegister($src2$$reg),
9900 as_FloatRegister($src1$$reg),
9901 cond);
9902 %}
9903
9904 ins_pipe(fp_cond_reg_reg_s);
9905 %}
9906
9907 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9908 %{
9909 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9910
9911 ins_cost(INSN_COST * 3);
9912
9913 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9914 ins_encode %{
9915 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9916 __ fcsels(as_FloatRegister($dst$$reg),
9917 as_FloatRegister($src2$$reg),
9918 as_FloatRegister($src1$$reg),
9919 cond);
9920 %}
9921
9922 ins_pipe(fp_cond_reg_reg_s);
9923 %}
9924
9925 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9926 %{
9927 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9928
9929 ins_cost(INSN_COST * 3);
9930
9931 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9932 ins_encode %{
9933 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9934 __ fcseld(as_FloatRegister($dst$$reg),
9935 as_FloatRegister($src2$$reg),
9936 as_FloatRegister($src1$$reg),
9937 cond);
9938 %}
9939
9940 ins_pipe(fp_cond_reg_reg_d);
9941 %}
9942
9943 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9944 %{
9945 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9946
9947 ins_cost(INSN_COST * 3);
9948
9949 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9950 ins_encode %{
9951 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9952 __ fcseld(as_FloatRegister($dst$$reg),
9953 as_FloatRegister($src2$$reg),
9954 as_FloatRegister($src1$$reg),
9955 cond);
9956 %}
9957
9958 ins_pipe(fp_cond_reg_reg_d);
9959 %}
9960
9961 // ============================================================================
9962 // Arithmetic Instructions
9963 //
9964
9965 // Integer Addition
9966
9967 // TODO
9968 // these currently employ operations which do not set CR and hence are
9969 // not flagged as killing CR but we would like to isolate the cases
9970 // where we want to set flags from those where we don't. need to work
9971 // out how to do that.
9972
9973 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9974 match(Set dst (AddI src1 src2));
9975
9976 ins_cost(INSN_COST);
9977 format %{ "addw $dst, $src1, $src2" %}
9978
9979 ins_encode %{
9980 __ addw(as_Register($dst$$reg),
9981 as_Register($src1$$reg),
9982 as_Register($src2$$reg));
9983 %}
9984
9985 ins_pipe(ialu_reg_reg);
9986 %}
9987
9988 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9989 match(Set dst (AddI src1 src2));
9990
9991 ins_cost(INSN_COST);
9992 format %{ "addw $dst, $src1, $src2" %}
9993
9994 // use opcode to indicate that this is an add not a sub
9995 opcode(0x0);
9996
9997 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9998
9999 ins_pipe(ialu_reg_imm);
10000 %}
10001
10002 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10003 match(Set dst (AddI (ConvL2I src1) src2));
10004
10005 ins_cost(INSN_COST);
10006 format %{ "addw $dst, $src1, $src2" %}
10007
10008 // use opcode to indicate that this is an add not a sub
10009 opcode(0x0);
10010
10011 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10012
10013 ins_pipe(ialu_reg_imm);
10014 %}
10015
10016 // Pointer Addition
10017 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
10018 match(Set dst (AddP src1 src2));
10019
10020 ins_cost(INSN_COST);
10021 format %{ "add $dst, $src1, $src2\t# ptr" %}
10022
10023 ins_encode %{
10024 __ add(as_Register($dst$$reg),
10025 as_Register($src1$$reg),
10026 as_Register($src2$$reg));
10027 %}
10028
10029 ins_pipe(ialu_reg_reg);
10030 %}
10031
10032 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
10033 match(Set dst (AddP src1 (ConvI2L src2)));
10034
10035 ins_cost(1.9 * INSN_COST);
10036 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10037
10038 ins_encode %{
10039 __ add(as_Register($dst$$reg),
10040 as_Register($src1$$reg),
10041 as_Register($src2$$reg), ext::sxtw);
10042 %}
10043
10044 ins_pipe(ialu_reg_reg);
10045 %}
10046
10047 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10048 match(Set dst (AddP src1 (LShiftL src2 scale)));
10049
10050 ins_cost(1.9 * INSN_COST);
10051 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10052
10053 ins_encode %{
10054 __ lea(as_Register($dst$$reg),
10055 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10056 Address::lsl($scale$$constant)));
10057 %}
10058
10059 ins_pipe(ialu_reg_reg_shift);
10060 %}
10061
10062 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10063 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10064
10065 ins_cost(1.9 * INSN_COST);
10066 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10067
10068 ins_encode %{
10069 __ lea(as_Register($dst$$reg),
10070 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10071 Address::sxtw($scale$$constant)));
10072 %}
10073
10074 ins_pipe(ialu_reg_reg_shift);
10075 %}
10076
10077 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10078 match(Set dst (LShiftL (ConvI2L src) scale));
10079
10080 ins_cost(INSN_COST);
10081 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10082
10083 ins_encode %{
10084 __ sbfiz(as_Register($dst$$reg),
10085 as_Register($src$$reg),
10086 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10087 %}
10088
10089 ins_pipe(ialu_reg_shift);
10090 %}
10091
10092 // Pointer Immediate Addition
10093 // n.b. this needs to be more expensive than using an indirect memory
10094 // operand
10095 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10096 match(Set dst (AddP src1 src2));
10097
10098 ins_cost(INSN_COST);
10099 format %{ "add $dst, $src1, $src2\t# ptr" %}
10100
10101 // use opcode to indicate that this is an add not a sub
10102 opcode(0x0);
10103
10104 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10105
10106 ins_pipe(ialu_reg_imm);
10107 %}
10108
10109 // Long Addition
10110 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10111
10112 match(Set dst (AddL src1 src2));
10113
10114 ins_cost(INSN_COST);
10115 format %{ "add $dst, $src1, $src2" %}
10116
10117 ins_encode %{
10118 __ add(as_Register($dst$$reg),
10119 as_Register($src1$$reg),
10120 as_Register($src2$$reg));
10121 %}
10122
10123 ins_pipe(ialu_reg_reg);
10124 %}
10125
10126 // No constant pool entries requiredLong Immediate Addition.
10127 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10128 match(Set dst (AddL src1 src2));
10129
10130 ins_cost(INSN_COST);
10131 format %{ "add $dst, $src1, $src2" %}
10132
10133 // use opcode to indicate that this is an add not a sub
10134 opcode(0x0);
10135
10136 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10137
10138 ins_pipe(ialu_reg_imm);
10139 %}
10140
10141 // Integer Subtraction
10142 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10143 match(Set dst (SubI src1 src2));
10144
10145 ins_cost(INSN_COST);
10146 format %{ "subw $dst, $src1, $src2" %}
10147
10148 ins_encode %{
10149 __ subw(as_Register($dst$$reg),
10150 as_Register($src1$$reg),
10151 as_Register($src2$$reg));
10152 %}
10153
10154 ins_pipe(ialu_reg_reg);
10155 %}
10156
10157 // Immediate Subtraction
10158 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10159 match(Set dst (SubI src1 src2));
10160
10161 ins_cost(INSN_COST);
10162 format %{ "subw $dst, $src1, $src2" %}
10163
10164 // use opcode to indicate that this is a sub not an add
10165 opcode(0x1);
10166
10167 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10168
10169 ins_pipe(ialu_reg_imm);
10170 %}
10171
10172 // Long Subtraction
10173 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10174
10175 match(Set dst (SubL src1 src2));
10176
10177 ins_cost(INSN_COST);
10178 format %{ "sub $dst, $src1, $src2" %}
10179
10180 ins_encode %{
10181 __ sub(as_Register($dst$$reg),
10182 as_Register($src1$$reg),
10183 as_Register($src2$$reg));
10184 %}
10185
10186 ins_pipe(ialu_reg_reg);
10187 %}
10188
10189 // No constant pool entries requiredLong Immediate Subtraction.
10190 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10191 match(Set dst (SubL src1 src2));
10192
10193 ins_cost(INSN_COST);
10194 format %{ "sub$dst, $src1, $src2" %}
10195
10196 // use opcode to indicate that this is a sub not an add
10197 opcode(0x1);
10198
10199 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10200
10201 ins_pipe(ialu_reg_imm);
10202 %}
10203
10204 // Integer Negation (special case for sub)
10205
10206 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10207 match(Set dst (SubI zero src));
10208
10209 ins_cost(INSN_COST);
10210 format %{ "negw $dst, $src\t# int" %}
10211
10212 ins_encode %{
10213 __ negw(as_Register($dst$$reg),
10214 as_Register($src$$reg));
10215 %}
10216
10217 ins_pipe(ialu_reg);
10218 %}
10219
10220 // Long Negation
10221
10222 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10223 match(Set dst (SubL zero src));
10224
10225 ins_cost(INSN_COST);
10226 format %{ "neg $dst, $src\t# long" %}
10227
10228 ins_encode %{
10229 __ neg(as_Register($dst$$reg),
10230 as_Register($src$$reg));
10231 %}
10232
10233 ins_pipe(ialu_reg);
10234 %}
10235
10236 // Integer Multiply
10237
10238 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10239 match(Set dst (MulI src1 src2));
10240
10241 ins_cost(INSN_COST * 3);
10242 format %{ "mulw $dst, $src1, $src2" %}
10243
10244 ins_encode %{
10245 __ mulw(as_Register($dst$$reg),
10246 as_Register($src1$$reg),
10247 as_Register($src2$$reg));
10248 %}
10249
10250 ins_pipe(imul_reg_reg);
10251 %}
10252
10253 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10254 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10255
10256 ins_cost(INSN_COST * 3);
10257 format %{ "smull $dst, $src1, $src2" %}
10258
10259 ins_encode %{
10260 __ smull(as_Register($dst$$reg),
10261 as_Register($src1$$reg),
10262 as_Register($src2$$reg));
10263 %}
10264
10265 ins_pipe(imul_reg_reg);
10266 %}
10267
10268 // Long Multiply
10269
10270 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10271 match(Set dst (MulL src1 src2));
10272
10273 ins_cost(INSN_COST * 5);
10274 format %{ "mul $dst, $src1, $src2" %}
10275
10276 ins_encode %{
10277 __ mul(as_Register($dst$$reg),
10278 as_Register($src1$$reg),
10279 as_Register($src2$$reg));
10280 %}
10281
10282 ins_pipe(lmul_reg_reg);
10283 %}
10284
10285 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10286 %{
10287 match(Set dst (MulHiL src1 src2));
10288
10289 ins_cost(INSN_COST * 7);
10290 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10291
10292 ins_encode %{
10293 __ smulh(as_Register($dst$$reg),
10294 as_Register($src1$$reg),
10295 as_Register($src2$$reg));
10296 %}
10297
10298 ins_pipe(lmul_reg_reg);
10299 %}
10300
10301 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10302 %{
10303 match(Set dst (UMulHiL src1 src2));
10304
10305 ins_cost(INSN_COST * 7);
10306 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10307
10308 ins_encode %{
10309 __ umulh(as_Register($dst$$reg),
10310 as_Register($src1$$reg),
10311 as_Register($src2$$reg));
10312 %}
10313
10314 ins_pipe(lmul_reg_reg);
10315 %}
10316
10317 // Combined Integer Multiply & Add/Sub
10318
10319 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10320 match(Set dst (AddI src3 (MulI src1 src2)));
10321
10322 ins_cost(INSN_COST * 3);
10323 format %{ "madd $dst, $src1, $src2, $src3" %}
10324
10325 ins_encode %{
10326 __ maddw(as_Register($dst$$reg),
10327 as_Register($src1$$reg),
10328 as_Register($src2$$reg),
10329 as_Register($src3$$reg));
10330 %}
10331
10332 ins_pipe(imac_reg_reg);
10333 %}
10334
10335 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10336 match(Set dst (SubI src3 (MulI src1 src2)));
10337
10338 ins_cost(INSN_COST * 3);
10339 format %{ "msub $dst, $src1, $src2, $src3" %}
10340
10341 ins_encode %{
10342 __ msubw(as_Register($dst$$reg),
10343 as_Register($src1$$reg),
10344 as_Register($src2$$reg),
10345 as_Register($src3$$reg));
10346 %}
10347
10348 ins_pipe(imac_reg_reg);
10349 %}
10350
10351 // Combined Integer Multiply & Neg
10352
10353 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10354 match(Set dst (MulI (SubI zero src1) src2));
10355
10356 ins_cost(INSN_COST * 3);
10357 format %{ "mneg $dst, $src1, $src2" %}
10358
10359 ins_encode %{
10360 __ mnegw(as_Register($dst$$reg),
10361 as_Register($src1$$reg),
10362 as_Register($src2$$reg));
10363 %}
10364
10365 ins_pipe(imac_reg_reg);
10366 %}
10367
10368 // Combined Long Multiply & Add/Sub
10369
10370 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10371 match(Set dst (AddL src3 (MulL src1 src2)));
10372
10373 ins_cost(INSN_COST * 5);
10374 format %{ "madd $dst, $src1, $src2, $src3" %}
10375
10376 ins_encode %{
10377 __ madd(as_Register($dst$$reg),
10378 as_Register($src1$$reg),
10379 as_Register($src2$$reg),
10380 as_Register($src3$$reg));
10381 %}
10382
10383 ins_pipe(lmac_reg_reg);
10384 %}
10385
10386 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10387 match(Set dst (SubL src3 (MulL src1 src2)));
10388
10389 ins_cost(INSN_COST * 5);
10390 format %{ "msub $dst, $src1, $src2, $src3" %}
10391
10392 ins_encode %{
10393 __ msub(as_Register($dst$$reg),
10394 as_Register($src1$$reg),
10395 as_Register($src2$$reg),
10396 as_Register($src3$$reg));
10397 %}
10398
10399 ins_pipe(lmac_reg_reg);
10400 %}
10401
10402 // Combined Long Multiply & Neg
10403
10404 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10405 match(Set dst (MulL (SubL zero src1) src2));
10406
10407 ins_cost(INSN_COST * 5);
10408 format %{ "mneg $dst, $src1, $src2" %}
10409
10410 ins_encode %{
10411 __ mneg(as_Register($dst$$reg),
10412 as_Register($src1$$reg),
10413 as_Register($src2$$reg));
10414 %}
10415
10416 ins_pipe(lmac_reg_reg);
10417 %}
10418
10419 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10420
10421 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10422 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10423
10424 ins_cost(INSN_COST * 3);
10425 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10426
10427 ins_encode %{
10428 __ smaddl(as_Register($dst$$reg),
10429 as_Register($src1$$reg),
10430 as_Register($src2$$reg),
10431 as_Register($src3$$reg));
10432 %}
10433
10434 ins_pipe(imac_reg_reg);
10435 %}
10436
10437 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10438 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10439
10440 ins_cost(INSN_COST * 3);
10441 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10442
10443 ins_encode %{
10444 __ smsubl(as_Register($dst$$reg),
10445 as_Register($src1$$reg),
10446 as_Register($src2$$reg),
10447 as_Register($src3$$reg));
10448 %}
10449
10450 ins_pipe(imac_reg_reg);
10451 %}
10452
10453 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10454 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10455
10456 ins_cost(INSN_COST * 3);
10457 format %{ "smnegl $dst, $src1, $src2" %}
10458
10459 ins_encode %{
10460 __ smnegl(as_Register($dst$$reg),
10461 as_Register($src1$$reg),
10462 as_Register($src2$$reg));
10463 %}
10464
10465 ins_pipe(imac_reg_reg);
10466 %}
10467
10468 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10469
10470 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10471 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10472
10473 ins_cost(INSN_COST * 5);
10474 format %{ "mulw rscratch1, $src1, $src2\n\t"
10475 "maddw $dst, $src3, $src4, rscratch1" %}
10476
10477 ins_encode %{
10478 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10479 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10480
10481 ins_pipe(imac_reg_reg);
10482 %}
10483
10484 // Integer Divide
10485
10486 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10487 match(Set dst (DivI src1 src2));
10488
10489 ins_cost(INSN_COST * 19);
10490 format %{ "sdivw $dst, $src1, $src2" %}
10491
10492 ins_encode(aarch64_enc_divw(dst, src1, src2));
10493 ins_pipe(idiv_reg_reg);
10494 %}
10495
10496 // Long Divide
10497
10498 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10499 match(Set dst (DivL src1 src2));
10500
10501 ins_cost(INSN_COST * 35);
10502 format %{ "sdiv $dst, $src1, $src2" %}
10503
10504 ins_encode(aarch64_enc_div(dst, src1, src2));
10505 ins_pipe(ldiv_reg_reg);
10506 %}
10507
10508 // Integer Remainder
10509
10510 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10511 match(Set dst (ModI src1 src2));
10512
10513 ins_cost(INSN_COST * 22);
10514 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10515 "msubw $dst, rscratch1, $src2, $src1" %}
10516
10517 ins_encode(aarch64_enc_modw(dst, src1, src2));
10518 ins_pipe(idiv_reg_reg);
10519 %}
10520
10521 // Long Remainder
10522
10523 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10524 match(Set dst (ModL src1 src2));
10525
10526 ins_cost(INSN_COST * 38);
10527 format %{ "sdiv rscratch1, $src1, $src2\n"
10528 "msub $dst, rscratch1, $src2, $src1" %}
10529
10530 ins_encode(aarch64_enc_mod(dst, src1, src2));
10531 ins_pipe(ldiv_reg_reg);
10532 %}
10533
10534 // Unsigned Integer Divide
10535
10536 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10537 match(Set dst (UDivI src1 src2));
10538
10539 ins_cost(INSN_COST * 19);
10540 format %{ "udivw $dst, $src1, $src2" %}
10541
10542 ins_encode %{
10543 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10544 %}
10545
10546 ins_pipe(idiv_reg_reg);
10547 %}
10548
10549 // Unsigned Long Divide
10550
10551 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10552 match(Set dst (UDivL src1 src2));
10553
10554 ins_cost(INSN_COST * 35);
10555 format %{ "udiv $dst, $src1, $src2" %}
10556
10557 ins_encode %{
10558 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10559 %}
10560
10561 ins_pipe(ldiv_reg_reg);
10562 %}
10563
10564 // Unsigned Integer Remainder
10565
10566 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10567 match(Set dst (UModI src1 src2));
10568
10569 ins_cost(INSN_COST * 22);
10570 format %{ "udivw rscratch1, $src1, $src2\n\t"
10571 "msubw $dst, rscratch1, $src2, $src1" %}
10572
10573 ins_encode %{
10574 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10575 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10576 %}
10577
10578 ins_pipe(idiv_reg_reg);
10579 %}
10580
10581 // Unsigned Long Remainder
10582
10583 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10584 match(Set dst (UModL src1 src2));
10585
10586 ins_cost(INSN_COST * 38);
10587 format %{ "udiv rscratch1, $src1, $src2\n"
10588 "msub $dst, rscratch1, $src2, $src1" %}
10589
10590 ins_encode %{
10591 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10592 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10593 %}
10594
10595 ins_pipe(ldiv_reg_reg);
10596 %}
10597
10598 // Integer Shifts
10599
10600 // Shift Left Register
10601 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10602 match(Set dst (LShiftI src1 src2));
10603
10604 ins_cost(INSN_COST * 2);
10605 format %{ "lslvw $dst, $src1, $src2" %}
10606
10607 ins_encode %{
10608 __ lslvw(as_Register($dst$$reg),
10609 as_Register($src1$$reg),
10610 as_Register($src2$$reg));
10611 %}
10612
10613 ins_pipe(ialu_reg_reg_vshift);
10614 %}
10615
10616 // Shift Left Immediate
10617 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10618 match(Set dst (LShiftI src1 src2));
10619
10620 ins_cost(INSN_COST);
10621 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10622
10623 ins_encode %{
10624 __ lslw(as_Register($dst$$reg),
10625 as_Register($src1$$reg),
10626 $src2$$constant & 0x1f);
10627 %}
10628
10629 ins_pipe(ialu_reg_shift);
10630 %}
10631
10632 // Shift Right Logical Register
10633 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10634 match(Set dst (URShiftI src1 src2));
10635
10636 ins_cost(INSN_COST * 2);
10637 format %{ "lsrvw $dst, $src1, $src2" %}
10638
10639 ins_encode %{
10640 __ lsrvw(as_Register($dst$$reg),
10641 as_Register($src1$$reg),
10642 as_Register($src2$$reg));
10643 %}
10644
10645 ins_pipe(ialu_reg_reg_vshift);
10646 %}
10647
10648 // Shift Right Logical Immediate
10649 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10650 match(Set dst (URShiftI src1 src2));
10651
10652 ins_cost(INSN_COST);
10653 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10654
10655 ins_encode %{
10656 __ lsrw(as_Register($dst$$reg),
10657 as_Register($src1$$reg),
10658 $src2$$constant & 0x1f);
10659 %}
10660
10661 ins_pipe(ialu_reg_shift);
10662 %}
10663
10664 // Shift Right Arithmetic Register
10665 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10666 match(Set dst (RShiftI src1 src2));
10667
10668 ins_cost(INSN_COST * 2);
10669 format %{ "asrvw $dst, $src1, $src2" %}
10670
10671 ins_encode %{
10672 __ asrvw(as_Register($dst$$reg),
10673 as_Register($src1$$reg),
10674 as_Register($src2$$reg));
10675 %}
10676
10677 ins_pipe(ialu_reg_reg_vshift);
10678 %}
10679
10680 // Shift Right Arithmetic Immediate
10681 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10682 match(Set dst (RShiftI src1 src2));
10683
10684 ins_cost(INSN_COST);
10685 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10686
10687 ins_encode %{
10688 __ asrw(as_Register($dst$$reg),
10689 as_Register($src1$$reg),
10690 $src2$$constant & 0x1f);
10691 %}
10692
10693 ins_pipe(ialu_reg_shift);
10694 %}
10695
10696 // Combined Int Mask and Right Shift (using UBFM)
10697 // TODO
10698
10699 // Long Shifts
10700
10701 // Shift Left Register
10702 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10703 match(Set dst (LShiftL src1 src2));
10704
10705 ins_cost(INSN_COST * 2);
10706 format %{ "lslv $dst, $src1, $src2" %}
10707
10708 ins_encode %{
10709 __ lslv(as_Register($dst$$reg),
10710 as_Register($src1$$reg),
10711 as_Register($src2$$reg));
10712 %}
10713
10714 ins_pipe(ialu_reg_reg_vshift);
10715 %}
10716
10717 // Shift Left Immediate
10718 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10719 match(Set dst (LShiftL src1 src2));
10720
10721 ins_cost(INSN_COST);
10722 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10723
10724 ins_encode %{
10725 __ lsl(as_Register($dst$$reg),
10726 as_Register($src1$$reg),
10727 $src2$$constant & 0x3f);
10728 %}
10729
10730 ins_pipe(ialu_reg_shift);
10731 %}
10732
10733 // Shift Right Logical Register
10734 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10735 match(Set dst (URShiftL src1 src2));
10736
10737 ins_cost(INSN_COST * 2);
10738 format %{ "lsrv $dst, $src1, $src2" %}
10739
10740 ins_encode %{
10741 __ lsrv(as_Register($dst$$reg),
10742 as_Register($src1$$reg),
10743 as_Register($src2$$reg));
10744 %}
10745
10746 ins_pipe(ialu_reg_reg_vshift);
10747 %}
10748
10749 // Shift Right Logical Immediate
10750 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10751 match(Set dst (URShiftL src1 src2));
10752
10753 ins_cost(INSN_COST);
10754 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10755
10756 ins_encode %{
10757 __ lsr(as_Register($dst$$reg),
10758 as_Register($src1$$reg),
10759 $src2$$constant & 0x3f);
10760 %}
10761
10762 ins_pipe(ialu_reg_shift);
10763 %}
10764
10765 // A special-case pattern for card table stores.
10766 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10767 match(Set dst (URShiftL (CastP2X src1) src2));
10768
10769 ins_cost(INSN_COST);
10770 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10771
10772 ins_encode %{
10773 __ lsr(as_Register($dst$$reg),
10774 as_Register($src1$$reg),
10775 $src2$$constant & 0x3f);
10776 %}
10777
10778 ins_pipe(ialu_reg_shift);
10779 %}
10780
10781 // Shift Right Arithmetic Register
10782 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10783 match(Set dst (RShiftL src1 src2));
10784
10785 ins_cost(INSN_COST * 2);
10786 format %{ "asrv $dst, $src1, $src2" %}
10787
10788 ins_encode %{
10789 __ asrv(as_Register($dst$$reg),
10790 as_Register($src1$$reg),
10791 as_Register($src2$$reg));
10792 %}
10793
10794 ins_pipe(ialu_reg_reg_vshift);
10795 %}
10796
10797 // Shift Right Arithmetic Immediate
10798 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10799 match(Set dst (RShiftL src1 src2));
10800
10801 ins_cost(INSN_COST);
10802 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10803
10804 ins_encode %{
10805 __ asr(as_Register($dst$$reg),
10806 as_Register($src1$$reg),
10807 $src2$$constant & 0x3f);
10808 %}
10809
10810 ins_pipe(ialu_reg_shift);
10811 %}
10812
10813 // BEGIN This section of the file is automatically generated. Do not edit --------------
10814 // This section is generated from aarch64_ad.m4
10815
10816 // This pattern is automatically generated from aarch64_ad.m4
10817 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10818 instruct regL_not_reg(iRegLNoSp dst,
10819 iRegL src1, immL_M1 m1,
10820 rFlagsReg cr) %{
10821 match(Set dst (XorL src1 m1));
10822 ins_cost(INSN_COST);
10823 format %{ "eon $dst, $src1, zr" %}
10824
10825 ins_encode %{
10826 __ eon(as_Register($dst$$reg),
10827 as_Register($src1$$reg),
10828 zr,
10829 Assembler::LSL, 0);
10830 %}
10831
10832 ins_pipe(ialu_reg);
10833 %}
10834
10835 // This pattern is automatically generated from aarch64_ad.m4
10836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10837 instruct regI_not_reg(iRegINoSp dst,
10838 iRegIorL2I src1, immI_M1 m1,
10839 rFlagsReg cr) %{
10840 match(Set dst (XorI src1 m1));
10841 ins_cost(INSN_COST);
10842 format %{ "eonw $dst, $src1, zr" %}
10843
10844 ins_encode %{
10845 __ eonw(as_Register($dst$$reg),
10846 as_Register($src1$$reg),
10847 zr,
10848 Assembler::LSL, 0);
10849 %}
10850
10851 ins_pipe(ialu_reg);
10852 %}
10853
10854 // This pattern is automatically generated from aarch64_ad.m4
10855 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10856 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10857 immI0 zero, iRegIorL2I src1, immI src2) %{
10858 match(Set dst (SubI zero (URShiftI src1 src2)));
10859
10860 ins_cost(1.9 * INSN_COST);
10861 format %{ "negw $dst, $src1, LSR $src2" %}
10862
10863 ins_encode %{
10864 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10865 Assembler::LSR, $src2$$constant & 0x1f);
10866 %}
10867
10868 ins_pipe(ialu_reg_shift);
10869 %}
10870
10871 // This pattern is automatically generated from aarch64_ad.m4
10872 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10873 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10874 immI0 zero, iRegIorL2I src1, immI src2) %{
10875 match(Set dst (SubI zero (RShiftI src1 src2)));
10876
10877 ins_cost(1.9 * INSN_COST);
10878 format %{ "negw $dst, $src1, ASR $src2" %}
10879
10880 ins_encode %{
10881 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10882 Assembler::ASR, $src2$$constant & 0x1f);
10883 %}
10884
10885 ins_pipe(ialu_reg_shift);
10886 %}
10887
10888 // This pattern is automatically generated from aarch64_ad.m4
10889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10890 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10891 immI0 zero, iRegIorL2I src1, immI src2) %{
10892 match(Set dst (SubI zero (LShiftI src1 src2)));
10893
10894 ins_cost(1.9 * INSN_COST);
10895 format %{ "negw $dst, $src1, LSL $src2" %}
10896
10897 ins_encode %{
10898 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10899 Assembler::LSL, $src2$$constant & 0x1f);
10900 %}
10901
10902 ins_pipe(ialu_reg_shift);
10903 %}
10904
10905 // This pattern is automatically generated from aarch64_ad.m4
10906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10907 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10908 immL0 zero, iRegL src1, immI src2) %{
10909 match(Set dst (SubL zero (URShiftL src1 src2)));
10910
10911 ins_cost(1.9 * INSN_COST);
10912 format %{ "neg $dst, $src1, LSR $src2" %}
10913
10914 ins_encode %{
10915 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10916 Assembler::LSR, $src2$$constant & 0x3f);
10917 %}
10918
10919 ins_pipe(ialu_reg_shift);
10920 %}
10921
10922 // This pattern is automatically generated from aarch64_ad.m4
10923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10924 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10925 immL0 zero, iRegL src1, immI src2) %{
10926 match(Set dst (SubL zero (RShiftL src1 src2)));
10927
10928 ins_cost(1.9 * INSN_COST);
10929 format %{ "neg $dst, $src1, ASR $src2" %}
10930
10931 ins_encode %{
10932 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10933 Assembler::ASR, $src2$$constant & 0x3f);
10934 %}
10935
10936 ins_pipe(ialu_reg_shift);
10937 %}
10938
10939 // This pattern is automatically generated from aarch64_ad.m4
10940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10941 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10942 immL0 zero, iRegL src1, immI src2) %{
10943 match(Set dst (SubL zero (LShiftL src1 src2)));
10944
10945 ins_cost(1.9 * INSN_COST);
10946 format %{ "neg $dst, $src1, LSL $src2" %}
10947
10948 ins_encode %{
10949 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10950 Assembler::LSL, $src2$$constant & 0x3f);
10951 %}
10952
10953 ins_pipe(ialu_reg_shift);
10954 %}
10955
10956 // This pattern is automatically generated from aarch64_ad.m4
10957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10958 instruct AndI_reg_not_reg(iRegINoSp dst,
10959 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10960 match(Set dst (AndI src1 (XorI src2 m1)));
10961 ins_cost(INSN_COST);
10962 format %{ "bicw $dst, $src1, $src2" %}
10963
10964 ins_encode %{
10965 __ bicw(as_Register($dst$$reg),
10966 as_Register($src1$$reg),
10967 as_Register($src2$$reg),
10968 Assembler::LSL, 0);
10969 %}
10970
10971 ins_pipe(ialu_reg_reg);
10972 %}
10973
10974 // This pattern is automatically generated from aarch64_ad.m4
10975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10976 instruct AndL_reg_not_reg(iRegLNoSp dst,
10977 iRegL src1, iRegL src2, immL_M1 m1) %{
10978 match(Set dst (AndL src1 (XorL src2 m1)));
10979 ins_cost(INSN_COST);
10980 format %{ "bic $dst, $src1, $src2" %}
10981
10982 ins_encode %{
10983 __ bic(as_Register($dst$$reg),
10984 as_Register($src1$$reg),
10985 as_Register($src2$$reg),
10986 Assembler::LSL, 0);
10987 %}
10988
10989 ins_pipe(ialu_reg_reg);
10990 %}
10991
10992 // This pattern is automatically generated from aarch64_ad.m4
10993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10994 instruct OrI_reg_not_reg(iRegINoSp dst,
10995 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10996 match(Set dst (OrI src1 (XorI src2 m1)));
10997 ins_cost(INSN_COST);
10998 format %{ "ornw $dst, $src1, $src2" %}
10999
11000 ins_encode %{
11001 __ ornw(as_Register($dst$$reg),
11002 as_Register($src1$$reg),
11003 as_Register($src2$$reg),
11004 Assembler::LSL, 0);
11005 %}
11006
11007 ins_pipe(ialu_reg_reg);
11008 %}
11009
11010 // This pattern is automatically generated from aarch64_ad.m4
11011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11012 instruct OrL_reg_not_reg(iRegLNoSp dst,
11013 iRegL src1, iRegL src2, immL_M1 m1) %{
11014 match(Set dst (OrL src1 (XorL src2 m1)));
11015 ins_cost(INSN_COST);
11016 format %{ "orn $dst, $src1, $src2" %}
11017
11018 ins_encode %{
11019 __ orn(as_Register($dst$$reg),
11020 as_Register($src1$$reg),
11021 as_Register($src2$$reg),
11022 Assembler::LSL, 0);
11023 %}
11024
11025 ins_pipe(ialu_reg_reg);
11026 %}
11027
11028 // This pattern is automatically generated from aarch64_ad.m4
11029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11030 instruct XorI_reg_not_reg(iRegINoSp dst,
11031 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11032 match(Set dst (XorI m1 (XorI src2 src1)));
11033 ins_cost(INSN_COST);
11034 format %{ "eonw $dst, $src1, $src2" %}
11035
11036 ins_encode %{
11037 __ eonw(as_Register($dst$$reg),
11038 as_Register($src1$$reg),
11039 as_Register($src2$$reg),
11040 Assembler::LSL, 0);
11041 %}
11042
11043 ins_pipe(ialu_reg_reg);
11044 %}
11045
11046 // This pattern is automatically generated from aarch64_ad.m4
11047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11048 instruct XorL_reg_not_reg(iRegLNoSp dst,
11049 iRegL src1, iRegL src2, immL_M1 m1) %{
11050 match(Set dst (XorL m1 (XorL src2 src1)));
11051 ins_cost(INSN_COST);
11052 format %{ "eon $dst, $src1, $src2" %}
11053
11054 ins_encode %{
11055 __ eon(as_Register($dst$$reg),
11056 as_Register($src1$$reg),
11057 as_Register($src2$$reg),
11058 Assembler::LSL, 0);
11059 %}
11060
11061 ins_pipe(ialu_reg_reg);
11062 %}
11063
11064 // This pattern is automatically generated from aarch64_ad.m4
11065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11066 // val & (-1 ^ (val >>> shift)) ==> bicw
11067 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11068 iRegIorL2I src1, iRegIorL2I src2,
11069 immI src3, immI_M1 src4) %{
11070 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11071 ins_cost(1.9 * INSN_COST);
11072 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11073
11074 ins_encode %{
11075 __ bicw(as_Register($dst$$reg),
11076 as_Register($src1$$reg),
11077 as_Register($src2$$reg),
11078 Assembler::LSR,
11079 $src3$$constant & 0x1f);
11080 %}
11081
11082 ins_pipe(ialu_reg_reg_shift);
11083 %}
11084
11085 // This pattern is automatically generated from aarch64_ad.m4
11086 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11087 // val & (-1 ^ (val >>> shift)) ==> bic
11088 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11089 iRegL src1, iRegL src2,
11090 immI src3, immL_M1 src4) %{
11091 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11092 ins_cost(1.9 * INSN_COST);
11093 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11094
11095 ins_encode %{
11096 __ bic(as_Register($dst$$reg),
11097 as_Register($src1$$reg),
11098 as_Register($src2$$reg),
11099 Assembler::LSR,
11100 $src3$$constant & 0x3f);
11101 %}
11102
11103 ins_pipe(ialu_reg_reg_shift);
11104 %}
11105
11106 // This pattern is automatically generated from aarch64_ad.m4
11107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11108 // val & (-1 ^ (val >> shift)) ==> bicw
11109 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11110 iRegIorL2I src1, iRegIorL2I src2,
11111 immI src3, immI_M1 src4) %{
11112 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11113 ins_cost(1.9 * INSN_COST);
11114 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11115
11116 ins_encode %{
11117 __ bicw(as_Register($dst$$reg),
11118 as_Register($src1$$reg),
11119 as_Register($src2$$reg),
11120 Assembler::ASR,
11121 $src3$$constant & 0x1f);
11122 %}
11123
11124 ins_pipe(ialu_reg_reg_shift);
11125 %}
11126
11127 // This pattern is automatically generated from aarch64_ad.m4
11128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11129 // val & (-1 ^ (val >> shift)) ==> bic
11130 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11131 iRegL src1, iRegL src2,
11132 immI src3, immL_M1 src4) %{
11133 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11134 ins_cost(1.9 * INSN_COST);
11135 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11136
11137 ins_encode %{
11138 __ bic(as_Register($dst$$reg),
11139 as_Register($src1$$reg),
11140 as_Register($src2$$reg),
11141 Assembler::ASR,
11142 $src3$$constant & 0x3f);
11143 %}
11144
11145 ins_pipe(ialu_reg_reg_shift);
11146 %}
11147
11148 // This pattern is automatically generated from aarch64_ad.m4
11149 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11150 // val & (-1 ^ (val ror shift)) ==> bicw
11151 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11152 iRegIorL2I src1, iRegIorL2I src2,
11153 immI src3, immI_M1 src4) %{
11154 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11155 ins_cost(1.9 * INSN_COST);
11156 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11157
11158 ins_encode %{
11159 __ bicw(as_Register($dst$$reg),
11160 as_Register($src1$$reg),
11161 as_Register($src2$$reg),
11162 Assembler::ROR,
11163 $src3$$constant & 0x1f);
11164 %}
11165
11166 ins_pipe(ialu_reg_reg_shift);
11167 %}
11168
11169 // This pattern is automatically generated from aarch64_ad.m4
11170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11171 // val & (-1 ^ (val ror shift)) ==> bic
11172 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11173 iRegL src1, iRegL src2,
11174 immI src3, immL_M1 src4) %{
11175 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11176 ins_cost(1.9 * INSN_COST);
11177 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11178
11179 ins_encode %{
11180 __ bic(as_Register($dst$$reg),
11181 as_Register($src1$$reg),
11182 as_Register($src2$$reg),
11183 Assembler::ROR,
11184 $src3$$constant & 0x3f);
11185 %}
11186
11187 ins_pipe(ialu_reg_reg_shift);
11188 %}
11189
11190 // This pattern is automatically generated from aarch64_ad.m4
11191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11192 // val & (-1 ^ (val << shift)) ==> bicw
11193 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11194 iRegIorL2I src1, iRegIorL2I src2,
11195 immI src3, immI_M1 src4) %{
11196 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11197 ins_cost(1.9 * INSN_COST);
11198 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11199
11200 ins_encode %{
11201 __ bicw(as_Register($dst$$reg),
11202 as_Register($src1$$reg),
11203 as_Register($src2$$reg),
11204 Assembler::LSL,
11205 $src3$$constant & 0x1f);
11206 %}
11207
11208 ins_pipe(ialu_reg_reg_shift);
11209 %}
11210
11211 // This pattern is automatically generated from aarch64_ad.m4
11212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11213 // val & (-1 ^ (val << shift)) ==> bic
11214 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11215 iRegL src1, iRegL src2,
11216 immI src3, immL_M1 src4) %{
11217 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11218 ins_cost(1.9 * INSN_COST);
11219 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11220
11221 ins_encode %{
11222 __ bic(as_Register($dst$$reg),
11223 as_Register($src1$$reg),
11224 as_Register($src2$$reg),
11225 Assembler::LSL,
11226 $src3$$constant & 0x3f);
11227 %}
11228
11229 ins_pipe(ialu_reg_reg_shift);
11230 %}
11231
11232 // This pattern is automatically generated from aarch64_ad.m4
11233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11234 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11235 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11236 iRegIorL2I src1, iRegIorL2I src2,
11237 immI src3, immI_M1 src4) %{
11238 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11239 ins_cost(1.9 * INSN_COST);
11240 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11241
11242 ins_encode %{
11243 __ eonw(as_Register($dst$$reg),
11244 as_Register($src1$$reg),
11245 as_Register($src2$$reg),
11246 Assembler::LSR,
11247 $src3$$constant & 0x1f);
11248 %}
11249
11250 ins_pipe(ialu_reg_reg_shift);
11251 %}
11252
11253 // This pattern is automatically generated from aarch64_ad.m4
11254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11255 // val ^ (-1 ^ (val >>> shift)) ==> eon
11256 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11257 iRegL src1, iRegL src2,
11258 immI src3, immL_M1 src4) %{
11259 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11260 ins_cost(1.9 * INSN_COST);
11261 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11262
11263 ins_encode %{
11264 __ eon(as_Register($dst$$reg),
11265 as_Register($src1$$reg),
11266 as_Register($src2$$reg),
11267 Assembler::LSR,
11268 $src3$$constant & 0x3f);
11269 %}
11270
11271 ins_pipe(ialu_reg_reg_shift);
11272 %}
11273
11274 // This pattern is automatically generated from aarch64_ad.m4
11275 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11276 // val ^ (-1 ^ (val >> shift)) ==> eonw
11277 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11278 iRegIorL2I src1, iRegIorL2I src2,
11279 immI src3, immI_M1 src4) %{
11280 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11281 ins_cost(1.9 * INSN_COST);
11282 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11283
11284 ins_encode %{
11285 __ eonw(as_Register($dst$$reg),
11286 as_Register($src1$$reg),
11287 as_Register($src2$$reg),
11288 Assembler::ASR,
11289 $src3$$constant & 0x1f);
11290 %}
11291
11292 ins_pipe(ialu_reg_reg_shift);
11293 %}
11294
11295 // This pattern is automatically generated from aarch64_ad.m4
11296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11297 // val ^ (-1 ^ (val >> shift)) ==> eon
11298 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11299 iRegL src1, iRegL src2,
11300 immI src3, immL_M1 src4) %{
11301 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11302 ins_cost(1.9 * INSN_COST);
11303 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11304
11305 ins_encode %{
11306 __ eon(as_Register($dst$$reg),
11307 as_Register($src1$$reg),
11308 as_Register($src2$$reg),
11309 Assembler::ASR,
11310 $src3$$constant & 0x3f);
11311 %}
11312
11313 ins_pipe(ialu_reg_reg_shift);
11314 %}
11315
11316 // This pattern is automatically generated from aarch64_ad.m4
11317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11318 // val ^ (-1 ^ (val ror shift)) ==> eonw
11319 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11320 iRegIorL2I src1, iRegIorL2I src2,
11321 immI src3, immI_M1 src4) %{
11322 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11323 ins_cost(1.9 * INSN_COST);
11324 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11325
11326 ins_encode %{
11327 __ eonw(as_Register($dst$$reg),
11328 as_Register($src1$$reg),
11329 as_Register($src2$$reg),
11330 Assembler::ROR,
11331 $src3$$constant & 0x1f);
11332 %}
11333
11334 ins_pipe(ialu_reg_reg_shift);
11335 %}
11336
11337 // This pattern is automatically generated from aarch64_ad.m4
11338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11339 // val ^ (-1 ^ (val ror shift)) ==> eon
11340 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11341 iRegL src1, iRegL src2,
11342 immI src3, immL_M1 src4) %{
11343 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11344 ins_cost(1.9 * INSN_COST);
11345 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11346
11347 ins_encode %{
11348 __ eon(as_Register($dst$$reg),
11349 as_Register($src1$$reg),
11350 as_Register($src2$$reg),
11351 Assembler::ROR,
11352 $src3$$constant & 0x3f);
11353 %}
11354
11355 ins_pipe(ialu_reg_reg_shift);
11356 %}
11357
11358 // This pattern is automatically generated from aarch64_ad.m4
11359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11360 // val ^ (-1 ^ (val << shift)) ==> eonw
11361 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11362 iRegIorL2I src1, iRegIorL2I src2,
11363 immI src3, immI_M1 src4) %{
11364 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11365 ins_cost(1.9 * INSN_COST);
11366 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11367
11368 ins_encode %{
11369 __ eonw(as_Register($dst$$reg),
11370 as_Register($src1$$reg),
11371 as_Register($src2$$reg),
11372 Assembler::LSL,
11373 $src3$$constant & 0x1f);
11374 %}
11375
11376 ins_pipe(ialu_reg_reg_shift);
11377 %}
11378
11379 // This pattern is automatically generated from aarch64_ad.m4
11380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11381 // val ^ (-1 ^ (val << shift)) ==> eon
11382 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11383 iRegL src1, iRegL src2,
11384 immI src3, immL_M1 src4) %{
11385 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11386 ins_cost(1.9 * INSN_COST);
11387 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11388
11389 ins_encode %{
11390 __ eon(as_Register($dst$$reg),
11391 as_Register($src1$$reg),
11392 as_Register($src2$$reg),
11393 Assembler::LSL,
11394 $src3$$constant & 0x3f);
11395 %}
11396
11397 ins_pipe(ialu_reg_reg_shift);
11398 %}
11399
11400 // This pattern is automatically generated from aarch64_ad.m4
11401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11402 // val | (-1 ^ (val >>> shift)) ==> ornw
11403 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11404 iRegIorL2I src1, iRegIorL2I src2,
11405 immI src3, immI_M1 src4) %{
11406 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11407 ins_cost(1.9 * INSN_COST);
11408 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11409
11410 ins_encode %{
11411 __ ornw(as_Register($dst$$reg),
11412 as_Register($src1$$reg),
11413 as_Register($src2$$reg),
11414 Assembler::LSR,
11415 $src3$$constant & 0x1f);
11416 %}
11417
11418 ins_pipe(ialu_reg_reg_shift);
11419 %}
11420
11421 // This pattern is automatically generated from aarch64_ad.m4
11422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11423 // val | (-1 ^ (val >>> shift)) ==> orn
11424 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11425 iRegL src1, iRegL src2,
11426 immI src3, immL_M1 src4) %{
11427 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11428 ins_cost(1.9 * INSN_COST);
11429 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11430
11431 ins_encode %{
11432 __ orn(as_Register($dst$$reg),
11433 as_Register($src1$$reg),
11434 as_Register($src2$$reg),
11435 Assembler::LSR,
11436 $src3$$constant & 0x3f);
11437 %}
11438
11439 ins_pipe(ialu_reg_reg_shift);
11440 %}
11441
11442 // This pattern is automatically generated from aarch64_ad.m4
11443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11444 // val | (-1 ^ (val >> shift)) ==> ornw
11445 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11446 iRegIorL2I src1, iRegIorL2I src2,
11447 immI src3, immI_M1 src4) %{
11448 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11449 ins_cost(1.9 * INSN_COST);
11450 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11451
11452 ins_encode %{
11453 __ ornw(as_Register($dst$$reg),
11454 as_Register($src1$$reg),
11455 as_Register($src2$$reg),
11456 Assembler::ASR,
11457 $src3$$constant & 0x1f);
11458 %}
11459
11460 ins_pipe(ialu_reg_reg_shift);
11461 %}
11462
11463 // This pattern is automatically generated from aarch64_ad.m4
11464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11465 // val | (-1 ^ (val >> shift)) ==> orn
11466 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11467 iRegL src1, iRegL src2,
11468 immI src3, immL_M1 src4) %{
11469 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11470 ins_cost(1.9 * INSN_COST);
11471 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11472
11473 ins_encode %{
11474 __ orn(as_Register($dst$$reg),
11475 as_Register($src1$$reg),
11476 as_Register($src2$$reg),
11477 Assembler::ASR,
11478 $src3$$constant & 0x3f);
11479 %}
11480
11481 ins_pipe(ialu_reg_reg_shift);
11482 %}
11483
11484 // This pattern is automatically generated from aarch64_ad.m4
11485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11486 // val | (-1 ^ (val ror shift)) ==> ornw
11487 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11488 iRegIorL2I src1, iRegIorL2I src2,
11489 immI src3, immI_M1 src4) %{
11490 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11491 ins_cost(1.9 * INSN_COST);
11492 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11493
11494 ins_encode %{
11495 __ ornw(as_Register($dst$$reg),
11496 as_Register($src1$$reg),
11497 as_Register($src2$$reg),
11498 Assembler::ROR,
11499 $src3$$constant & 0x1f);
11500 %}
11501
11502 ins_pipe(ialu_reg_reg_shift);
11503 %}
11504
11505 // This pattern is automatically generated from aarch64_ad.m4
11506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11507 // val | (-1 ^ (val ror shift)) ==> orn
11508 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11509 iRegL src1, iRegL src2,
11510 immI src3, immL_M1 src4) %{
11511 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11512 ins_cost(1.9 * INSN_COST);
11513 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11514
11515 ins_encode %{
11516 __ orn(as_Register($dst$$reg),
11517 as_Register($src1$$reg),
11518 as_Register($src2$$reg),
11519 Assembler::ROR,
11520 $src3$$constant & 0x3f);
11521 %}
11522
11523 ins_pipe(ialu_reg_reg_shift);
11524 %}
11525
11526 // This pattern is automatically generated from aarch64_ad.m4
11527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11528 // val | (-1 ^ (val << shift)) ==> ornw
11529 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11530 iRegIorL2I src1, iRegIorL2I src2,
11531 immI src3, immI_M1 src4) %{
11532 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11533 ins_cost(1.9 * INSN_COST);
11534 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11535
11536 ins_encode %{
11537 __ ornw(as_Register($dst$$reg),
11538 as_Register($src1$$reg),
11539 as_Register($src2$$reg),
11540 Assembler::LSL,
11541 $src3$$constant & 0x1f);
11542 %}
11543
11544 ins_pipe(ialu_reg_reg_shift);
11545 %}
11546
11547 // This pattern is automatically generated from aarch64_ad.m4
11548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11549 // val | (-1 ^ (val << shift)) ==> orn
11550 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11551 iRegL src1, iRegL src2,
11552 immI src3, immL_M1 src4) %{
11553 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11554 ins_cost(1.9 * INSN_COST);
11555 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11556
11557 ins_encode %{
11558 __ orn(as_Register($dst$$reg),
11559 as_Register($src1$$reg),
11560 as_Register($src2$$reg),
11561 Assembler::LSL,
11562 $src3$$constant & 0x3f);
11563 %}
11564
11565 ins_pipe(ialu_reg_reg_shift);
11566 %}
11567
11568 // This pattern is automatically generated from aarch64_ad.m4
11569 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11570 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11571 iRegIorL2I src1, iRegIorL2I src2,
11572 immI src3) %{
11573 match(Set dst (AndI src1 (URShiftI src2 src3)));
11574
11575 ins_cost(1.9 * INSN_COST);
11576 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11577
11578 ins_encode %{
11579 __ andw(as_Register($dst$$reg),
11580 as_Register($src1$$reg),
11581 as_Register($src2$$reg),
11582 Assembler::LSR,
11583 $src3$$constant & 0x1f);
11584 %}
11585
11586 ins_pipe(ialu_reg_reg_shift);
11587 %}
11588
11589 // This pattern is automatically generated from aarch64_ad.m4
11590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11591 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11592 iRegL src1, iRegL src2,
11593 immI src3) %{
11594 match(Set dst (AndL src1 (URShiftL src2 src3)));
11595
11596 ins_cost(1.9 * INSN_COST);
11597 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11598
11599 ins_encode %{
11600 __ andr(as_Register($dst$$reg),
11601 as_Register($src1$$reg),
11602 as_Register($src2$$reg),
11603 Assembler::LSR,
11604 $src3$$constant & 0x3f);
11605 %}
11606
11607 ins_pipe(ialu_reg_reg_shift);
11608 %}
11609
11610 // This pattern is automatically generated from aarch64_ad.m4
11611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11612 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11613 iRegIorL2I src1, iRegIorL2I src2,
11614 immI src3) %{
11615 match(Set dst (AndI src1 (RShiftI src2 src3)));
11616
11617 ins_cost(1.9 * INSN_COST);
11618 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11619
11620 ins_encode %{
11621 __ andw(as_Register($dst$$reg),
11622 as_Register($src1$$reg),
11623 as_Register($src2$$reg),
11624 Assembler::ASR,
11625 $src3$$constant & 0x1f);
11626 %}
11627
11628 ins_pipe(ialu_reg_reg_shift);
11629 %}
11630
11631 // This pattern is automatically generated from aarch64_ad.m4
11632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11633 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11634 iRegL src1, iRegL src2,
11635 immI src3) %{
11636 match(Set dst (AndL src1 (RShiftL src2 src3)));
11637
11638 ins_cost(1.9 * INSN_COST);
11639 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11640
11641 ins_encode %{
11642 __ andr(as_Register($dst$$reg),
11643 as_Register($src1$$reg),
11644 as_Register($src2$$reg),
11645 Assembler::ASR,
11646 $src3$$constant & 0x3f);
11647 %}
11648
11649 ins_pipe(ialu_reg_reg_shift);
11650 %}
11651
11652 // This pattern is automatically generated from aarch64_ad.m4
11653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11654 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11655 iRegIorL2I src1, iRegIorL2I src2,
11656 immI src3) %{
11657 match(Set dst (AndI src1 (LShiftI src2 src3)));
11658
11659 ins_cost(1.9 * INSN_COST);
11660 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11661
11662 ins_encode %{
11663 __ andw(as_Register($dst$$reg),
11664 as_Register($src1$$reg),
11665 as_Register($src2$$reg),
11666 Assembler::LSL,
11667 $src3$$constant & 0x1f);
11668 %}
11669
11670 ins_pipe(ialu_reg_reg_shift);
11671 %}
11672
11673 // This pattern is automatically generated from aarch64_ad.m4
11674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11675 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11676 iRegL src1, iRegL src2,
11677 immI src3) %{
11678 match(Set dst (AndL src1 (LShiftL src2 src3)));
11679
11680 ins_cost(1.9 * INSN_COST);
11681 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11682
11683 ins_encode %{
11684 __ andr(as_Register($dst$$reg),
11685 as_Register($src1$$reg),
11686 as_Register($src2$$reg),
11687 Assembler::LSL,
11688 $src3$$constant & 0x3f);
11689 %}
11690
11691 ins_pipe(ialu_reg_reg_shift);
11692 %}
11693
11694 // This pattern is automatically generated from aarch64_ad.m4
11695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11696 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11697 iRegIorL2I src1, iRegIorL2I src2,
11698 immI src3) %{
11699 match(Set dst (AndI src1 (RotateRight src2 src3)));
11700
11701 ins_cost(1.9 * INSN_COST);
11702 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11703
11704 ins_encode %{
11705 __ andw(as_Register($dst$$reg),
11706 as_Register($src1$$reg),
11707 as_Register($src2$$reg),
11708 Assembler::ROR,
11709 $src3$$constant & 0x1f);
11710 %}
11711
11712 ins_pipe(ialu_reg_reg_shift);
11713 %}
11714
11715 // This pattern is automatically generated from aarch64_ad.m4
11716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11717 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11718 iRegL src1, iRegL src2,
11719 immI src3) %{
11720 match(Set dst (AndL src1 (RotateRight src2 src3)));
11721
11722 ins_cost(1.9 * INSN_COST);
11723 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11724
11725 ins_encode %{
11726 __ andr(as_Register($dst$$reg),
11727 as_Register($src1$$reg),
11728 as_Register($src2$$reg),
11729 Assembler::ROR,
11730 $src3$$constant & 0x3f);
11731 %}
11732
11733 ins_pipe(ialu_reg_reg_shift);
11734 %}
11735
11736 // This pattern is automatically generated from aarch64_ad.m4
11737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11738 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11739 iRegIorL2I src1, iRegIorL2I src2,
11740 immI src3) %{
11741 match(Set dst (XorI src1 (URShiftI src2 src3)));
11742
11743 ins_cost(1.9 * INSN_COST);
11744 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11745
11746 ins_encode %{
11747 __ eorw(as_Register($dst$$reg),
11748 as_Register($src1$$reg),
11749 as_Register($src2$$reg),
11750 Assembler::LSR,
11751 $src3$$constant & 0x1f);
11752 %}
11753
11754 ins_pipe(ialu_reg_reg_shift);
11755 %}
11756
11757 // This pattern is automatically generated from aarch64_ad.m4
11758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11759 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11760 iRegL src1, iRegL src2,
11761 immI src3) %{
11762 match(Set dst (XorL src1 (URShiftL src2 src3)));
11763
11764 ins_cost(1.9 * INSN_COST);
11765 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11766
11767 ins_encode %{
11768 __ eor(as_Register($dst$$reg),
11769 as_Register($src1$$reg),
11770 as_Register($src2$$reg),
11771 Assembler::LSR,
11772 $src3$$constant & 0x3f);
11773 %}
11774
11775 ins_pipe(ialu_reg_reg_shift);
11776 %}
11777
11778 // This pattern is automatically generated from aarch64_ad.m4
11779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11780 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11781 iRegIorL2I src1, iRegIorL2I src2,
11782 immI src3) %{
11783 match(Set dst (XorI src1 (RShiftI src2 src3)));
11784
11785 ins_cost(1.9 * INSN_COST);
11786 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11787
11788 ins_encode %{
11789 __ eorw(as_Register($dst$$reg),
11790 as_Register($src1$$reg),
11791 as_Register($src2$$reg),
11792 Assembler::ASR,
11793 $src3$$constant & 0x1f);
11794 %}
11795
11796 ins_pipe(ialu_reg_reg_shift);
11797 %}
11798
11799 // This pattern is automatically generated from aarch64_ad.m4
11800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11801 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11802 iRegL src1, iRegL src2,
11803 immI src3) %{
11804 match(Set dst (XorL src1 (RShiftL src2 src3)));
11805
11806 ins_cost(1.9 * INSN_COST);
11807 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11808
11809 ins_encode %{
11810 __ eor(as_Register($dst$$reg),
11811 as_Register($src1$$reg),
11812 as_Register($src2$$reg),
11813 Assembler::ASR,
11814 $src3$$constant & 0x3f);
11815 %}
11816
11817 ins_pipe(ialu_reg_reg_shift);
11818 %}
11819
11820 // This pattern is automatically generated from aarch64_ad.m4
11821 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11822 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11823 iRegIorL2I src1, iRegIorL2I src2,
11824 immI src3) %{
11825 match(Set dst (XorI src1 (LShiftI src2 src3)));
11826
11827 ins_cost(1.9 * INSN_COST);
11828 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11829
11830 ins_encode %{
11831 __ eorw(as_Register($dst$$reg),
11832 as_Register($src1$$reg),
11833 as_Register($src2$$reg),
11834 Assembler::LSL,
11835 $src3$$constant & 0x1f);
11836 %}
11837
11838 ins_pipe(ialu_reg_reg_shift);
11839 %}
11840
11841 // This pattern is automatically generated from aarch64_ad.m4
11842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11843 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11844 iRegL src1, iRegL src2,
11845 immI src3) %{
11846 match(Set dst (XorL src1 (LShiftL src2 src3)));
11847
11848 ins_cost(1.9 * INSN_COST);
11849 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11850
11851 ins_encode %{
11852 __ eor(as_Register($dst$$reg),
11853 as_Register($src1$$reg),
11854 as_Register($src2$$reg),
11855 Assembler::LSL,
11856 $src3$$constant & 0x3f);
11857 %}
11858
11859 ins_pipe(ialu_reg_reg_shift);
11860 %}
11861
11862 // This pattern is automatically generated from aarch64_ad.m4
11863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11864 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11865 iRegIorL2I src1, iRegIorL2I src2,
11866 immI src3) %{
11867 match(Set dst (XorI src1 (RotateRight src2 src3)));
11868
11869 ins_cost(1.9 * INSN_COST);
11870 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11871
11872 ins_encode %{
11873 __ eorw(as_Register($dst$$reg),
11874 as_Register($src1$$reg),
11875 as_Register($src2$$reg),
11876 Assembler::ROR,
11877 $src3$$constant & 0x1f);
11878 %}
11879
11880 ins_pipe(ialu_reg_reg_shift);
11881 %}
11882
11883 // This pattern is automatically generated from aarch64_ad.m4
11884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11885 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11886 iRegL src1, iRegL src2,
11887 immI src3) %{
11888 match(Set dst (XorL src1 (RotateRight src2 src3)));
11889
11890 ins_cost(1.9 * INSN_COST);
11891 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11892
11893 ins_encode %{
11894 __ eor(as_Register($dst$$reg),
11895 as_Register($src1$$reg),
11896 as_Register($src2$$reg),
11897 Assembler::ROR,
11898 $src3$$constant & 0x3f);
11899 %}
11900
11901 ins_pipe(ialu_reg_reg_shift);
11902 %}
11903
11904 // This pattern is automatically generated from aarch64_ad.m4
11905 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11906 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11907 iRegIorL2I src1, iRegIorL2I src2,
11908 immI src3) %{
11909 match(Set dst (OrI src1 (URShiftI src2 src3)));
11910
11911 ins_cost(1.9 * INSN_COST);
11912 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11913
11914 ins_encode %{
11915 __ orrw(as_Register($dst$$reg),
11916 as_Register($src1$$reg),
11917 as_Register($src2$$reg),
11918 Assembler::LSR,
11919 $src3$$constant & 0x1f);
11920 %}
11921
11922 ins_pipe(ialu_reg_reg_shift);
11923 %}
11924
11925 // This pattern is automatically generated from aarch64_ad.m4
11926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11927 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11928 iRegL src1, iRegL src2,
11929 immI src3) %{
11930 match(Set dst (OrL src1 (URShiftL src2 src3)));
11931
11932 ins_cost(1.9 * INSN_COST);
11933 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11934
11935 ins_encode %{
11936 __ orr(as_Register($dst$$reg),
11937 as_Register($src1$$reg),
11938 as_Register($src2$$reg),
11939 Assembler::LSR,
11940 $src3$$constant & 0x3f);
11941 %}
11942
11943 ins_pipe(ialu_reg_reg_shift);
11944 %}
11945
11946 // This pattern is automatically generated from aarch64_ad.m4
11947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11948 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11949 iRegIorL2I src1, iRegIorL2I src2,
11950 immI src3) %{
11951 match(Set dst (OrI src1 (RShiftI src2 src3)));
11952
11953 ins_cost(1.9 * INSN_COST);
11954 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11955
11956 ins_encode %{
11957 __ orrw(as_Register($dst$$reg),
11958 as_Register($src1$$reg),
11959 as_Register($src2$$reg),
11960 Assembler::ASR,
11961 $src3$$constant & 0x1f);
11962 %}
11963
11964 ins_pipe(ialu_reg_reg_shift);
11965 %}
11966
11967 // This pattern is automatically generated from aarch64_ad.m4
11968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11969 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11970 iRegL src1, iRegL src2,
11971 immI src3) %{
11972 match(Set dst (OrL src1 (RShiftL src2 src3)));
11973
11974 ins_cost(1.9 * INSN_COST);
11975 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11976
11977 ins_encode %{
11978 __ orr(as_Register($dst$$reg),
11979 as_Register($src1$$reg),
11980 as_Register($src2$$reg),
11981 Assembler::ASR,
11982 $src3$$constant & 0x3f);
11983 %}
11984
11985 ins_pipe(ialu_reg_reg_shift);
11986 %}
11987
11988 // This pattern is automatically generated from aarch64_ad.m4
11989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11990 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11991 iRegIorL2I src1, iRegIorL2I src2,
11992 immI src3) %{
11993 match(Set dst (OrI src1 (LShiftI src2 src3)));
11994
11995 ins_cost(1.9 * INSN_COST);
11996 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11997
11998 ins_encode %{
11999 __ orrw(as_Register($dst$$reg),
12000 as_Register($src1$$reg),
12001 as_Register($src2$$reg),
12002 Assembler::LSL,
12003 $src3$$constant & 0x1f);
12004 %}
12005
12006 ins_pipe(ialu_reg_reg_shift);
12007 %}
12008
12009 // This pattern is automatically generated from aarch64_ad.m4
12010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12011 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12012 iRegL src1, iRegL src2,
12013 immI src3) %{
12014 match(Set dst (OrL src1 (LShiftL src2 src3)));
12015
12016 ins_cost(1.9 * INSN_COST);
12017 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12018
12019 ins_encode %{
12020 __ orr(as_Register($dst$$reg),
12021 as_Register($src1$$reg),
12022 as_Register($src2$$reg),
12023 Assembler::LSL,
12024 $src3$$constant & 0x3f);
12025 %}
12026
12027 ins_pipe(ialu_reg_reg_shift);
12028 %}
12029
12030 // This pattern is automatically generated from aarch64_ad.m4
12031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12032 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12033 iRegIorL2I src1, iRegIorL2I src2,
12034 immI src3) %{
12035 match(Set dst (OrI src1 (RotateRight src2 src3)));
12036
12037 ins_cost(1.9 * INSN_COST);
12038 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12039
12040 ins_encode %{
12041 __ orrw(as_Register($dst$$reg),
12042 as_Register($src1$$reg),
12043 as_Register($src2$$reg),
12044 Assembler::ROR,
12045 $src3$$constant & 0x1f);
12046 %}
12047
12048 ins_pipe(ialu_reg_reg_shift);
12049 %}
12050
12051 // This pattern is automatically generated from aarch64_ad.m4
12052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12053 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12054 iRegL src1, iRegL src2,
12055 immI src3) %{
12056 match(Set dst (OrL src1 (RotateRight src2 src3)));
12057
12058 ins_cost(1.9 * INSN_COST);
12059 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12060
12061 ins_encode %{
12062 __ orr(as_Register($dst$$reg),
12063 as_Register($src1$$reg),
12064 as_Register($src2$$reg),
12065 Assembler::ROR,
12066 $src3$$constant & 0x3f);
12067 %}
12068
12069 ins_pipe(ialu_reg_reg_shift);
12070 %}
12071
12072 // This pattern is automatically generated from aarch64_ad.m4
12073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12074 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12075 iRegIorL2I src1, iRegIorL2I src2,
12076 immI src3) %{
12077 match(Set dst (AddI src1 (URShiftI src2 src3)));
12078
12079 ins_cost(1.9 * INSN_COST);
12080 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12081
12082 ins_encode %{
12083 __ addw(as_Register($dst$$reg),
12084 as_Register($src1$$reg),
12085 as_Register($src2$$reg),
12086 Assembler::LSR,
12087 $src3$$constant & 0x1f);
12088 %}
12089
12090 ins_pipe(ialu_reg_reg_shift);
12091 %}
12092
12093 // This pattern is automatically generated from aarch64_ad.m4
12094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12095 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12096 iRegL src1, iRegL src2,
12097 immI src3) %{
12098 match(Set dst (AddL src1 (URShiftL src2 src3)));
12099
12100 ins_cost(1.9 * INSN_COST);
12101 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12102
12103 ins_encode %{
12104 __ add(as_Register($dst$$reg),
12105 as_Register($src1$$reg),
12106 as_Register($src2$$reg),
12107 Assembler::LSR,
12108 $src3$$constant & 0x3f);
12109 %}
12110
12111 ins_pipe(ialu_reg_reg_shift);
12112 %}
12113
12114 // This pattern is automatically generated from aarch64_ad.m4
12115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12116 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12117 iRegIorL2I src1, iRegIorL2I src2,
12118 immI src3) %{
12119 match(Set dst (AddI src1 (RShiftI src2 src3)));
12120
12121 ins_cost(1.9 * INSN_COST);
12122 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12123
12124 ins_encode %{
12125 __ addw(as_Register($dst$$reg),
12126 as_Register($src1$$reg),
12127 as_Register($src2$$reg),
12128 Assembler::ASR,
12129 $src3$$constant & 0x1f);
12130 %}
12131
12132 ins_pipe(ialu_reg_reg_shift);
12133 %}
12134
12135 // This pattern is automatically generated from aarch64_ad.m4
12136 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12137 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12138 iRegL src1, iRegL src2,
12139 immI src3) %{
12140 match(Set dst (AddL src1 (RShiftL src2 src3)));
12141
12142 ins_cost(1.9 * INSN_COST);
12143 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12144
12145 ins_encode %{
12146 __ add(as_Register($dst$$reg),
12147 as_Register($src1$$reg),
12148 as_Register($src2$$reg),
12149 Assembler::ASR,
12150 $src3$$constant & 0x3f);
12151 %}
12152
12153 ins_pipe(ialu_reg_reg_shift);
12154 %}
12155
12156 // This pattern is automatically generated from aarch64_ad.m4
12157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12158 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12159 iRegIorL2I src1, iRegIorL2I src2,
12160 immI src3) %{
12161 match(Set dst (AddI src1 (LShiftI src2 src3)));
12162
12163 ins_cost(1.9 * INSN_COST);
12164 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12165
12166 ins_encode %{
12167 __ addw(as_Register($dst$$reg),
12168 as_Register($src1$$reg),
12169 as_Register($src2$$reg),
12170 Assembler::LSL,
12171 $src3$$constant & 0x1f);
12172 %}
12173
12174 ins_pipe(ialu_reg_reg_shift);
12175 %}
12176
12177 // This pattern is automatically generated from aarch64_ad.m4
12178 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12179 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12180 iRegL src1, iRegL src2,
12181 immI src3) %{
12182 match(Set dst (AddL src1 (LShiftL src2 src3)));
12183
12184 ins_cost(1.9 * INSN_COST);
12185 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12186
12187 ins_encode %{
12188 __ add(as_Register($dst$$reg),
12189 as_Register($src1$$reg),
12190 as_Register($src2$$reg),
12191 Assembler::LSL,
12192 $src3$$constant & 0x3f);
12193 %}
12194
12195 ins_pipe(ialu_reg_reg_shift);
12196 %}
12197
12198 // This pattern is automatically generated from aarch64_ad.m4
12199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12200 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12201 iRegIorL2I src1, iRegIorL2I src2,
12202 immI src3) %{
12203 match(Set dst (SubI src1 (URShiftI src2 src3)));
12204
12205 ins_cost(1.9 * INSN_COST);
12206 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12207
12208 ins_encode %{
12209 __ subw(as_Register($dst$$reg),
12210 as_Register($src1$$reg),
12211 as_Register($src2$$reg),
12212 Assembler::LSR,
12213 $src3$$constant & 0x1f);
12214 %}
12215
12216 ins_pipe(ialu_reg_reg_shift);
12217 %}
12218
12219 // This pattern is automatically generated from aarch64_ad.m4
12220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12221 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12222 iRegL src1, iRegL src2,
12223 immI src3) %{
12224 match(Set dst (SubL src1 (URShiftL src2 src3)));
12225
12226 ins_cost(1.9 * INSN_COST);
12227 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12228
12229 ins_encode %{
12230 __ sub(as_Register($dst$$reg),
12231 as_Register($src1$$reg),
12232 as_Register($src2$$reg),
12233 Assembler::LSR,
12234 $src3$$constant & 0x3f);
12235 %}
12236
12237 ins_pipe(ialu_reg_reg_shift);
12238 %}
12239
12240 // This pattern is automatically generated from aarch64_ad.m4
12241 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12242 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12243 iRegIorL2I src1, iRegIorL2I src2,
12244 immI src3) %{
12245 match(Set dst (SubI src1 (RShiftI src2 src3)));
12246
12247 ins_cost(1.9 * INSN_COST);
12248 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12249
12250 ins_encode %{
12251 __ subw(as_Register($dst$$reg),
12252 as_Register($src1$$reg),
12253 as_Register($src2$$reg),
12254 Assembler::ASR,
12255 $src3$$constant & 0x1f);
12256 %}
12257
12258 ins_pipe(ialu_reg_reg_shift);
12259 %}
12260
12261 // This pattern is automatically generated from aarch64_ad.m4
12262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12263 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12264 iRegL src1, iRegL src2,
12265 immI src3) %{
12266 match(Set dst (SubL src1 (RShiftL src2 src3)));
12267
12268 ins_cost(1.9 * INSN_COST);
12269 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12270
12271 ins_encode %{
12272 __ sub(as_Register($dst$$reg),
12273 as_Register($src1$$reg),
12274 as_Register($src2$$reg),
12275 Assembler::ASR,
12276 $src3$$constant & 0x3f);
12277 %}
12278
12279 ins_pipe(ialu_reg_reg_shift);
12280 %}
12281
12282 // This pattern is automatically generated from aarch64_ad.m4
12283 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12284 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12285 iRegIorL2I src1, iRegIorL2I src2,
12286 immI src3) %{
12287 match(Set dst (SubI src1 (LShiftI src2 src3)));
12288
12289 ins_cost(1.9 * INSN_COST);
12290 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12291
12292 ins_encode %{
12293 __ subw(as_Register($dst$$reg),
12294 as_Register($src1$$reg),
12295 as_Register($src2$$reg),
12296 Assembler::LSL,
12297 $src3$$constant & 0x1f);
12298 %}
12299
12300 ins_pipe(ialu_reg_reg_shift);
12301 %}
12302
12303 // This pattern is automatically generated from aarch64_ad.m4
12304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12305 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12306 iRegL src1, iRegL src2,
12307 immI src3) %{
12308 match(Set dst (SubL src1 (LShiftL src2 src3)));
12309
12310 ins_cost(1.9 * INSN_COST);
12311 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12312
12313 ins_encode %{
12314 __ sub(as_Register($dst$$reg),
12315 as_Register($src1$$reg),
12316 as_Register($src2$$reg),
12317 Assembler::LSL,
12318 $src3$$constant & 0x3f);
12319 %}
12320
12321 ins_pipe(ialu_reg_reg_shift);
12322 %}
12323
12324 // This pattern is automatically generated from aarch64_ad.m4
12325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12326
12327 // Shift Left followed by Shift Right.
12328 // This idiom is used by the compiler for the i2b bytecode etc.
12329 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12330 %{
12331 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12332 ins_cost(INSN_COST * 2);
12333 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12334 ins_encode %{
12335 int lshift = $lshift_count$$constant & 63;
12336 int rshift = $rshift_count$$constant & 63;
12337 int s = 63 - lshift;
12338 int r = (rshift - lshift) & 63;
12339 __ sbfm(as_Register($dst$$reg),
12340 as_Register($src$$reg),
12341 r, s);
12342 %}
12343
12344 ins_pipe(ialu_reg_shift);
12345 %}
12346
12347 // This pattern is automatically generated from aarch64_ad.m4
12348 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12349
12350 // Shift Left followed by Shift Right.
12351 // This idiom is used by the compiler for the i2b bytecode etc.
12352 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12353 %{
12354 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12355 ins_cost(INSN_COST * 2);
12356 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12357 ins_encode %{
12358 int lshift = $lshift_count$$constant & 31;
12359 int rshift = $rshift_count$$constant & 31;
12360 int s = 31 - lshift;
12361 int r = (rshift - lshift) & 31;
12362 __ sbfmw(as_Register($dst$$reg),
12363 as_Register($src$$reg),
12364 r, s);
12365 %}
12366
12367 ins_pipe(ialu_reg_shift);
12368 %}
12369
12370 // This pattern is automatically generated from aarch64_ad.m4
12371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12372
12373 // Shift Left followed by Shift Right.
12374 // This idiom is used by the compiler for the i2b bytecode etc.
12375 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12376 %{
12377 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12378 ins_cost(INSN_COST * 2);
12379 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12380 ins_encode %{
12381 int lshift = $lshift_count$$constant & 63;
12382 int rshift = $rshift_count$$constant & 63;
12383 int s = 63 - lshift;
12384 int r = (rshift - lshift) & 63;
12385 __ ubfm(as_Register($dst$$reg),
12386 as_Register($src$$reg),
12387 r, s);
12388 %}
12389
12390 ins_pipe(ialu_reg_shift);
12391 %}
12392
12393 // This pattern is automatically generated from aarch64_ad.m4
12394 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12395
12396 // Shift Left followed by Shift Right.
12397 // This idiom is used by the compiler for the i2b bytecode etc.
12398 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12399 %{
12400 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12401 ins_cost(INSN_COST * 2);
12402 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12403 ins_encode %{
12404 int lshift = $lshift_count$$constant & 31;
12405 int rshift = $rshift_count$$constant & 31;
12406 int s = 31 - lshift;
12407 int r = (rshift - lshift) & 31;
12408 __ ubfmw(as_Register($dst$$reg),
12409 as_Register($src$$reg),
12410 r, s);
12411 %}
12412
12413 ins_pipe(ialu_reg_shift);
12414 %}
12415
12416 // Bitfield extract with shift & mask
12417
12418 // This pattern is automatically generated from aarch64_ad.m4
12419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12420 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12421 %{
12422 match(Set dst (AndI (URShiftI src rshift) mask));
12423 // Make sure we are not going to exceed what ubfxw can do.
12424 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12425
12426 ins_cost(INSN_COST);
12427 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12428 ins_encode %{
12429 int rshift = $rshift$$constant & 31;
12430 intptr_t mask = $mask$$constant;
12431 int width = exact_log2(mask+1);
12432 __ ubfxw(as_Register($dst$$reg),
12433 as_Register($src$$reg), rshift, width);
12434 %}
12435 ins_pipe(ialu_reg_shift);
12436 %}
12437
12438 // This pattern is automatically generated from aarch64_ad.m4
12439 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12440 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12441 %{
12442 match(Set dst (AndL (URShiftL src rshift) mask));
12443 // Make sure we are not going to exceed what ubfx can do.
12444 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12445
12446 ins_cost(INSN_COST);
12447 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12448 ins_encode %{
12449 int rshift = $rshift$$constant & 63;
12450 intptr_t mask = $mask$$constant;
12451 int width = exact_log2_long(mask+1);
12452 __ ubfx(as_Register($dst$$reg),
12453 as_Register($src$$reg), rshift, width);
12454 %}
12455 ins_pipe(ialu_reg_shift);
12456 %}
12457
12458
12459 // This pattern is automatically generated from aarch64_ad.m4
12460 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12461
12462 // We can use ubfx when extending an And with a mask when we know mask
12463 // is positive. We know that because immI_bitmask guarantees it.
12464 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12465 %{
12466 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12467 // Make sure we are not going to exceed what ubfxw can do.
12468 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12469
12470 ins_cost(INSN_COST * 2);
12471 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12472 ins_encode %{
12473 int rshift = $rshift$$constant & 31;
12474 intptr_t mask = $mask$$constant;
12475 int width = exact_log2(mask+1);
12476 __ ubfx(as_Register($dst$$reg),
12477 as_Register($src$$reg), rshift, width);
12478 %}
12479 ins_pipe(ialu_reg_shift);
12480 %}
12481
12482
12483 // This pattern is automatically generated from aarch64_ad.m4
12484 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12485
12486 // We can use ubfiz when masking by a positive number and then left shifting the result.
12487 // We know that the mask is positive because immI_bitmask guarantees it.
12488 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12489 %{
12490 match(Set dst (LShiftI (AndI src mask) lshift));
12491 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12492
12493 ins_cost(INSN_COST);
12494 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12495 ins_encode %{
12496 int lshift = $lshift$$constant & 31;
12497 intptr_t mask = $mask$$constant;
12498 int width = exact_log2(mask+1);
12499 __ ubfizw(as_Register($dst$$reg),
12500 as_Register($src$$reg), lshift, width);
12501 %}
12502 ins_pipe(ialu_reg_shift);
12503 %}
12504
12505 // This pattern is automatically generated from aarch64_ad.m4
12506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12507
12508 // We can use ubfiz when masking by a positive number and then left shifting the result.
12509 // We know that the mask is positive because immL_bitmask guarantees it.
12510 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12511 %{
12512 match(Set dst (LShiftL (AndL src mask) lshift));
12513 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12514
12515 ins_cost(INSN_COST);
12516 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12517 ins_encode %{
12518 int lshift = $lshift$$constant & 63;
12519 intptr_t mask = $mask$$constant;
12520 int width = exact_log2_long(mask+1);
12521 __ ubfiz(as_Register($dst$$reg),
12522 as_Register($src$$reg), lshift, width);
12523 %}
12524 ins_pipe(ialu_reg_shift);
12525 %}
12526
12527 // This pattern is automatically generated from aarch64_ad.m4
12528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12529
12530 // We can use ubfiz when masking by a positive number and then left shifting the result.
12531 // We know that the mask is positive because immI_bitmask guarantees it.
12532 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12533 %{
12534 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12535 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12536
12537 ins_cost(INSN_COST);
12538 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12539 ins_encode %{
12540 int lshift = $lshift$$constant & 31;
12541 intptr_t mask = $mask$$constant;
12542 int width = exact_log2(mask+1);
12543 __ ubfizw(as_Register($dst$$reg),
12544 as_Register($src$$reg), lshift, width);
12545 %}
12546 ins_pipe(ialu_reg_shift);
12547 %}
12548
12549 // This pattern is automatically generated from aarch64_ad.m4
12550 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12551
12552 // We can use ubfiz when masking by a positive number and then left shifting the result.
12553 // We know that the mask is positive because immL_bitmask guarantees it.
12554 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12555 %{
12556 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12557 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12558
12559 ins_cost(INSN_COST);
12560 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12561 ins_encode %{
12562 int lshift = $lshift$$constant & 63;
12563 intptr_t mask = $mask$$constant;
12564 int width = exact_log2_long(mask+1);
12565 __ ubfiz(as_Register($dst$$reg),
12566 as_Register($src$$reg), lshift, width);
12567 %}
12568 ins_pipe(ialu_reg_shift);
12569 %}
12570
12571
12572 // This pattern is automatically generated from aarch64_ad.m4
12573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12574
12575 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12576 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12577 %{
12578 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12579 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12580
12581 ins_cost(INSN_COST);
12582 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12583 ins_encode %{
12584 int lshift = $lshift$$constant & 63;
12585 intptr_t mask = $mask$$constant;
12586 int width = exact_log2(mask+1);
12587 __ ubfiz(as_Register($dst$$reg),
12588 as_Register($src$$reg), lshift, width);
12589 %}
12590 ins_pipe(ialu_reg_shift);
12591 %}
12592
12593 // This pattern is automatically generated from aarch64_ad.m4
12594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12595
12596 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12597 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12598 %{
12599 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12600 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12601
12602 ins_cost(INSN_COST);
12603 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12604 ins_encode %{
12605 int lshift = $lshift$$constant & 31;
12606 intptr_t mask = $mask$$constant;
12607 int width = exact_log2(mask+1);
12608 __ ubfiz(as_Register($dst$$reg),
12609 as_Register($src$$reg), lshift, width);
12610 %}
12611 ins_pipe(ialu_reg_shift);
12612 %}
12613
12614 // This pattern is automatically generated from aarch64_ad.m4
12615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12616
12617 // Can skip int2long conversions after AND with small bitmask
12618 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12619 %{
12620 match(Set dst (ConvI2L (AndI src msk)));
12621 ins_cost(INSN_COST);
12622 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12623 ins_encode %{
12624 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12625 %}
12626 ins_pipe(ialu_reg_shift);
12627 %}
12628
12629
12630 // Rotations
12631
12632 // This pattern is automatically generated from aarch64_ad.m4
12633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12634 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12635 %{
12636 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12637 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12638
12639 ins_cost(INSN_COST);
12640 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12641
12642 ins_encode %{
12643 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12644 $rshift$$constant & 63);
12645 %}
12646 ins_pipe(ialu_reg_reg_extr);
12647 %}
12648
12649
12650 // This pattern is automatically generated from aarch64_ad.m4
12651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12652 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12653 %{
12654 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12655 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12656
12657 ins_cost(INSN_COST);
12658 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12659
12660 ins_encode %{
12661 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12662 $rshift$$constant & 31);
12663 %}
12664 ins_pipe(ialu_reg_reg_extr);
12665 %}
12666
12667
12668 // This pattern is automatically generated from aarch64_ad.m4
12669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12670 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12671 %{
12672 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12673 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12674
12675 ins_cost(INSN_COST);
12676 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12677
12678 ins_encode %{
12679 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12680 $rshift$$constant & 63);
12681 %}
12682 ins_pipe(ialu_reg_reg_extr);
12683 %}
12684
12685
12686 // This pattern is automatically generated from aarch64_ad.m4
12687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12688 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12689 %{
12690 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12691 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12692
12693 ins_cost(INSN_COST);
12694 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12695
12696 ins_encode %{
12697 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12698 $rshift$$constant & 31);
12699 %}
12700 ins_pipe(ialu_reg_reg_extr);
12701 %}
12702
12703 // This pattern is automatically generated from aarch64_ad.m4
12704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12705 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12706 %{
12707 match(Set dst (RotateRight src shift));
12708
12709 ins_cost(INSN_COST);
12710 format %{ "ror $dst, $src, $shift" %}
12711
12712 ins_encode %{
12713 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12714 $shift$$constant & 0x1f);
12715 %}
12716 ins_pipe(ialu_reg_reg_vshift);
12717 %}
12718
12719 // This pattern is automatically generated from aarch64_ad.m4
12720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12721 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12722 %{
12723 match(Set dst (RotateRight src shift));
12724
12725 ins_cost(INSN_COST);
12726 format %{ "ror $dst, $src, $shift" %}
12727
12728 ins_encode %{
12729 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12730 $shift$$constant & 0x3f);
12731 %}
12732 ins_pipe(ialu_reg_reg_vshift);
12733 %}
12734
12735 // This pattern is automatically generated from aarch64_ad.m4
12736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12737 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12738 %{
12739 match(Set dst (RotateRight src shift));
12740
12741 ins_cost(INSN_COST);
12742 format %{ "ror $dst, $src, $shift" %}
12743
12744 ins_encode %{
12745 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12746 %}
12747 ins_pipe(ialu_reg_reg_vshift);
12748 %}
12749
12750 // This pattern is automatically generated from aarch64_ad.m4
12751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12752 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12753 %{
12754 match(Set dst (RotateRight src shift));
12755
12756 ins_cost(INSN_COST);
12757 format %{ "ror $dst, $src, $shift" %}
12758
12759 ins_encode %{
12760 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12761 %}
12762 ins_pipe(ialu_reg_reg_vshift);
12763 %}
12764
12765 // This pattern is automatically generated from aarch64_ad.m4
12766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12767 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12768 %{
12769 match(Set dst (RotateLeft src shift));
12770
12771 ins_cost(INSN_COST);
12772 format %{ "rol $dst, $src, $shift" %}
12773
12774 ins_encode %{
12775 __ subw(rscratch1, zr, as_Register($shift$$reg));
12776 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12777 %}
12778 ins_pipe(ialu_reg_reg_vshift);
12779 %}
12780
12781 // This pattern is automatically generated from aarch64_ad.m4
12782 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12783 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12784 %{
12785 match(Set dst (RotateLeft src shift));
12786
12787 ins_cost(INSN_COST);
12788 format %{ "rol $dst, $src, $shift" %}
12789
12790 ins_encode %{
12791 __ subw(rscratch1, zr, as_Register($shift$$reg));
12792 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12793 %}
12794 ins_pipe(ialu_reg_reg_vshift);
12795 %}
12796
12797
12798 // Add/subtract (extended)
12799
12800 // This pattern is automatically generated from aarch64_ad.m4
12801 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12802 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12803 %{
12804 match(Set dst (AddL src1 (ConvI2L src2)));
12805 ins_cost(INSN_COST);
12806 format %{ "add $dst, $src1, $src2, sxtw" %}
12807
12808 ins_encode %{
12809 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12810 as_Register($src2$$reg), ext::sxtw);
12811 %}
12812 ins_pipe(ialu_reg_reg);
12813 %}
12814
12815 // This pattern is automatically generated from aarch64_ad.m4
12816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12817 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12818 %{
12819 match(Set dst (SubL src1 (ConvI2L src2)));
12820 ins_cost(INSN_COST);
12821 format %{ "sub $dst, $src1, $src2, sxtw" %}
12822
12823 ins_encode %{
12824 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12825 as_Register($src2$$reg), ext::sxtw);
12826 %}
12827 ins_pipe(ialu_reg_reg);
12828 %}
12829
12830 // This pattern is automatically generated from aarch64_ad.m4
12831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12832 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12833 %{
12834 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12835 ins_cost(INSN_COST);
12836 format %{ "add $dst, $src1, $src2, sxth" %}
12837
12838 ins_encode %{
12839 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12840 as_Register($src2$$reg), ext::sxth);
12841 %}
12842 ins_pipe(ialu_reg_reg);
12843 %}
12844
12845 // This pattern is automatically generated from aarch64_ad.m4
12846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12847 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12848 %{
12849 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12850 ins_cost(INSN_COST);
12851 format %{ "add $dst, $src1, $src2, sxtb" %}
12852
12853 ins_encode %{
12854 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12855 as_Register($src2$$reg), ext::sxtb);
12856 %}
12857 ins_pipe(ialu_reg_reg);
12858 %}
12859
12860 // This pattern is automatically generated from aarch64_ad.m4
12861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12862 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12863 %{
12864 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12865 ins_cost(INSN_COST);
12866 format %{ "add $dst, $src1, $src2, uxtb" %}
12867
12868 ins_encode %{
12869 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12870 as_Register($src2$$reg), ext::uxtb);
12871 %}
12872 ins_pipe(ialu_reg_reg);
12873 %}
12874
12875 // This pattern is automatically generated from aarch64_ad.m4
12876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12877 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12878 %{
12879 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12880 ins_cost(INSN_COST);
12881 format %{ "add $dst, $src1, $src2, sxth" %}
12882
12883 ins_encode %{
12884 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12885 as_Register($src2$$reg), ext::sxth);
12886 %}
12887 ins_pipe(ialu_reg_reg);
12888 %}
12889
12890 // This pattern is automatically generated from aarch64_ad.m4
12891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12892 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12893 %{
12894 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12895 ins_cost(INSN_COST);
12896 format %{ "add $dst, $src1, $src2, sxtw" %}
12897
12898 ins_encode %{
12899 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12900 as_Register($src2$$reg), ext::sxtw);
12901 %}
12902 ins_pipe(ialu_reg_reg);
12903 %}
12904
12905 // This pattern is automatically generated from aarch64_ad.m4
12906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12907 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12908 %{
12909 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12910 ins_cost(INSN_COST);
12911 format %{ "add $dst, $src1, $src2, sxtb" %}
12912
12913 ins_encode %{
12914 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12915 as_Register($src2$$reg), ext::sxtb);
12916 %}
12917 ins_pipe(ialu_reg_reg);
12918 %}
12919
12920 // This pattern is automatically generated from aarch64_ad.m4
12921 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12922 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12923 %{
12924 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12925 ins_cost(INSN_COST);
12926 format %{ "add $dst, $src1, $src2, uxtb" %}
12927
12928 ins_encode %{
12929 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12930 as_Register($src2$$reg), ext::uxtb);
12931 %}
12932 ins_pipe(ialu_reg_reg);
12933 %}
12934
12935 // This pattern is automatically generated from aarch64_ad.m4
12936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12937 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12938 %{
12939 match(Set dst (AddI src1 (AndI src2 mask)));
12940 ins_cost(INSN_COST);
12941 format %{ "addw $dst, $src1, $src2, uxtb" %}
12942
12943 ins_encode %{
12944 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12945 as_Register($src2$$reg), ext::uxtb);
12946 %}
12947 ins_pipe(ialu_reg_reg);
12948 %}
12949
12950 // This pattern is automatically generated from aarch64_ad.m4
12951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12952 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12953 %{
12954 match(Set dst (AddI src1 (AndI src2 mask)));
12955 ins_cost(INSN_COST);
12956 format %{ "addw $dst, $src1, $src2, uxth" %}
12957
12958 ins_encode %{
12959 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12960 as_Register($src2$$reg), ext::uxth);
12961 %}
12962 ins_pipe(ialu_reg_reg);
12963 %}
12964
12965 // This pattern is automatically generated from aarch64_ad.m4
12966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12967 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12968 %{
12969 match(Set dst (AddL src1 (AndL src2 mask)));
12970 ins_cost(INSN_COST);
12971 format %{ "add $dst, $src1, $src2, uxtb" %}
12972
12973 ins_encode %{
12974 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12975 as_Register($src2$$reg), ext::uxtb);
12976 %}
12977 ins_pipe(ialu_reg_reg);
12978 %}
12979
12980 // This pattern is automatically generated from aarch64_ad.m4
12981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12982 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12983 %{
12984 match(Set dst (AddL src1 (AndL src2 mask)));
12985 ins_cost(INSN_COST);
12986 format %{ "add $dst, $src1, $src2, uxth" %}
12987
12988 ins_encode %{
12989 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12990 as_Register($src2$$reg), ext::uxth);
12991 %}
12992 ins_pipe(ialu_reg_reg);
12993 %}
12994
12995 // This pattern is automatically generated from aarch64_ad.m4
12996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12997 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12998 %{
12999 match(Set dst (AddL src1 (AndL src2 mask)));
13000 ins_cost(INSN_COST);
13001 format %{ "add $dst, $src1, $src2, uxtw" %}
13002
13003 ins_encode %{
13004 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13005 as_Register($src2$$reg), ext::uxtw);
13006 %}
13007 ins_pipe(ialu_reg_reg);
13008 %}
13009
13010 // This pattern is automatically generated from aarch64_ad.m4
13011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13012 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13013 %{
13014 match(Set dst (SubI src1 (AndI src2 mask)));
13015 ins_cost(INSN_COST);
13016 format %{ "subw $dst, $src1, $src2, uxtb" %}
13017
13018 ins_encode %{
13019 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13020 as_Register($src2$$reg), ext::uxtb);
13021 %}
13022 ins_pipe(ialu_reg_reg);
13023 %}
13024
13025 // This pattern is automatically generated from aarch64_ad.m4
13026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13027 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13028 %{
13029 match(Set dst (SubI src1 (AndI src2 mask)));
13030 ins_cost(INSN_COST);
13031 format %{ "subw $dst, $src1, $src2, uxth" %}
13032
13033 ins_encode %{
13034 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13035 as_Register($src2$$reg), ext::uxth);
13036 %}
13037 ins_pipe(ialu_reg_reg);
13038 %}
13039
13040 // This pattern is automatically generated from aarch64_ad.m4
13041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13042 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13043 %{
13044 match(Set dst (SubL src1 (AndL src2 mask)));
13045 ins_cost(INSN_COST);
13046 format %{ "sub $dst, $src1, $src2, uxtb" %}
13047
13048 ins_encode %{
13049 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13050 as_Register($src2$$reg), ext::uxtb);
13051 %}
13052 ins_pipe(ialu_reg_reg);
13053 %}
13054
13055 // This pattern is automatically generated from aarch64_ad.m4
13056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13057 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13058 %{
13059 match(Set dst (SubL src1 (AndL src2 mask)));
13060 ins_cost(INSN_COST);
13061 format %{ "sub $dst, $src1, $src2, uxth" %}
13062
13063 ins_encode %{
13064 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13065 as_Register($src2$$reg), ext::uxth);
13066 %}
13067 ins_pipe(ialu_reg_reg);
13068 %}
13069
13070 // This pattern is automatically generated from aarch64_ad.m4
13071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13072 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13073 %{
13074 match(Set dst (SubL src1 (AndL src2 mask)));
13075 ins_cost(INSN_COST);
13076 format %{ "sub $dst, $src1, $src2, uxtw" %}
13077
13078 ins_encode %{
13079 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13080 as_Register($src2$$reg), ext::uxtw);
13081 %}
13082 ins_pipe(ialu_reg_reg);
13083 %}
13084
13085
13086 // This pattern is automatically generated from aarch64_ad.m4
13087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13088 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13089 %{
13090 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13091 ins_cost(1.9 * INSN_COST);
13092 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13093
13094 ins_encode %{
13095 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13096 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13097 %}
13098 ins_pipe(ialu_reg_reg_shift);
13099 %}
13100
13101 // This pattern is automatically generated from aarch64_ad.m4
13102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13103 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13104 %{
13105 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13106 ins_cost(1.9 * INSN_COST);
13107 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13108
13109 ins_encode %{
13110 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13111 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13112 %}
13113 ins_pipe(ialu_reg_reg_shift);
13114 %}
13115
13116 // This pattern is automatically generated from aarch64_ad.m4
13117 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13118 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13119 %{
13120 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13121 ins_cost(1.9 * INSN_COST);
13122 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13123
13124 ins_encode %{
13125 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13126 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13127 %}
13128 ins_pipe(ialu_reg_reg_shift);
13129 %}
13130
13131 // This pattern is automatically generated from aarch64_ad.m4
13132 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13133 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13134 %{
13135 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13136 ins_cost(1.9 * INSN_COST);
13137 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13138
13139 ins_encode %{
13140 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13141 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13142 %}
13143 ins_pipe(ialu_reg_reg_shift);
13144 %}
13145
13146 // This pattern is automatically generated from aarch64_ad.m4
13147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13148 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13149 %{
13150 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13151 ins_cost(1.9 * INSN_COST);
13152 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13153
13154 ins_encode %{
13155 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13156 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13157 %}
13158 ins_pipe(ialu_reg_reg_shift);
13159 %}
13160
13161 // This pattern is automatically generated from aarch64_ad.m4
13162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13163 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13164 %{
13165 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13166 ins_cost(1.9 * INSN_COST);
13167 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13168
13169 ins_encode %{
13170 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13171 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13172 %}
13173 ins_pipe(ialu_reg_reg_shift);
13174 %}
13175
13176 // This pattern is automatically generated from aarch64_ad.m4
13177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13178 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13179 %{
13180 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13181 ins_cost(1.9 * INSN_COST);
13182 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13183
13184 ins_encode %{
13185 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13186 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13187 %}
13188 ins_pipe(ialu_reg_reg_shift);
13189 %}
13190
13191 // This pattern is automatically generated from aarch64_ad.m4
13192 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13193 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13194 %{
13195 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13196 ins_cost(1.9 * INSN_COST);
13197 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13198
13199 ins_encode %{
13200 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13201 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13202 %}
13203 ins_pipe(ialu_reg_reg_shift);
13204 %}
13205
13206 // This pattern is automatically generated from aarch64_ad.m4
13207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13208 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13209 %{
13210 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13211 ins_cost(1.9 * INSN_COST);
13212 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13213
13214 ins_encode %{
13215 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13216 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13217 %}
13218 ins_pipe(ialu_reg_reg_shift);
13219 %}
13220
13221 // This pattern is automatically generated from aarch64_ad.m4
13222 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13223 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13224 %{
13225 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13226 ins_cost(1.9 * INSN_COST);
13227 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13228
13229 ins_encode %{
13230 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13231 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13232 %}
13233 ins_pipe(ialu_reg_reg_shift);
13234 %}
13235
13236 // This pattern is automatically generated from aarch64_ad.m4
13237 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13238 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13239 %{
13240 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13241 ins_cost(1.9 * INSN_COST);
13242 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13243
13244 ins_encode %{
13245 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13246 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13247 %}
13248 ins_pipe(ialu_reg_reg_shift);
13249 %}
13250
13251 // This pattern is automatically generated from aarch64_ad.m4
13252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13253 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13254 %{
13255 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13256 ins_cost(1.9 * INSN_COST);
13257 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13258
13259 ins_encode %{
13260 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13261 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13262 %}
13263 ins_pipe(ialu_reg_reg_shift);
13264 %}
13265
13266 // This pattern is automatically generated from aarch64_ad.m4
13267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13268 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13269 %{
13270 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13271 ins_cost(1.9 * INSN_COST);
13272 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13273
13274 ins_encode %{
13275 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13276 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13277 %}
13278 ins_pipe(ialu_reg_reg_shift);
13279 %}
13280
13281 // This pattern is automatically generated from aarch64_ad.m4
13282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13283 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13284 %{
13285 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13286 ins_cost(1.9 * INSN_COST);
13287 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13288
13289 ins_encode %{
13290 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13291 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13292 %}
13293 ins_pipe(ialu_reg_reg_shift);
13294 %}
13295
13296 // This pattern is automatically generated from aarch64_ad.m4
13297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13298 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13299 %{
13300 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13301 ins_cost(1.9 * INSN_COST);
13302 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13303
13304 ins_encode %{
13305 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13306 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13307 %}
13308 ins_pipe(ialu_reg_reg_shift);
13309 %}
13310
13311 // This pattern is automatically generated from aarch64_ad.m4
13312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13313 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13314 %{
13315 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13316 ins_cost(1.9 * INSN_COST);
13317 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13318
13319 ins_encode %{
13320 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13321 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13322 %}
13323 ins_pipe(ialu_reg_reg_shift);
13324 %}
13325
13326 // This pattern is automatically generated from aarch64_ad.m4
13327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13328 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13329 %{
13330 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13331 ins_cost(1.9 * INSN_COST);
13332 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13333
13334 ins_encode %{
13335 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13336 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13337 %}
13338 ins_pipe(ialu_reg_reg_shift);
13339 %}
13340
13341 // This pattern is automatically generated from aarch64_ad.m4
13342 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13343 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13344 %{
13345 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13346 ins_cost(1.9 * INSN_COST);
13347 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13348
13349 ins_encode %{
13350 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13351 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13352 %}
13353 ins_pipe(ialu_reg_reg_shift);
13354 %}
13355
13356 // This pattern is automatically generated from aarch64_ad.m4
13357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13358 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13359 %{
13360 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13361 ins_cost(1.9 * INSN_COST);
13362 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13363
13364 ins_encode %{
13365 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13366 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13367 %}
13368 ins_pipe(ialu_reg_reg_shift);
13369 %}
13370
13371 // This pattern is automatically generated from aarch64_ad.m4
13372 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13373 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13374 %{
13375 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13376 ins_cost(1.9 * INSN_COST);
13377 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13378
13379 ins_encode %{
13380 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13381 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13382 %}
13383 ins_pipe(ialu_reg_reg_shift);
13384 %}
13385
13386 // This pattern is automatically generated from aarch64_ad.m4
13387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13388 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13389 %{
13390 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13391 ins_cost(1.9 * INSN_COST);
13392 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13393
13394 ins_encode %{
13395 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13396 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13397 %}
13398 ins_pipe(ialu_reg_reg_shift);
13399 %}
13400
13401 // This pattern is automatically generated from aarch64_ad.m4
13402 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13403 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13404 %{
13405 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13406 ins_cost(1.9 * INSN_COST);
13407 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13408
13409 ins_encode %{
13410 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13411 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13412 %}
13413 ins_pipe(ialu_reg_reg_shift);
13414 %}
13415
13416 // This pattern is automatically generated from aarch64_ad.m4
13417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13418 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13419 %{
13420 effect(DEF dst, USE src1, USE src2, USE cr);
13421 ins_cost(INSN_COST * 2);
13422 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13423
13424 ins_encode %{
13425 __ cselw($dst$$Register,
13426 $src1$$Register,
13427 $src2$$Register,
13428 Assembler::LT);
13429 %}
13430 ins_pipe(icond_reg_reg);
13431 %}
13432
13433 // This pattern is automatically generated from aarch64_ad.m4
13434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13435 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13436 %{
13437 effect(DEF dst, USE src1, USE src2, USE cr);
13438 ins_cost(INSN_COST * 2);
13439 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13440
13441 ins_encode %{
13442 __ cselw($dst$$Register,
13443 $src1$$Register,
13444 $src2$$Register,
13445 Assembler::GT);
13446 %}
13447 ins_pipe(icond_reg_reg);
13448 %}
13449
13450 // This pattern is automatically generated from aarch64_ad.m4
13451 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13452 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13453 %{
13454 effect(DEF dst, USE src1, USE cr);
13455 ins_cost(INSN_COST * 2);
13456 format %{ "cselw $dst, $src1, zr lt\t" %}
13457
13458 ins_encode %{
13459 __ cselw($dst$$Register,
13460 $src1$$Register,
13461 zr,
13462 Assembler::LT);
13463 %}
13464 ins_pipe(icond_reg);
13465 %}
13466
13467 // This pattern is automatically generated from aarch64_ad.m4
13468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13469 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13470 %{
13471 effect(DEF dst, USE src1, USE cr);
13472 ins_cost(INSN_COST * 2);
13473 format %{ "cselw $dst, $src1, zr gt\t" %}
13474
13475 ins_encode %{
13476 __ cselw($dst$$Register,
13477 $src1$$Register,
13478 zr,
13479 Assembler::GT);
13480 %}
13481 ins_pipe(icond_reg);
13482 %}
13483
13484 // This pattern is automatically generated from aarch64_ad.m4
13485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13486 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13487 %{
13488 effect(DEF dst, USE src1, USE cr);
13489 ins_cost(INSN_COST * 2);
13490 format %{ "csincw $dst, $src1, zr le\t" %}
13491
13492 ins_encode %{
13493 __ csincw($dst$$Register,
13494 $src1$$Register,
13495 zr,
13496 Assembler::LE);
13497 %}
13498 ins_pipe(icond_reg);
13499 %}
13500
13501 // This pattern is automatically generated from aarch64_ad.m4
13502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13503 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13504 %{
13505 effect(DEF dst, USE src1, USE cr);
13506 ins_cost(INSN_COST * 2);
13507 format %{ "csincw $dst, $src1, zr gt\t" %}
13508
13509 ins_encode %{
13510 __ csincw($dst$$Register,
13511 $src1$$Register,
13512 zr,
13513 Assembler::GT);
13514 %}
13515 ins_pipe(icond_reg);
13516 %}
13517
13518 // This pattern is automatically generated from aarch64_ad.m4
13519 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13520 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13521 %{
13522 effect(DEF dst, USE src1, USE cr);
13523 ins_cost(INSN_COST * 2);
13524 format %{ "csinvw $dst, $src1, zr lt\t" %}
13525
13526 ins_encode %{
13527 __ csinvw($dst$$Register,
13528 $src1$$Register,
13529 zr,
13530 Assembler::LT);
13531 %}
13532 ins_pipe(icond_reg);
13533 %}
13534
13535 // This pattern is automatically generated from aarch64_ad.m4
13536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13537 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13538 %{
13539 effect(DEF dst, USE src1, USE cr);
13540 ins_cost(INSN_COST * 2);
13541 format %{ "csinvw $dst, $src1, zr ge\t" %}
13542
13543 ins_encode %{
13544 __ csinvw($dst$$Register,
13545 $src1$$Register,
13546 zr,
13547 Assembler::GE);
13548 %}
13549 ins_pipe(icond_reg);
13550 %}
13551
13552 // This pattern is automatically generated from aarch64_ad.m4
13553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13554 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13555 %{
13556 match(Set dst (MinI src imm));
13557 ins_cost(INSN_COST * 3);
13558 expand %{
13559 rFlagsReg cr;
13560 compI_reg_imm0(cr, src);
13561 cmovI_reg_imm0_lt(dst, src, cr);
13562 %}
13563 %}
13564
13565 // This pattern is automatically generated from aarch64_ad.m4
13566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13567 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13568 %{
13569 match(Set dst (MinI imm src));
13570 ins_cost(INSN_COST * 3);
13571 expand %{
13572 rFlagsReg cr;
13573 compI_reg_imm0(cr, src);
13574 cmovI_reg_imm0_lt(dst, src, cr);
13575 %}
13576 %}
13577
13578 // This pattern is automatically generated from aarch64_ad.m4
13579 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13580 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13581 %{
13582 match(Set dst (MinI src imm));
13583 ins_cost(INSN_COST * 3);
13584 expand %{
13585 rFlagsReg cr;
13586 compI_reg_imm0(cr, src);
13587 cmovI_reg_imm1_le(dst, src, cr);
13588 %}
13589 %}
13590
13591 // This pattern is automatically generated from aarch64_ad.m4
13592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13593 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13594 %{
13595 match(Set dst (MinI imm src));
13596 ins_cost(INSN_COST * 3);
13597 expand %{
13598 rFlagsReg cr;
13599 compI_reg_imm0(cr, src);
13600 cmovI_reg_imm1_le(dst, src, cr);
13601 %}
13602 %}
13603
13604 // This pattern is automatically generated from aarch64_ad.m4
13605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13606 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13607 %{
13608 match(Set dst (MinI src imm));
13609 ins_cost(INSN_COST * 3);
13610 expand %{
13611 rFlagsReg cr;
13612 compI_reg_imm0(cr, src);
13613 cmovI_reg_immM1_lt(dst, src, cr);
13614 %}
13615 %}
13616
13617 // This pattern is automatically generated from aarch64_ad.m4
13618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13619 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13620 %{
13621 match(Set dst (MinI imm src));
13622 ins_cost(INSN_COST * 3);
13623 expand %{
13624 rFlagsReg cr;
13625 compI_reg_imm0(cr, src);
13626 cmovI_reg_immM1_lt(dst, src, cr);
13627 %}
13628 %}
13629
13630 // This pattern is automatically generated from aarch64_ad.m4
13631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13632 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13633 %{
13634 match(Set dst (MaxI src imm));
13635 ins_cost(INSN_COST * 3);
13636 expand %{
13637 rFlagsReg cr;
13638 compI_reg_imm0(cr, src);
13639 cmovI_reg_imm0_gt(dst, src, cr);
13640 %}
13641 %}
13642
13643 // This pattern is automatically generated from aarch64_ad.m4
13644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13645 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13646 %{
13647 match(Set dst (MaxI imm src));
13648 ins_cost(INSN_COST * 3);
13649 expand %{
13650 rFlagsReg cr;
13651 compI_reg_imm0(cr, src);
13652 cmovI_reg_imm0_gt(dst, src, cr);
13653 %}
13654 %}
13655
13656 // This pattern is automatically generated from aarch64_ad.m4
13657 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13658 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13659 %{
13660 match(Set dst (MaxI src imm));
13661 ins_cost(INSN_COST * 3);
13662 expand %{
13663 rFlagsReg cr;
13664 compI_reg_imm0(cr, src);
13665 cmovI_reg_imm1_gt(dst, src, cr);
13666 %}
13667 %}
13668
13669 // This pattern is automatically generated from aarch64_ad.m4
13670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13671 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13672 %{
13673 match(Set dst (MaxI imm src));
13674 ins_cost(INSN_COST * 3);
13675 expand %{
13676 rFlagsReg cr;
13677 compI_reg_imm0(cr, src);
13678 cmovI_reg_imm1_gt(dst, src, cr);
13679 %}
13680 %}
13681
13682 // This pattern is automatically generated from aarch64_ad.m4
13683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13684 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13685 %{
13686 match(Set dst (MaxI src imm));
13687 ins_cost(INSN_COST * 3);
13688 expand %{
13689 rFlagsReg cr;
13690 compI_reg_imm0(cr, src);
13691 cmovI_reg_immM1_ge(dst, src, cr);
13692 %}
13693 %}
13694
13695 // This pattern is automatically generated from aarch64_ad.m4
13696 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13697 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13698 %{
13699 match(Set dst (MaxI imm src));
13700 ins_cost(INSN_COST * 3);
13701 expand %{
13702 rFlagsReg cr;
13703 compI_reg_imm0(cr, src);
13704 cmovI_reg_immM1_ge(dst, src, cr);
13705 %}
13706 %}
13707
13708 // This pattern is automatically generated from aarch64_ad.m4
13709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13710 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13711 %{
13712 match(Set dst (ReverseI src));
13713 ins_cost(INSN_COST);
13714 format %{ "rbitw $dst, $src" %}
13715 ins_encode %{
13716 __ rbitw($dst$$Register, $src$$Register);
13717 %}
13718 ins_pipe(ialu_reg);
13719 %}
13720
13721 // This pattern is automatically generated from aarch64_ad.m4
13722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13723 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13724 %{
13725 match(Set dst (ReverseL src));
13726 ins_cost(INSN_COST);
13727 format %{ "rbit $dst, $src" %}
13728 ins_encode %{
13729 __ rbit($dst$$Register, $src$$Register);
13730 %}
13731 ins_pipe(ialu_reg);
13732 %}
13733
13734
13735 // END This section of the file is automatically generated. Do not edit --------------
13736
13737
13738 // ============================================================================
13739 // Floating Point Arithmetic Instructions
13740
13741 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13742 match(Set dst (AddHF src1 src2));
13743 format %{ "faddh $dst, $src1, $src2" %}
13744 ins_encode %{
13745 __ faddh($dst$$FloatRegister,
13746 $src1$$FloatRegister,
13747 $src2$$FloatRegister);
13748 %}
13749 ins_pipe(fp_dop_reg_reg_s);
13750 %}
13751
13752 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13753 match(Set dst (AddF src1 src2));
13754
13755 ins_cost(INSN_COST * 5);
13756 format %{ "fadds $dst, $src1, $src2" %}
13757
13758 ins_encode %{
13759 __ fadds(as_FloatRegister($dst$$reg),
13760 as_FloatRegister($src1$$reg),
13761 as_FloatRegister($src2$$reg));
13762 %}
13763
13764 ins_pipe(fp_dop_reg_reg_s);
13765 %}
13766
13767 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13768 match(Set dst (AddD src1 src2));
13769
13770 ins_cost(INSN_COST * 5);
13771 format %{ "faddd $dst, $src1, $src2" %}
13772
13773 ins_encode %{
13774 __ faddd(as_FloatRegister($dst$$reg),
13775 as_FloatRegister($src1$$reg),
13776 as_FloatRegister($src2$$reg));
13777 %}
13778
13779 ins_pipe(fp_dop_reg_reg_d);
13780 %}
13781
13782 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13783 match(Set dst (SubHF src1 src2));
13784 format %{ "fsubh $dst, $src1, $src2" %}
13785 ins_encode %{
13786 __ fsubh($dst$$FloatRegister,
13787 $src1$$FloatRegister,
13788 $src2$$FloatRegister);
13789 %}
13790 ins_pipe(fp_dop_reg_reg_s);
13791 %}
13792
13793 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13794 match(Set dst (SubF src1 src2));
13795
13796 ins_cost(INSN_COST * 5);
13797 format %{ "fsubs $dst, $src1, $src2" %}
13798
13799 ins_encode %{
13800 __ fsubs(as_FloatRegister($dst$$reg),
13801 as_FloatRegister($src1$$reg),
13802 as_FloatRegister($src2$$reg));
13803 %}
13804
13805 ins_pipe(fp_dop_reg_reg_s);
13806 %}
13807
13808 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13809 match(Set dst (SubD src1 src2));
13810
13811 ins_cost(INSN_COST * 5);
13812 format %{ "fsubd $dst, $src1, $src2" %}
13813
13814 ins_encode %{
13815 __ fsubd(as_FloatRegister($dst$$reg),
13816 as_FloatRegister($src1$$reg),
13817 as_FloatRegister($src2$$reg));
13818 %}
13819
13820 ins_pipe(fp_dop_reg_reg_d);
13821 %}
13822
13823 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13824 match(Set dst (MulHF src1 src2));
13825 format %{ "fmulh $dst, $src1, $src2" %}
13826 ins_encode %{
13827 __ fmulh($dst$$FloatRegister,
13828 $src1$$FloatRegister,
13829 $src2$$FloatRegister);
13830 %}
13831 ins_pipe(fp_dop_reg_reg_s);
13832 %}
13833
13834 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13835 match(Set dst (MulF src1 src2));
13836
13837 ins_cost(INSN_COST * 6);
13838 format %{ "fmuls $dst, $src1, $src2" %}
13839
13840 ins_encode %{
13841 __ fmuls(as_FloatRegister($dst$$reg),
13842 as_FloatRegister($src1$$reg),
13843 as_FloatRegister($src2$$reg));
13844 %}
13845
13846 ins_pipe(fp_dop_reg_reg_s);
13847 %}
13848
13849 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13850 match(Set dst (MulD src1 src2));
13851
13852 ins_cost(INSN_COST * 6);
13853 format %{ "fmuld $dst, $src1, $src2" %}
13854
13855 ins_encode %{
13856 __ fmuld(as_FloatRegister($dst$$reg),
13857 as_FloatRegister($src1$$reg),
13858 as_FloatRegister($src2$$reg));
13859 %}
13860
13861 ins_pipe(fp_dop_reg_reg_d);
13862 %}
13863
13864 // src1 * src2 + src3 (half-precision float)
13865 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13866 match(Set dst (FmaHF src3 (Binary src1 src2)));
13867 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13868 ins_encode %{
13869 assert(UseFMA, "Needs FMA instructions support.");
13870 __ fmaddh($dst$$FloatRegister,
13871 $src1$$FloatRegister,
13872 $src2$$FloatRegister,
13873 $src3$$FloatRegister);
13874 %}
13875 ins_pipe(pipe_class_default);
13876 %}
13877
13878 // src1 * src2 + src3
13879 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13880 match(Set dst (FmaF src3 (Binary src1 src2)));
13881
13882 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13883
13884 ins_encode %{
13885 assert(UseFMA, "Needs FMA instructions support.");
13886 __ fmadds(as_FloatRegister($dst$$reg),
13887 as_FloatRegister($src1$$reg),
13888 as_FloatRegister($src2$$reg),
13889 as_FloatRegister($src3$$reg));
13890 %}
13891
13892 ins_pipe(pipe_class_default);
13893 %}
13894
13895 // src1 * src2 + src3
13896 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13897 match(Set dst (FmaD src3 (Binary src1 src2)));
13898
13899 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13900
13901 ins_encode %{
13902 assert(UseFMA, "Needs FMA instructions support.");
13903 __ fmaddd(as_FloatRegister($dst$$reg),
13904 as_FloatRegister($src1$$reg),
13905 as_FloatRegister($src2$$reg),
13906 as_FloatRegister($src3$$reg));
13907 %}
13908
13909 ins_pipe(pipe_class_default);
13910 %}
13911
13912 // src1 * (-src2) + src3
13913 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13914 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13915 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13916
13917 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13918
13919 ins_encode %{
13920 assert(UseFMA, "Needs FMA instructions support.");
13921 __ fmsubs(as_FloatRegister($dst$$reg),
13922 as_FloatRegister($src1$$reg),
13923 as_FloatRegister($src2$$reg),
13924 as_FloatRegister($src3$$reg));
13925 %}
13926
13927 ins_pipe(pipe_class_default);
13928 %}
13929
13930 // src1 * (-src2) + src3
13931 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13932 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13933 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13934
13935 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13936
13937 ins_encode %{
13938 assert(UseFMA, "Needs FMA instructions support.");
13939 __ fmsubd(as_FloatRegister($dst$$reg),
13940 as_FloatRegister($src1$$reg),
13941 as_FloatRegister($src2$$reg),
13942 as_FloatRegister($src3$$reg));
13943 %}
13944
13945 ins_pipe(pipe_class_default);
13946 %}
13947
13948 // src1 * (-src2) - src3
13949 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13950 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13951 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13952
13953 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13954
13955 ins_encode %{
13956 assert(UseFMA, "Needs FMA instructions support.");
13957 __ fnmadds(as_FloatRegister($dst$$reg),
13958 as_FloatRegister($src1$$reg),
13959 as_FloatRegister($src2$$reg),
13960 as_FloatRegister($src3$$reg));
13961 %}
13962
13963 ins_pipe(pipe_class_default);
13964 %}
13965
13966 // src1 * (-src2) - src3
13967 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13968 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13969 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13970
13971 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13972
13973 ins_encode %{
13974 assert(UseFMA, "Needs FMA instructions support.");
13975 __ fnmaddd(as_FloatRegister($dst$$reg),
13976 as_FloatRegister($src1$$reg),
13977 as_FloatRegister($src2$$reg),
13978 as_FloatRegister($src3$$reg));
13979 %}
13980
13981 ins_pipe(pipe_class_default);
13982 %}
13983
13984 // src1 * src2 - src3
13985 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13986 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13987
13988 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13989
13990 ins_encode %{
13991 assert(UseFMA, "Needs FMA instructions support.");
13992 __ fnmsubs(as_FloatRegister($dst$$reg),
13993 as_FloatRegister($src1$$reg),
13994 as_FloatRegister($src2$$reg),
13995 as_FloatRegister($src3$$reg));
13996 %}
13997
13998 ins_pipe(pipe_class_default);
13999 %}
14000
14001 // src1 * src2 - src3
14002 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14003 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14004
14005 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14006
14007 ins_encode %{
14008 assert(UseFMA, "Needs FMA instructions support.");
14009 // n.b. insn name should be fnmsubd
14010 __ fnmsub(as_FloatRegister($dst$$reg),
14011 as_FloatRegister($src1$$reg),
14012 as_FloatRegister($src2$$reg),
14013 as_FloatRegister($src3$$reg));
14014 %}
14015
14016 ins_pipe(pipe_class_default);
14017 %}
14018
14019 // Math.max(HH)H (half-precision float)
14020 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14021 match(Set dst (MaxHF src1 src2));
14022 format %{ "fmaxh $dst, $src1, $src2" %}
14023 ins_encode %{
14024 __ fmaxh($dst$$FloatRegister,
14025 $src1$$FloatRegister,
14026 $src2$$FloatRegister);
14027 %}
14028 ins_pipe(fp_dop_reg_reg_s);
14029 %}
14030
14031 // Math.min(HH)H (half-precision float)
14032 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14033 match(Set dst (MinHF src1 src2));
14034 format %{ "fminh $dst, $src1, $src2" %}
14035 ins_encode %{
14036 __ fminh($dst$$FloatRegister,
14037 $src1$$FloatRegister,
14038 $src2$$FloatRegister);
14039 %}
14040 ins_pipe(fp_dop_reg_reg_s);
14041 %}
14042
14043 // Math.max(FF)F
14044 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14045 match(Set dst (MaxF src1 src2));
14046
14047 format %{ "fmaxs $dst, $src1, $src2" %}
14048 ins_encode %{
14049 __ fmaxs(as_FloatRegister($dst$$reg),
14050 as_FloatRegister($src1$$reg),
14051 as_FloatRegister($src2$$reg));
14052 %}
14053
14054 ins_pipe(fp_dop_reg_reg_s);
14055 %}
14056
14057 // Math.min(FF)F
14058 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14059 match(Set dst (MinF src1 src2));
14060
14061 format %{ "fmins $dst, $src1, $src2" %}
14062 ins_encode %{
14063 __ fmins(as_FloatRegister($dst$$reg),
14064 as_FloatRegister($src1$$reg),
14065 as_FloatRegister($src2$$reg));
14066 %}
14067
14068 ins_pipe(fp_dop_reg_reg_s);
14069 %}
14070
14071 // Math.max(DD)D
14072 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14073 match(Set dst (MaxD src1 src2));
14074
14075 format %{ "fmaxd $dst, $src1, $src2" %}
14076 ins_encode %{
14077 __ fmaxd(as_FloatRegister($dst$$reg),
14078 as_FloatRegister($src1$$reg),
14079 as_FloatRegister($src2$$reg));
14080 %}
14081
14082 ins_pipe(fp_dop_reg_reg_d);
14083 %}
14084
14085 // Math.min(DD)D
14086 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14087 match(Set dst (MinD src1 src2));
14088
14089 format %{ "fmind $dst, $src1, $src2" %}
14090 ins_encode %{
14091 __ fmind(as_FloatRegister($dst$$reg),
14092 as_FloatRegister($src1$$reg),
14093 as_FloatRegister($src2$$reg));
14094 %}
14095
14096 ins_pipe(fp_dop_reg_reg_d);
14097 %}
14098
14099 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14100 match(Set dst (DivHF src1 src2));
14101 format %{ "fdivh $dst, $src1, $src2" %}
14102 ins_encode %{
14103 __ fdivh($dst$$FloatRegister,
14104 $src1$$FloatRegister,
14105 $src2$$FloatRegister);
14106 %}
14107 ins_pipe(fp_div_s);
14108 %}
14109
14110 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14111 match(Set dst (DivF src1 src2));
14112
14113 ins_cost(INSN_COST * 18);
14114 format %{ "fdivs $dst, $src1, $src2" %}
14115
14116 ins_encode %{
14117 __ fdivs(as_FloatRegister($dst$$reg),
14118 as_FloatRegister($src1$$reg),
14119 as_FloatRegister($src2$$reg));
14120 %}
14121
14122 ins_pipe(fp_div_s);
14123 %}
14124
14125 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14126 match(Set dst (DivD src1 src2));
14127
14128 ins_cost(INSN_COST * 32);
14129 format %{ "fdivd $dst, $src1, $src2" %}
14130
14131 ins_encode %{
14132 __ fdivd(as_FloatRegister($dst$$reg),
14133 as_FloatRegister($src1$$reg),
14134 as_FloatRegister($src2$$reg));
14135 %}
14136
14137 ins_pipe(fp_div_d);
14138 %}
14139
14140 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14141 match(Set dst (NegF src));
14142
14143 ins_cost(INSN_COST * 3);
14144 format %{ "fneg $dst, $src" %}
14145
14146 ins_encode %{
14147 __ fnegs(as_FloatRegister($dst$$reg),
14148 as_FloatRegister($src$$reg));
14149 %}
14150
14151 ins_pipe(fp_uop_s);
14152 %}
14153
14154 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14155 match(Set dst (NegD src));
14156
14157 ins_cost(INSN_COST * 3);
14158 format %{ "fnegd $dst, $src" %}
14159
14160 ins_encode %{
14161 __ fnegd(as_FloatRegister($dst$$reg),
14162 as_FloatRegister($src$$reg));
14163 %}
14164
14165 ins_pipe(fp_uop_d);
14166 %}
14167
14168 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14169 %{
14170 match(Set dst (AbsI src));
14171
14172 effect(KILL cr);
14173 ins_cost(INSN_COST * 2);
14174 format %{ "cmpw $src, zr\n\t"
14175 "cnegw $dst, $src, Assembler::LT\t# int abs"
14176 %}
14177
14178 ins_encode %{
14179 __ cmpw(as_Register($src$$reg), zr);
14180 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14181 %}
14182 ins_pipe(pipe_class_default);
14183 %}
14184
14185 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14186 %{
14187 match(Set dst (AbsL src));
14188
14189 effect(KILL cr);
14190 ins_cost(INSN_COST * 2);
14191 format %{ "cmp $src, zr\n\t"
14192 "cneg $dst, $src, Assembler::LT\t# long abs"
14193 %}
14194
14195 ins_encode %{
14196 __ cmp(as_Register($src$$reg), zr);
14197 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14198 %}
14199 ins_pipe(pipe_class_default);
14200 %}
14201
14202 instruct absF_reg(vRegF dst, vRegF src) %{
14203 match(Set dst (AbsF src));
14204
14205 ins_cost(INSN_COST * 3);
14206 format %{ "fabss $dst, $src" %}
14207 ins_encode %{
14208 __ fabss(as_FloatRegister($dst$$reg),
14209 as_FloatRegister($src$$reg));
14210 %}
14211
14212 ins_pipe(fp_uop_s);
14213 %}
14214
14215 instruct absD_reg(vRegD dst, vRegD src) %{
14216 match(Set dst (AbsD src));
14217
14218 ins_cost(INSN_COST * 3);
14219 format %{ "fabsd $dst, $src" %}
14220 ins_encode %{
14221 __ fabsd(as_FloatRegister($dst$$reg),
14222 as_FloatRegister($src$$reg));
14223 %}
14224
14225 ins_pipe(fp_uop_d);
14226 %}
14227
14228 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14229 match(Set dst (AbsF (SubF src1 src2)));
14230
14231 ins_cost(INSN_COST * 3);
14232 format %{ "fabds $dst, $src1, $src2" %}
14233 ins_encode %{
14234 __ fabds(as_FloatRegister($dst$$reg),
14235 as_FloatRegister($src1$$reg),
14236 as_FloatRegister($src2$$reg));
14237 %}
14238
14239 ins_pipe(fp_uop_s);
14240 %}
14241
14242 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14243 match(Set dst (AbsD (SubD src1 src2)));
14244
14245 ins_cost(INSN_COST * 3);
14246 format %{ "fabdd $dst, $src1, $src2" %}
14247 ins_encode %{
14248 __ fabdd(as_FloatRegister($dst$$reg),
14249 as_FloatRegister($src1$$reg),
14250 as_FloatRegister($src2$$reg));
14251 %}
14252
14253 ins_pipe(fp_uop_d);
14254 %}
14255
14256 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14257 match(Set dst (SqrtD src));
14258
14259 ins_cost(INSN_COST * 50);
14260 format %{ "fsqrtd $dst, $src" %}
14261 ins_encode %{
14262 __ fsqrtd(as_FloatRegister($dst$$reg),
14263 as_FloatRegister($src$$reg));
14264 %}
14265
14266 ins_pipe(fp_div_s);
14267 %}
14268
14269 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14270 match(Set dst (SqrtF src));
14271
14272 ins_cost(INSN_COST * 50);
14273 format %{ "fsqrts $dst, $src" %}
14274 ins_encode %{
14275 __ fsqrts(as_FloatRegister($dst$$reg),
14276 as_FloatRegister($src$$reg));
14277 %}
14278
14279 ins_pipe(fp_div_d);
14280 %}
14281
14282 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14283 match(Set dst (SqrtHF src));
14284 format %{ "fsqrth $dst, $src" %}
14285 ins_encode %{
14286 __ fsqrth($dst$$FloatRegister,
14287 $src$$FloatRegister);
14288 %}
14289 ins_pipe(fp_div_s);
14290 %}
14291
14292 // Math.rint, floor, ceil
14293 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14294 match(Set dst (RoundDoubleMode src rmode));
14295 format %{ "frint $dst, $src, $rmode" %}
14296 ins_encode %{
14297 switch ($rmode$$constant) {
14298 case RoundDoubleModeNode::rmode_rint:
14299 __ frintnd(as_FloatRegister($dst$$reg),
14300 as_FloatRegister($src$$reg));
14301 break;
14302 case RoundDoubleModeNode::rmode_floor:
14303 __ frintmd(as_FloatRegister($dst$$reg),
14304 as_FloatRegister($src$$reg));
14305 break;
14306 case RoundDoubleModeNode::rmode_ceil:
14307 __ frintpd(as_FloatRegister($dst$$reg),
14308 as_FloatRegister($src$$reg));
14309 break;
14310 }
14311 %}
14312 ins_pipe(fp_uop_d);
14313 %}
14314
14315 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14316 match(Set dst (CopySignD src1 (Binary src2 zero)));
14317 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14318 format %{ "CopySignD $dst $src1 $src2" %}
14319 ins_encode %{
14320 FloatRegister dst = as_FloatRegister($dst$$reg),
14321 src1 = as_FloatRegister($src1$$reg),
14322 src2 = as_FloatRegister($src2$$reg),
14323 zero = as_FloatRegister($zero$$reg);
14324 __ fnegd(dst, zero);
14325 __ bsl(dst, __ T8B, src2, src1);
14326 %}
14327 ins_pipe(fp_uop_d);
14328 %}
14329
14330 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14331 match(Set dst (CopySignF src1 src2));
14332 effect(TEMP_DEF dst, USE src1, USE src2);
14333 format %{ "CopySignF $dst $src1 $src2" %}
14334 ins_encode %{
14335 FloatRegister dst = as_FloatRegister($dst$$reg),
14336 src1 = as_FloatRegister($src1$$reg),
14337 src2 = as_FloatRegister($src2$$reg);
14338 __ movi(dst, __ T2S, 0x80, 24);
14339 __ bsl(dst, __ T8B, src2, src1);
14340 %}
14341 ins_pipe(fp_uop_d);
14342 %}
14343
14344 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14345 match(Set dst (SignumD src (Binary zero one)));
14346 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14347 format %{ "signumD $dst, $src" %}
14348 ins_encode %{
14349 FloatRegister src = as_FloatRegister($src$$reg),
14350 dst = as_FloatRegister($dst$$reg),
14351 zero = as_FloatRegister($zero$$reg),
14352 one = as_FloatRegister($one$$reg);
14353 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14354 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14355 // Bit selection instruction gets bit from "one" for each enabled bit in
14356 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14357 // NaN the whole "src" will be copied because "dst" is zero. For all other
14358 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14359 // from "src", and all other bits are copied from 1.0.
14360 __ bsl(dst, __ T8B, one, src);
14361 %}
14362 ins_pipe(fp_uop_d);
14363 %}
14364
14365 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14366 match(Set dst (SignumF src (Binary zero one)));
14367 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14368 format %{ "signumF $dst, $src" %}
14369 ins_encode %{
14370 FloatRegister src = as_FloatRegister($src$$reg),
14371 dst = as_FloatRegister($dst$$reg),
14372 zero = as_FloatRegister($zero$$reg),
14373 one = as_FloatRegister($one$$reg);
14374 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14375 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14376 // Bit selection instruction gets bit from "one" for each enabled bit in
14377 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14378 // NaN the whole "src" will be copied because "dst" is zero. For all other
14379 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14380 // from "src", and all other bits are copied from 1.0.
14381 __ bsl(dst, __ T8B, one, src);
14382 %}
14383 ins_pipe(fp_uop_d);
14384 %}
14385
14386 instruct onspinwait() %{
14387 match(OnSpinWait);
14388 ins_cost(INSN_COST);
14389
14390 format %{ "onspinwait" %}
14391
14392 ins_encode %{
14393 __ spin_wait();
14394 %}
14395 ins_pipe(pipe_class_empty);
14396 %}
14397
14398 // ============================================================================
14399 // Logical Instructions
14400
14401 // Integer Logical Instructions
14402
14403 // And Instructions
14404
14405
14406 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14407 match(Set dst (AndI src1 src2));
14408
14409 format %{ "andw $dst, $src1, $src2\t# int" %}
14410
14411 ins_cost(INSN_COST);
14412 ins_encode %{
14413 __ andw(as_Register($dst$$reg),
14414 as_Register($src1$$reg),
14415 as_Register($src2$$reg));
14416 %}
14417
14418 ins_pipe(ialu_reg_reg);
14419 %}
14420
14421 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14422 match(Set dst (AndI src1 src2));
14423
14424 format %{ "andsw $dst, $src1, $src2\t# int" %}
14425
14426 ins_cost(INSN_COST);
14427 ins_encode %{
14428 __ andw(as_Register($dst$$reg),
14429 as_Register($src1$$reg),
14430 (uint64_t)($src2$$constant));
14431 %}
14432
14433 ins_pipe(ialu_reg_imm);
14434 %}
14435
14436 // Or Instructions
14437
14438 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14439 match(Set dst (OrI src1 src2));
14440
14441 format %{ "orrw $dst, $src1, $src2\t# int" %}
14442
14443 ins_cost(INSN_COST);
14444 ins_encode %{
14445 __ orrw(as_Register($dst$$reg),
14446 as_Register($src1$$reg),
14447 as_Register($src2$$reg));
14448 %}
14449
14450 ins_pipe(ialu_reg_reg);
14451 %}
14452
14453 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14454 match(Set dst (OrI src1 src2));
14455
14456 format %{ "orrw $dst, $src1, $src2\t# int" %}
14457
14458 ins_cost(INSN_COST);
14459 ins_encode %{
14460 __ orrw(as_Register($dst$$reg),
14461 as_Register($src1$$reg),
14462 (uint64_t)($src2$$constant));
14463 %}
14464
14465 ins_pipe(ialu_reg_imm);
14466 %}
14467
14468 // Xor Instructions
14469
14470 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14471 match(Set dst (XorI src1 src2));
14472
14473 format %{ "eorw $dst, $src1, $src2\t# int" %}
14474
14475 ins_cost(INSN_COST);
14476 ins_encode %{
14477 __ eorw(as_Register($dst$$reg),
14478 as_Register($src1$$reg),
14479 as_Register($src2$$reg));
14480 %}
14481
14482 ins_pipe(ialu_reg_reg);
14483 %}
14484
14485 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14486 match(Set dst (XorI src1 src2));
14487
14488 format %{ "eorw $dst, $src1, $src2\t# int" %}
14489
14490 ins_cost(INSN_COST);
14491 ins_encode %{
14492 __ eorw(as_Register($dst$$reg),
14493 as_Register($src1$$reg),
14494 (uint64_t)($src2$$constant));
14495 %}
14496
14497 ins_pipe(ialu_reg_imm);
14498 %}
14499
14500 // Long Logical Instructions
14501 // TODO
14502
14503 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14504 match(Set dst (AndL src1 src2));
14505
14506 format %{ "and $dst, $src1, $src2\t# int" %}
14507
14508 ins_cost(INSN_COST);
14509 ins_encode %{
14510 __ andr(as_Register($dst$$reg),
14511 as_Register($src1$$reg),
14512 as_Register($src2$$reg));
14513 %}
14514
14515 ins_pipe(ialu_reg_reg);
14516 %}
14517
14518 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14519 match(Set dst (AndL src1 src2));
14520
14521 format %{ "and $dst, $src1, $src2\t# int" %}
14522
14523 ins_cost(INSN_COST);
14524 ins_encode %{
14525 __ andr(as_Register($dst$$reg),
14526 as_Register($src1$$reg),
14527 (uint64_t)($src2$$constant));
14528 %}
14529
14530 ins_pipe(ialu_reg_imm);
14531 %}
14532
14533 // Or Instructions
14534
14535 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14536 match(Set dst (OrL src1 src2));
14537
14538 format %{ "orr $dst, $src1, $src2\t# int" %}
14539
14540 ins_cost(INSN_COST);
14541 ins_encode %{
14542 __ orr(as_Register($dst$$reg),
14543 as_Register($src1$$reg),
14544 as_Register($src2$$reg));
14545 %}
14546
14547 ins_pipe(ialu_reg_reg);
14548 %}
14549
14550 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14551 match(Set dst (OrL src1 src2));
14552
14553 format %{ "orr $dst, $src1, $src2\t# int" %}
14554
14555 ins_cost(INSN_COST);
14556 ins_encode %{
14557 __ orr(as_Register($dst$$reg),
14558 as_Register($src1$$reg),
14559 (uint64_t)($src2$$constant));
14560 %}
14561
14562 ins_pipe(ialu_reg_imm);
14563 %}
14564
14565 // Xor Instructions
14566
14567 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14568 match(Set dst (XorL src1 src2));
14569
14570 format %{ "eor $dst, $src1, $src2\t# int" %}
14571
14572 ins_cost(INSN_COST);
14573 ins_encode %{
14574 __ eor(as_Register($dst$$reg),
14575 as_Register($src1$$reg),
14576 as_Register($src2$$reg));
14577 %}
14578
14579 ins_pipe(ialu_reg_reg);
14580 %}
14581
14582 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14583 match(Set dst (XorL src1 src2));
14584
14585 ins_cost(INSN_COST);
14586 format %{ "eor $dst, $src1, $src2\t# int" %}
14587
14588 ins_encode %{
14589 __ eor(as_Register($dst$$reg),
14590 as_Register($src1$$reg),
14591 (uint64_t)($src2$$constant));
14592 %}
14593
14594 ins_pipe(ialu_reg_imm);
14595 %}
14596
14597 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14598 %{
14599 match(Set dst (ConvI2L src));
14600
14601 ins_cost(INSN_COST);
14602 format %{ "sxtw $dst, $src\t# i2l" %}
14603 ins_encode %{
14604 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14605 %}
14606 ins_pipe(ialu_reg_shift);
14607 %}
14608
14609 // this pattern occurs in bigmath arithmetic
14610 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14611 %{
14612 match(Set dst (AndL (ConvI2L src) mask));
14613
14614 ins_cost(INSN_COST);
14615 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14616 ins_encode %{
14617 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14618 %}
14619
14620 ins_pipe(ialu_reg_shift);
14621 %}
14622
14623 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14624 match(Set dst (ConvL2I src));
14625
14626 ins_cost(INSN_COST);
14627 format %{ "movw $dst, $src \t// l2i" %}
14628
14629 ins_encode %{
14630 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14631 %}
14632
14633 ins_pipe(ialu_reg);
14634 %}
14635
14636 instruct convD2F_reg(vRegF dst, vRegD src) %{
14637 match(Set dst (ConvD2F src));
14638
14639 ins_cost(INSN_COST * 5);
14640 format %{ "fcvtd $dst, $src \t// d2f" %}
14641
14642 ins_encode %{
14643 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14644 %}
14645
14646 ins_pipe(fp_d2f);
14647 %}
14648
14649 instruct convF2D_reg(vRegD dst, vRegF src) %{
14650 match(Set dst (ConvF2D src));
14651
14652 ins_cost(INSN_COST * 5);
14653 format %{ "fcvts $dst, $src \t// f2d" %}
14654
14655 ins_encode %{
14656 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14657 %}
14658
14659 ins_pipe(fp_f2d);
14660 %}
14661
14662 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14663 match(Set dst (ConvF2I src));
14664
14665 ins_cost(INSN_COST * 5);
14666 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14667
14668 ins_encode %{
14669 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14670 %}
14671
14672 ins_pipe(fp_f2i);
14673 %}
14674
14675 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14676 match(Set dst (ConvF2L src));
14677
14678 ins_cost(INSN_COST * 5);
14679 format %{ "fcvtzs $dst, $src \t// f2l" %}
14680
14681 ins_encode %{
14682 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14683 %}
14684
14685 ins_pipe(fp_f2l);
14686 %}
14687
14688 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14689 match(Set dst (ConvF2HF src));
14690 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14691 "smov $dst, $tmp\t# move result from $tmp to $dst"
14692 %}
14693 effect(TEMP tmp);
14694 ins_encode %{
14695 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14696 %}
14697 ins_pipe(pipe_slow);
14698 %}
14699
14700 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14701 match(Set dst (ConvHF2F src));
14702 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14703 "fcvt $dst, $tmp\t# convert half to single precision"
14704 %}
14705 effect(TEMP tmp);
14706 ins_encode %{
14707 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14708 %}
14709 ins_pipe(pipe_slow);
14710 %}
14711
14712 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14713 match(Set dst (ConvI2F src));
14714
14715 ins_cost(INSN_COST * 5);
14716 format %{ "scvtfws $dst, $src \t// i2f" %}
14717
14718 ins_encode %{
14719 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14720 %}
14721
14722 ins_pipe(fp_i2f);
14723 %}
14724
14725 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14726 match(Set dst (ConvL2F src));
14727
14728 ins_cost(INSN_COST * 5);
14729 format %{ "scvtfs $dst, $src \t// l2f" %}
14730
14731 ins_encode %{
14732 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14733 %}
14734
14735 ins_pipe(fp_l2f);
14736 %}
14737
14738 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14739 match(Set dst (ConvD2I src));
14740
14741 ins_cost(INSN_COST * 5);
14742 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14743
14744 ins_encode %{
14745 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14746 %}
14747
14748 ins_pipe(fp_d2i);
14749 %}
14750
14751 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14752 match(Set dst (ConvD2L src));
14753
14754 ins_cost(INSN_COST * 5);
14755 format %{ "fcvtzd $dst, $src \t// d2l" %}
14756
14757 ins_encode %{
14758 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14759 %}
14760
14761 ins_pipe(fp_d2l);
14762 %}
14763
14764 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14765 match(Set dst (ConvI2D src));
14766
14767 ins_cost(INSN_COST * 5);
14768 format %{ "scvtfwd $dst, $src \t// i2d" %}
14769
14770 ins_encode %{
14771 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14772 %}
14773
14774 ins_pipe(fp_i2d);
14775 %}
14776
14777 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14778 match(Set dst (ConvL2D src));
14779
14780 ins_cost(INSN_COST * 5);
14781 format %{ "scvtfd $dst, $src \t// l2d" %}
14782
14783 ins_encode %{
14784 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14785 %}
14786
14787 ins_pipe(fp_l2d);
14788 %}
14789
14790 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14791 %{
14792 match(Set dst (RoundD src));
14793 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14794 format %{ "java_round_double $dst,$src"%}
14795 ins_encode %{
14796 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14797 as_FloatRegister($ftmp$$reg));
14798 %}
14799 ins_pipe(pipe_slow);
14800 %}
14801
14802 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14803 %{
14804 match(Set dst (RoundF src));
14805 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14806 format %{ "java_round_float $dst,$src"%}
14807 ins_encode %{
14808 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14809 as_FloatRegister($ftmp$$reg));
14810 %}
14811 ins_pipe(pipe_slow);
14812 %}
14813
14814 // stack <-> reg and reg <-> reg shuffles with no conversion
14815
14816 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14817
14818 match(Set dst (MoveF2I src));
14819
14820 effect(DEF dst, USE src);
14821
14822 ins_cost(4 * INSN_COST);
14823
14824 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14825
14826 ins_encode %{
14827 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14828 %}
14829
14830 ins_pipe(iload_reg_reg);
14831
14832 %}
14833
14834 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14835
14836 match(Set dst (MoveI2F src));
14837
14838 effect(DEF dst, USE src);
14839
14840 ins_cost(4 * INSN_COST);
14841
14842 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14843
14844 ins_encode %{
14845 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14846 %}
14847
14848 ins_pipe(pipe_class_memory);
14849
14850 %}
14851
14852 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14853
14854 match(Set dst (MoveD2L src));
14855
14856 effect(DEF dst, USE src);
14857
14858 ins_cost(4 * INSN_COST);
14859
14860 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14861
14862 ins_encode %{
14863 __ ldr($dst$$Register, Address(sp, $src$$disp));
14864 %}
14865
14866 ins_pipe(iload_reg_reg);
14867
14868 %}
14869
14870 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14871
14872 match(Set dst (MoveL2D src));
14873
14874 effect(DEF dst, USE src);
14875
14876 ins_cost(4 * INSN_COST);
14877
14878 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14879
14880 ins_encode %{
14881 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14882 %}
14883
14884 ins_pipe(pipe_class_memory);
14885
14886 %}
14887
14888 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14889
14890 match(Set dst (MoveF2I src));
14891
14892 effect(DEF dst, USE src);
14893
14894 ins_cost(INSN_COST);
14895
14896 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14897
14898 ins_encode %{
14899 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14900 %}
14901
14902 ins_pipe(pipe_class_memory);
14903
14904 %}
14905
14906 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14907
14908 match(Set dst (MoveI2F src));
14909
14910 effect(DEF dst, USE src);
14911
14912 ins_cost(INSN_COST);
14913
14914 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14915
14916 ins_encode %{
14917 __ strw($src$$Register, Address(sp, $dst$$disp));
14918 %}
14919
14920 ins_pipe(istore_reg_reg);
14921
14922 %}
14923
14924 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14925
14926 match(Set dst (MoveD2L src));
14927
14928 effect(DEF dst, USE src);
14929
14930 ins_cost(INSN_COST);
14931
14932 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14933
14934 ins_encode %{
14935 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14936 %}
14937
14938 ins_pipe(pipe_class_memory);
14939
14940 %}
14941
14942 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14943
14944 match(Set dst (MoveL2D src));
14945
14946 effect(DEF dst, USE src);
14947
14948 ins_cost(INSN_COST);
14949
14950 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14951
14952 ins_encode %{
14953 __ str($src$$Register, Address(sp, $dst$$disp));
14954 %}
14955
14956 ins_pipe(istore_reg_reg);
14957
14958 %}
14959
14960 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14961
14962 match(Set dst (MoveF2I src));
14963
14964 effect(DEF dst, USE src);
14965
14966 ins_cost(INSN_COST);
14967
14968 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14969
14970 ins_encode %{
14971 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14972 %}
14973
14974 ins_pipe(fp_f2i);
14975
14976 %}
14977
14978 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14979
14980 match(Set dst (MoveI2F src));
14981
14982 effect(DEF dst, USE src);
14983
14984 ins_cost(INSN_COST);
14985
14986 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14987
14988 ins_encode %{
14989 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14990 %}
14991
14992 ins_pipe(fp_i2f);
14993
14994 %}
14995
14996 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14997
14998 match(Set dst (MoveD2L src));
14999
15000 effect(DEF dst, USE src);
15001
15002 ins_cost(INSN_COST);
15003
15004 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15005
15006 ins_encode %{
15007 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15008 %}
15009
15010 ins_pipe(fp_d2l);
15011
15012 %}
15013
15014 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15015
15016 match(Set dst (MoveL2D src));
15017
15018 effect(DEF dst, USE src);
15019
15020 ins_cost(INSN_COST);
15021
15022 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15023
15024 ins_encode %{
15025 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15026 %}
15027
15028 ins_pipe(fp_l2d);
15029
15030 %}
15031
15032 // ============================================================================
15033 // clearing of an array
15034
15035 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
15036 %{
15037 match(Set dummy (ClearArray (Binary cnt base) zero));
15038 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15039
15040 ins_cost(4 * INSN_COST);
15041 format %{ "ClearArray $cnt, $base" %}
15042
15043 ins_encode %{
15044 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15045 if (tpc == nullptr) {
15046 ciEnv::current()->record_failure("CodeCache is full");
15047 return;
15048 }
15049 %}
15050
15051 ins_pipe(pipe_class_memory);
15052 %}
15053
15054 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
15055 %{
15056 predicate(((ClearArrayNode*)n)->word_copy_only());
15057 match(Set dummy (ClearArray (Binary cnt base) val));
15058 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15059
15060 ins_cost(4 * INSN_COST);
15061 format %{ "ClearArray $cnt, $base, $val" %}
15062
15063 ins_encode %{
15064 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
15065 %}
15066
15067 ins_pipe(pipe_class_memory);
15068 %}
15069
15070 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15071 %{
15072 predicate((uint64_t)n->in(2)->get_long()
15073 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
15074 && !((ClearArrayNode*)n)->word_copy_only());
15075 match(Set dummy (ClearArray cnt base));
15076 effect(TEMP temp, USE_KILL base, KILL cr);
15077
15078 ins_cost(4 * INSN_COST);
15079 format %{ "ClearArray $cnt, $base" %}
15080
15081 ins_encode %{
15082 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15083 if (tpc == nullptr) {
15084 ciEnv::current()->record_failure("CodeCache is full");
15085 return;
15086 }
15087 %}
15088
15089 ins_pipe(pipe_class_memory);
15090 %}
15091
15092 // ============================================================================
15093 // Overflow Math Instructions
15094
15095 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15096 %{
15097 match(Set cr (OverflowAddI op1 op2));
15098
15099 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15100 ins_cost(INSN_COST);
15101 ins_encode %{
15102 __ cmnw($op1$$Register, $op2$$Register);
15103 %}
15104
15105 ins_pipe(icmp_reg_reg);
15106 %}
15107
15108 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15109 %{
15110 match(Set cr (OverflowAddI op1 op2));
15111
15112 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15113 ins_cost(INSN_COST);
15114 ins_encode %{
15115 __ cmnw($op1$$Register, $op2$$constant);
15116 %}
15117
15118 ins_pipe(icmp_reg_imm);
15119 %}
15120
15121 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15122 %{
15123 match(Set cr (OverflowAddL op1 op2));
15124
15125 format %{ "cmn $op1, $op2\t# overflow check long" %}
15126 ins_cost(INSN_COST);
15127 ins_encode %{
15128 __ cmn($op1$$Register, $op2$$Register);
15129 %}
15130
15131 ins_pipe(icmp_reg_reg);
15132 %}
15133
15134 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15135 %{
15136 match(Set cr (OverflowAddL op1 op2));
15137
15138 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15139 ins_cost(INSN_COST);
15140 ins_encode %{
15141 __ adds(zr, $op1$$Register, $op2$$constant);
15142 %}
15143
15144 ins_pipe(icmp_reg_imm);
15145 %}
15146
15147 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15148 %{
15149 match(Set cr (OverflowSubI op1 op2));
15150
15151 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15152 ins_cost(INSN_COST);
15153 ins_encode %{
15154 __ cmpw($op1$$Register, $op2$$Register);
15155 %}
15156
15157 ins_pipe(icmp_reg_reg);
15158 %}
15159
15160 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15161 %{
15162 match(Set cr (OverflowSubI op1 op2));
15163
15164 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15165 ins_cost(INSN_COST);
15166 ins_encode %{
15167 __ cmpw($op1$$Register, $op2$$constant);
15168 %}
15169
15170 ins_pipe(icmp_reg_imm);
15171 %}
15172
15173 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15174 %{
15175 match(Set cr (OverflowSubL op1 op2));
15176
15177 format %{ "cmp $op1, $op2\t# overflow check long" %}
15178 ins_cost(INSN_COST);
15179 ins_encode %{
15180 __ cmp($op1$$Register, $op2$$Register);
15181 %}
15182
15183 ins_pipe(icmp_reg_reg);
15184 %}
15185
15186 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15187 %{
15188 match(Set cr (OverflowSubL op1 op2));
15189
15190 format %{ "cmp $op1, $op2\t# overflow check long" %}
15191 ins_cost(INSN_COST);
15192 ins_encode %{
15193 __ subs(zr, $op1$$Register, $op2$$constant);
15194 %}
15195
15196 ins_pipe(icmp_reg_imm);
15197 %}
15198
15199 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15200 %{
15201 match(Set cr (OverflowSubI zero op1));
15202
15203 format %{ "cmpw zr, $op1\t# overflow check int" %}
15204 ins_cost(INSN_COST);
15205 ins_encode %{
15206 __ cmpw(zr, $op1$$Register);
15207 %}
15208
15209 ins_pipe(icmp_reg_imm);
15210 %}
15211
15212 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15213 %{
15214 match(Set cr (OverflowSubL zero op1));
15215
15216 format %{ "cmp zr, $op1\t# overflow check long" %}
15217 ins_cost(INSN_COST);
15218 ins_encode %{
15219 __ cmp(zr, $op1$$Register);
15220 %}
15221
15222 ins_pipe(icmp_reg_imm);
15223 %}
15224
15225 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15226 %{
15227 match(Set cr (OverflowMulI op1 op2));
15228
15229 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15230 "cmp rscratch1, rscratch1, sxtw\n\t"
15231 "movw rscratch1, #0x80000000\n\t"
15232 "cselw rscratch1, rscratch1, zr, NE\n\t"
15233 "cmpw rscratch1, #1" %}
15234 ins_cost(5 * INSN_COST);
15235 ins_encode %{
15236 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15237 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15238 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15239 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15240 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15241 %}
15242
15243 ins_pipe(pipe_slow);
15244 %}
15245
15246 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15247 %{
15248 match(If cmp (OverflowMulI op1 op2));
15249 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15250 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15251 effect(USE labl, KILL cr);
15252
15253 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15254 "cmp rscratch1, rscratch1, sxtw\n\t"
15255 "b$cmp $labl" %}
15256 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15257 ins_encode %{
15258 Label* L = $labl$$label;
15259 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15260 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15261 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15262 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15263 %}
15264
15265 ins_pipe(pipe_serial);
15266 %}
15267
15268 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15269 %{
15270 match(Set cr (OverflowMulL op1 op2));
15271
15272 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15273 "smulh rscratch2, $op1, $op2\n\t"
15274 "cmp rscratch2, rscratch1, ASR #63\n\t"
15275 "movw rscratch1, #0x80000000\n\t"
15276 "cselw rscratch1, rscratch1, zr, NE\n\t"
15277 "cmpw rscratch1, #1" %}
15278 ins_cost(6 * INSN_COST);
15279 ins_encode %{
15280 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15281 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15282 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15283 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15284 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15285 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15286 %}
15287
15288 ins_pipe(pipe_slow);
15289 %}
15290
15291 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15292 %{
15293 match(If cmp (OverflowMulL op1 op2));
15294 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15295 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15296 effect(USE labl, KILL cr);
15297
15298 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15299 "smulh rscratch2, $op1, $op2\n\t"
15300 "cmp rscratch2, rscratch1, ASR #63\n\t"
15301 "b$cmp $labl" %}
15302 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15303 ins_encode %{
15304 Label* L = $labl$$label;
15305 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15306 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15307 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15308 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15309 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15310 %}
15311
15312 ins_pipe(pipe_serial);
15313 %}
15314
15315 // ============================================================================
15316 // Compare Instructions
15317
15318 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15319 %{
15320 match(Set cr (CmpI op1 op2));
15321
15322 effect(DEF cr, USE op1, USE op2);
15323
15324 ins_cost(INSN_COST);
15325 format %{ "cmpw $op1, $op2" %}
15326
15327 ins_encode(aarch64_enc_cmpw(op1, op2));
15328
15329 ins_pipe(icmp_reg_reg);
15330 %}
15331
15332 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15333 %{
15334 match(Set cr (CmpI op1 zero));
15335
15336 effect(DEF cr, USE op1);
15337
15338 ins_cost(INSN_COST);
15339 format %{ "cmpw $op1, 0" %}
15340
15341 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15342
15343 ins_pipe(icmp_reg_imm);
15344 %}
15345
15346 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15347 %{
15348 match(Set cr (CmpI op1 op2));
15349
15350 effect(DEF cr, USE op1);
15351
15352 ins_cost(INSN_COST);
15353 format %{ "cmpw $op1, $op2" %}
15354
15355 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15356
15357 ins_pipe(icmp_reg_imm);
15358 %}
15359
15360 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15361 %{
15362 match(Set cr (CmpI op1 op2));
15363
15364 effect(DEF cr, USE op1);
15365
15366 ins_cost(INSN_COST * 2);
15367 format %{ "cmpw $op1, $op2" %}
15368
15369 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15370
15371 ins_pipe(icmp_reg_imm);
15372 %}
15373
15374 // Unsigned compare Instructions; really, same as signed compare
15375 // except it should only be used to feed an If or a CMovI which takes a
15376 // cmpOpU.
15377
15378 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15379 %{
15380 match(Set cr (CmpU op1 op2));
15381
15382 effect(DEF cr, USE op1, USE op2);
15383
15384 ins_cost(INSN_COST);
15385 format %{ "cmpw $op1, $op2\t# unsigned" %}
15386
15387 ins_encode(aarch64_enc_cmpw(op1, op2));
15388
15389 ins_pipe(icmp_reg_reg);
15390 %}
15391
15392 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15393 %{
15394 match(Set cr (CmpU op1 zero));
15395
15396 effect(DEF cr, USE op1);
15397
15398 ins_cost(INSN_COST);
15399 format %{ "cmpw $op1, #0\t# unsigned" %}
15400
15401 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15402
15403 ins_pipe(icmp_reg_imm);
15404 %}
15405
15406 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15407 %{
15408 match(Set cr (CmpU op1 op2));
15409
15410 effect(DEF cr, USE op1);
15411
15412 ins_cost(INSN_COST);
15413 format %{ "cmpw $op1, $op2\t# unsigned" %}
15414
15415 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15416
15417 ins_pipe(icmp_reg_imm);
15418 %}
15419
15420 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15421 %{
15422 match(Set cr (CmpU op1 op2));
15423
15424 effect(DEF cr, USE op1);
15425
15426 ins_cost(INSN_COST * 2);
15427 format %{ "cmpw $op1, $op2\t# unsigned" %}
15428
15429 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15430
15431 ins_pipe(icmp_reg_imm);
15432 %}
15433
15434 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15435 %{
15436 match(Set cr (CmpL op1 op2));
15437
15438 effect(DEF cr, USE op1, USE op2);
15439
15440 ins_cost(INSN_COST);
15441 format %{ "cmp $op1, $op2" %}
15442
15443 ins_encode(aarch64_enc_cmp(op1, op2));
15444
15445 ins_pipe(icmp_reg_reg);
15446 %}
15447
15448 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15449 %{
15450 match(Set cr (CmpL op1 zero));
15451
15452 effect(DEF cr, USE op1);
15453
15454 ins_cost(INSN_COST);
15455 format %{ "tst $op1" %}
15456
15457 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15458
15459 ins_pipe(icmp_reg_imm);
15460 %}
15461
15462 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15463 %{
15464 match(Set cr (CmpL op1 op2));
15465
15466 effect(DEF cr, USE op1);
15467
15468 ins_cost(INSN_COST);
15469 format %{ "cmp $op1, $op2" %}
15470
15471 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15472
15473 ins_pipe(icmp_reg_imm);
15474 %}
15475
15476 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15477 %{
15478 match(Set cr (CmpL op1 op2));
15479
15480 effect(DEF cr, USE op1);
15481
15482 ins_cost(INSN_COST * 2);
15483 format %{ "cmp $op1, $op2" %}
15484
15485 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15486
15487 ins_pipe(icmp_reg_imm);
15488 %}
15489
15490 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15491 %{
15492 match(Set cr (CmpUL op1 op2));
15493
15494 effect(DEF cr, USE op1, USE op2);
15495
15496 ins_cost(INSN_COST);
15497 format %{ "cmp $op1, $op2" %}
15498
15499 ins_encode(aarch64_enc_cmp(op1, op2));
15500
15501 ins_pipe(icmp_reg_reg);
15502 %}
15503
15504 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15505 %{
15506 match(Set cr (CmpUL op1 zero));
15507
15508 effect(DEF cr, USE op1);
15509
15510 ins_cost(INSN_COST);
15511 format %{ "tst $op1" %}
15512
15513 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15514
15515 ins_pipe(icmp_reg_imm);
15516 %}
15517
15518 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15519 %{
15520 match(Set cr (CmpUL op1 op2));
15521
15522 effect(DEF cr, USE op1);
15523
15524 ins_cost(INSN_COST);
15525 format %{ "cmp $op1, $op2" %}
15526
15527 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15528
15529 ins_pipe(icmp_reg_imm);
15530 %}
15531
15532 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15533 %{
15534 match(Set cr (CmpUL op1 op2));
15535
15536 effect(DEF cr, USE op1);
15537
15538 ins_cost(INSN_COST * 2);
15539 format %{ "cmp $op1, $op2" %}
15540
15541 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15542
15543 ins_pipe(icmp_reg_imm);
15544 %}
15545
15546 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15547 %{
15548 match(Set cr (CmpP op1 op2));
15549
15550 effect(DEF cr, USE op1, USE op2);
15551
15552 ins_cost(INSN_COST);
15553 format %{ "cmp $op1, $op2\t // ptr" %}
15554
15555 ins_encode(aarch64_enc_cmpp(op1, op2));
15556
15557 ins_pipe(icmp_reg_reg);
15558 %}
15559
15560 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15561 %{
15562 match(Set cr (CmpN op1 op2));
15563
15564 effect(DEF cr, USE op1, USE op2);
15565
15566 ins_cost(INSN_COST);
15567 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15568
15569 ins_encode(aarch64_enc_cmpn(op1, op2));
15570
15571 ins_pipe(icmp_reg_reg);
15572 %}
15573
15574 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15575 %{
15576 match(Set cr (CmpP op1 zero));
15577
15578 effect(DEF cr, USE op1, USE zero);
15579
15580 ins_cost(INSN_COST);
15581 format %{ "cmp $op1, 0\t // ptr" %}
15582
15583 ins_encode(aarch64_enc_testp(op1));
15584
15585 ins_pipe(icmp_reg_imm);
15586 %}
15587
15588 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15589 %{
15590 match(Set cr (CmpN op1 zero));
15591
15592 effect(DEF cr, USE op1, USE zero);
15593
15594 ins_cost(INSN_COST);
15595 format %{ "cmp $op1, 0\t // compressed ptr" %}
15596
15597 ins_encode(aarch64_enc_testn(op1));
15598
15599 ins_pipe(icmp_reg_imm);
15600 %}
15601
15602 // FP comparisons
15603 //
15604 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15605 // using normal cmpOp. See declaration of rFlagsReg for details.
15606
15607 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15608 %{
15609 match(Set cr (CmpF src1 src2));
15610
15611 ins_cost(3 * INSN_COST);
15612 format %{ "fcmps $src1, $src2" %}
15613
15614 ins_encode %{
15615 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15616 %}
15617
15618 ins_pipe(pipe_class_compare);
15619 %}
15620
15621 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15622 %{
15623 match(Set cr (CmpF src1 src2));
15624
15625 ins_cost(3 * INSN_COST);
15626 format %{ "fcmps $src1, 0.0" %}
15627
15628 ins_encode %{
15629 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15630 %}
15631
15632 ins_pipe(pipe_class_compare);
15633 %}
15634 // FROM HERE
15635
15636 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15637 %{
15638 match(Set cr (CmpD src1 src2));
15639
15640 ins_cost(3 * INSN_COST);
15641 format %{ "fcmpd $src1, $src2" %}
15642
15643 ins_encode %{
15644 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15645 %}
15646
15647 ins_pipe(pipe_class_compare);
15648 %}
15649
15650 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15651 %{
15652 match(Set cr (CmpD src1 src2));
15653
15654 ins_cost(3 * INSN_COST);
15655 format %{ "fcmpd $src1, 0.0" %}
15656
15657 ins_encode %{
15658 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15659 %}
15660
15661 ins_pipe(pipe_class_compare);
15662 %}
15663
15664 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15665 %{
15666 match(Set dst (CmpF3 src1 src2));
15667 effect(KILL cr);
15668
15669 ins_cost(5 * INSN_COST);
15670 format %{ "fcmps $src1, $src2\n\t"
15671 "csinvw($dst, zr, zr, eq\n\t"
15672 "csnegw($dst, $dst, $dst, lt)"
15673 %}
15674
15675 ins_encode %{
15676 Label done;
15677 FloatRegister s1 = as_FloatRegister($src1$$reg);
15678 FloatRegister s2 = as_FloatRegister($src2$$reg);
15679 Register d = as_Register($dst$$reg);
15680 __ fcmps(s1, s2);
15681 // installs 0 if EQ else -1
15682 __ csinvw(d, zr, zr, Assembler::EQ);
15683 // keeps -1 if less or unordered else installs 1
15684 __ csnegw(d, d, d, Assembler::LT);
15685 __ bind(done);
15686 %}
15687
15688 ins_pipe(pipe_class_default);
15689
15690 %}
15691
15692 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15693 %{
15694 match(Set dst (CmpD3 src1 src2));
15695 effect(KILL cr);
15696
15697 ins_cost(5 * INSN_COST);
15698 format %{ "fcmpd $src1, $src2\n\t"
15699 "csinvw($dst, zr, zr, eq\n\t"
15700 "csnegw($dst, $dst, $dst, lt)"
15701 %}
15702
15703 ins_encode %{
15704 Label done;
15705 FloatRegister s1 = as_FloatRegister($src1$$reg);
15706 FloatRegister s2 = as_FloatRegister($src2$$reg);
15707 Register d = as_Register($dst$$reg);
15708 __ fcmpd(s1, s2);
15709 // installs 0 if EQ else -1
15710 __ csinvw(d, zr, zr, Assembler::EQ);
15711 // keeps -1 if less or unordered else installs 1
15712 __ csnegw(d, d, d, Assembler::LT);
15713 __ bind(done);
15714 %}
15715 ins_pipe(pipe_class_default);
15716
15717 %}
15718
15719 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15720 %{
15721 match(Set dst (CmpF3 src1 zero));
15722 effect(KILL cr);
15723
15724 ins_cost(5 * INSN_COST);
15725 format %{ "fcmps $src1, 0.0\n\t"
15726 "csinvw($dst, zr, zr, eq\n\t"
15727 "csnegw($dst, $dst, $dst, lt)"
15728 %}
15729
15730 ins_encode %{
15731 Label done;
15732 FloatRegister s1 = as_FloatRegister($src1$$reg);
15733 Register d = as_Register($dst$$reg);
15734 __ fcmps(s1, 0.0);
15735 // installs 0 if EQ else -1
15736 __ csinvw(d, zr, zr, Assembler::EQ);
15737 // keeps -1 if less or unordered else installs 1
15738 __ csnegw(d, d, d, Assembler::LT);
15739 __ bind(done);
15740 %}
15741
15742 ins_pipe(pipe_class_default);
15743
15744 %}
15745
15746 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15747 %{
15748 match(Set dst (CmpD3 src1 zero));
15749 effect(KILL cr);
15750
15751 ins_cost(5 * INSN_COST);
15752 format %{ "fcmpd $src1, 0.0\n\t"
15753 "csinvw($dst, zr, zr, eq\n\t"
15754 "csnegw($dst, $dst, $dst, lt)"
15755 %}
15756
15757 ins_encode %{
15758 Label done;
15759 FloatRegister s1 = as_FloatRegister($src1$$reg);
15760 Register d = as_Register($dst$$reg);
15761 __ fcmpd(s1, 0.0);
15762 // installs 0 if EQ else -1
15763 __ csinvw(d, zr, zr, Assembler::EQ);
15764 // keeps -1 if less or unordered else installs 1
15765 __ csnegw(d, d, d, Assembler::LT);
15766 __ bind(done);
15767 %}
15768 ins_pipe(pipe_class_default);
15769
15770 %}
15771
15772 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15773 %{
15774 match(Set dst (CmpLTMask p q));
15775 effect(KILL cr);
15776
15777 ins_cost(3 * INSN_COST);
15778
15779 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15780 "csetw $dst, lt\n\t"
15781 "subw $dst, zr, $dst"
15782 %}
15783
15784 ins_encode %{
15785 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15786 __ csetw(as_Register($dst$$reg), Assembler::LT);
15787 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15788 %}
15789
15790 ins_pipe(ialu_reg_reg);
15791 %}
15792
15793 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15794 %{
15795 match(Set dst (CmpLTMask src zero));
15796 effect(KILL cr);
15797
15798 ins_cost(INSN_COST);
15799
15800 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15801
15802 ins_encode %{
15803 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15804 %}
15805
15806 ins_pipe(ialu_reg_shift);
15807 %}
15808
15809 // ============================================================================
15810 // Max and Min
15811
15812 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15813
15814 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15815 %{
15816 effect(DEF cr, USE src);
15817 ins_cost(INSN_COST);
15818 format %{ "cmpw $src, 0" %}
15819
15820 ins_encode %{
15821 __ cmpw($src$$Register, 0);
15822 %}
15823 ins_pipe(icmp_reg_imm);
15824 %}
15825
15826 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15827 %{
15828 match(Set dst (MinI src1 src2));
15829 ins_cost(INSN_COST * 3);
15830
15831 expand %{
15832 rFlagsReg cr;
15833 compI_reg_reg(cr, src1, src2);
15834 cmovI_reg_reg_lt(dst, src1, src2, cr);
15835 %}
15836 %}
15837
15838 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15839 %{
15840 match(Set dst (MaxI src1 src2));
15841 ins_cost(INSN_COST * 3);
15842
15843 expand %{
15844 rFlagsReg cr;
15845 compI_reg_reg(cr, src1, src2);
15846 cmovI_reg_reg_gt(dst, src1, src2, cr);
15847 %}
15848 %}
15849
15850
15851 // ============================================================================
15852 // Branch Instructions
15853
15854 // Direct Branch.
15855 instruct branch(label lbl)
15856 %{
15857 match(Goto);
15858
15859 effect(USE lbl);
15860
15861 ins_cost(BRANCH_COST);
15862 format %{ "b $lbl" %}
15863
15864 ins_encode(aarch64_enc_b(lbl));
15865
15866 ins_pipe(pipe_branch);
15867 %}
15868
15869 // Conditional Near Branch
15870 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15871 %{
15872 // Same match rule as `branchConFar'.
15873 match(If cmp cr);
15874
15875 effect(USE lbl);
15876
15877 ins_cost(BRANCH_COST);
15878 // If set to 1 this indicates that the current instruction is a
15879 // short variant of a long branch. This avoids using this
15880 // instruction in first-pass matching. It will then only be used in
15881 // the `Shorten_branches' pass.
15882 // ins_short_branch(1);
15883 format %{ "b$cmp $lbl" %}
15884
15885 ins_encode(aarch64_enc_br_con(cmp, lbl));
15886
15887 ins_pipe(pipe_branch_cond);
15888 %}
15889
15890 // Conditional Near Branch Unsigned
15891 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15892 %{
15893 // Same match rule as `branchConFar'.
15894 match(If cmp cr);
15895
15896 effect(USE lbl);
15897
15898 ins_cost(BRANCH_COST);
15899 // If set to 1 this indicates that the current instruction is a
15900 // short variant of a long branch. This avoids using this
15901 // instruction in first-pass matching. It will then only be used in
15902 // the `Shorten_branches' pass.
15903 // ins_short_branch(1);
15904 format %{ "b$cmp $lbl\t# unsigned" %}
15905
15906 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15907
15908 ins_pipe(pipe_branch_cond);
15909 %}
15910
15911 // Make use of CBZ and CBNZ. These instructions, as well as being
15912 // shorter than (cmp; branch), have the additional benefit of not
15913 // killing the flags.
15914
15915 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15916 match(If cmp (CmpI op1 op2));
15917 effect(USE labl);
15918
15919 ins_cost(BRANCH_COST);
15920 format %{ "cbw$cmp $op1, $labl" %}
15921 ins_encode %{
15922 Label* L = $labl$$label;
15923 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15924 if (cond == Assembler::EQ)
15925 __ cbzw($op1$$Register, *L);
15926 else
15927 __ cbnzw($op1$$Register, *L);
15928 %}
15929 ins_pipe(pipe_cmp_branch);
15930 %}
15931
15932 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15933 match(If cmp (CmpL op1 op2));
15934 effect(USE labl);
15935
15936 ins_cost(BRANCH_COST);
15937 format %{ "cb$cmp $op1, $labl" %}
15938 ins_encode %{
15939 Label* L = $labl$$label;
15940 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15941 if (cond == Assembler::EQ)
15942 __ cbz($op1$$Register, *L);
15943 else
15944 __ cbnz($op1$$Register, *L);
15945 %}
15946 ins_pipe(pipe_cmp_branch);
15947 %}
15948
15949 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15950 match(If cmp (CmpP op1 op2));
15951 effect(USE labl);
15952
15953 ins_cost(BRANCH_COST);
15954 format %{ "cb$cmp $op1, $labl" %}
15955 ins_encode %{
15956 Label* L = $labl$$label;
15957 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15958 if (cond == Assembler::EQ)
15959 __ cbz($op1$$Register, *L);
15960 else
15961 __ cbnz($op1$$Register, *L);
15962 %}
15963 ins_pipe(pipe_cmp_branch);
15964 %}
15965
15966 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15967 match(If cmp (CmpN op1 op2));
15968 effect(USE labl);
15969
15970 ins_cost(BRANCH_COST);
15971 format %{ "cbw$cmp $op1, $labl" %}
15972 ins_encode %{
15973 Label* L = $labl$$label;
15974 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15975 if (cond == Assembler::EQ)
15976 __ cbzw($op1$$Register, *L);
15977 else
15978 __ cbnzw($op1$$Register, *L);
15979 %}
15980 ins_pipe(pipe_cmp_branch);
15981 %}
15982
15983 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15984 match(If cmp (CmpP (DecodeN oop) zero));
15985 effect(USE labl);
15986
15987 ins_cost(BRANCH_COST);
15988 format %{ "cb$cmp $oop, $labl" %}
15989 ins_encode %{
15990 Label* L = $labl$$label;
15991 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15992 if (cond == Assembler::EQ)
15993 __ cbzw($oop$$Register, *L);
15994 else
15995 __ cbnzw($oop$$Register, *L);
15996 %}
15997 ins_pipe(pipe_cmp_branch);
15998 %}
15999
16000 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16001 match(If cmp (CmpU op1 op2));
16002 effect(USE labl);
16003
16004 ins_cost(BRANCH_COST);
16005 format %{ "cbw$cmp $op1, $labl" %}
16006 ins_encode %{
16007 Label* L = $labl$$label;
16008 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16009 if (cond == Assembler::EQ || cond == Assembler::LS) {
16010 __ cbzw($op1$$Register, *L);
16011 } else {
16012 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16013 __ cbnzw($op1$$Register, *L);
16014 }
16015 %}
16016 ins_pipe(pipe_cmp_branch);
16017 %}
16018
16019 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
16020 match(If cmp (CmpUL op1 op2));
16021 effect(USE labl);
16022
16023 ins_cost(BRANCH_COST);
16024 format %{ "cb$cmp $op1, $labl" %}
16025 ins_encode %{
16026 Label* L = $labl$$label;
16027 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16028 if (cond == Assembler::EQ || cond == Assembler::LS) {
16029 __ cbz($op1$$Register, *L);
16030 } else {
16031 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16032 __ cbnz($op1$$Register, *L);
16033 }
16034 %}
16035 ins_pipe(pipe_cmp_branch);
16036 %}
16037
16038 // Test bit and Branch
16039
16040 // Patterns for short (< 32KiB) variants
16041 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16042 match(If cmp (CmpL op1 op2));
16043 effect(USE labl);
16044
16045 ins_cost(BRANCH_COST);
16046 format %{ "cb$cmp $op1, $labl # long" %}
16047 ins_encode %{
16048 Label* L = $labl$$label;
16049 Assembler::Condition cond =
16050 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16051 __ tbr(cond, $op1$$Register, 63, *L);
16052 %}
16053 ins_pipe(pipe_cmp_branch);
16054 ins_short_branch(1);
16055 %}
16056
16057 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16058 match(If cmp (CmpI op1 op2));
16059 effect(USE labl);
16060
16061 ins_cost(BRANCH_COST);
16062 format %{ "cb$cmp $op1, $labl # int" %}
16063 ins_encode %{
16064 Label* L = $labl$$label;
16065 Assembler::Condition cond =
16066 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16067 __ tbr(cond, $op1$$Register, 31, *L);
16068 %}
16069 ins_pipe(pipe_cmp_branch);
16070 ins_short_branch(1);
16071 %}
16072
16073 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16074 match(If cmp (CmpL (AndL op1 op2) op3));
16075 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16076 effect(USE labl);
16077
16078 ins_cost(BRANCH_COST);
16079 format %{ "tb$cmp $op1, $op2, $labl" %}
16080 ins_encode %{
16081 Label* L = $labl$$label;
16082 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16083 int bit = exact_log2_long($op2$$constant);
16084 __ tbr(cond, $op1$$Register, bit, *L);
16085 %}
16086 ins_pipe(pipe_cmp_branch);
16087 ins_short_branch(1);
16088 %}
16089
16090 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16091 match(If cmp (CmpI (AndI op1 op2) op3));
16092 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16093 effect(USE labl);
16094
16095 ins_cost(BRANCH_COST);
16096 format %{ "tb$cmp $op1, $op2, $labl" %}
16097 ins_encode %{
16098 Label* L = $labl$$label;
16099 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16100 int bit = exact_log2((juint)$op2$$constant);
16101 __ tbr(cond, $op1$$Register, bit, *L);
16102 %}
16103 ins_pipe(pipe_cmp_branch);
16104 ins_short_branch(1);
16105 %}
16106
16107 // And far variants
16108 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16109 match(If cmp (CmpL op1 op2));
16110 effect(USE labl);
16111
16112 ins_cost(BRANCH_COST);
16113 format %{ "cb$cmp $op1, $labl # long" %}
16114 ins_encode %{
16115 Label* L = $labl$$label;
16116 Assembler::Condition cond =
16117 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16118 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16119 %}
16120 ins_pipe(pipe_cmp_branch);
16121 %}
16122
16123 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16124 match(If cmp (CmpI op1 op2));
16125 effect(USE labl);
16126
16127 ins_cost(BRANCH_COST);
16128 format %{ "cb$cmp $op1, $labl # int" %}
16129 ins_encode %{
16130 Label* L = $labl$$label;
16131 Assembler::Condition cond =
16132 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16133 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16134 %}
16135 ins_pipe(pipe_cmp_branch);
16136 %}
16137
16138 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16139 match(If cmp (CmpL (AndL op1 op2) op3));
16140 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16141 effect(USE labl);
16142
16143 ins_cost(BRANCH_COST);
16144 format %{ "tb$cmp $op1, $op2, $labl" %}
16145 ins_encode %{
16146 Label* L = $labl$$label;
16147 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16148 int bit = exact_log2_long($op2$$constant);
16149 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16150 %}
16151 ins_pipe(pipe_cmp_branch);
16152 %}
16153
16154 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16155 match(If cmp (CmpI (AndI op1 op2) op3));
16156 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16157 effect(USE labl);
16158
16159 ins_cost(BRANCH_COST);
16160 format %{ "tb$cmp $op1, $op2, $labl" %}
16161 ins_encode %{
16162 Label* L = $labl$$label;
16163 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16164 int bit = exact_log2((juint)$op2$$constant);
16165 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16166 %}
16167 ins_pipe(pipe_cmp_branch);
16168 %}
16169
16170 // Test bits
16171
16172 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16173 match(Set cr (CmpL (AndL op1 op2) op3));
16174 predicate(Assembler::operand_valid_for_logical_immediate
16175 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16176
16177 ins_cost(INSN_COST);
16178 format %{ "tst $op1, $op2 # long" %}
16179 ins_encode %{
16180 __ tst($op1$$Register, $op2$$constant);
16181 %}
16182 ins_pipe(ialu_reg_reg);
16183 %}
16184
16185 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16186 match(Set cr (CmpI (AndI op1 op2) op3));
16187 predicate(Assembler::operand_valid_for_logical_immediate
16188 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16189
16190 ins_cost(INSN_COST);
16191 format %{ "tst $op1, $op2 # int" %}
16192 ins_encode %{
16193 __ tstw($op1$$Register, $op2$$constant);
16194 %}
16195 ins_pipe(ialu_reg_reg);
16196 %}
16197
16198 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16199 match(Set cr (CmpL (AndL op1 op2) op3));
16200
16201 ins_cost(INSN_COST);
16202 format %{ "tst $op1, $op2 # long" %}
16203 ins_encode %{
16204 __ tst($op1$$Register, $op2$$Register);
16205 %}
16206 ins_pipe(ialu_reg_reg);
16207 %}
16208
16209 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16210 match(Set cr (CmpI (AndI op1 op2) op3));
16211
16212 ins_cost(INSN_COST);
16213 format %{ "tstw $op1, $op2 # int" %}
16214 ins_encode %{
16215 __ tstw($op1$$Register, $op2$$Register);
16216 %}
16217 ins_pipe(ialu_reg_reg);
16218 %}
16219
16220
16221 // Conditional Far Branch
16222 // Conditional Far Branch Unsigned
16223 // TODO: fixme
16224
16225 // counted loop end branch near
16226 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16227 %{
16228 match(CountedLoopEnd cmp cr);
16229
16230 effect(USE lbl);
16231
16232 ins_cost(BRANCH_COST);
16233 // short variant.
16234 // ins_short_branch(1);
16235 format %{ "b$cmp $lbl \t// counted loop end" %}
16236
16237 ins_encode(aarch64_enc_br_con(cmp, lbl));
16238
16239 ins_pipe(pipe_branch);
16240 %}
16241
16242 // counted loop end branch far
16243 // TODO: fixme
16244
16245 // ============================================================================
16246 // inlined locking and unlocking
16247
16248 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16249 %{
16250 predicate(LockingMode != LM_LIGHTWEIGHT);
16251 match(Set cr (FastLock object box));
16252 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16253
16254 ins_cost(5 * INSN_COST);
16255 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16256
16257 ins_encode %{
16258 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16259 %}
16260
16261 ins_pipe(pipe_serial);
16262 %}
16263
16264 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16265 %{
16266 predicate(LockingMode != LM_LIGHTWEIGHT);
16267 match(Set cr (FastUnlock object box));
16268 effect(TEMP tmp, TEMP tmp2);
16269
16270 ins_cost(5 * INSN_COST);
16271 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16272
16273 ins_encode %{
16274 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16275 %}
16276
16277 ins_pipe(pipe_serial);
16278 %}
16279
16280 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16281 %{
16282 predicate(LockingMode == LM_LIGHTWEIGHT);
16283 match(Set cr (FastLock object box));
16284 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16285
16286 ins_cost(5 * INSN_COST);
16287 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16288
16289 ins_encode %{
16290 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16291 %}
16292
16293 ins_pipe(pipe_serial);
16294 %}
16295
16296 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16297 %{
16298 predicate(LockingMode == LM_LIGHTWEIGHT);
16299 match(Set cr (FastUnlock object box));
16300 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16301
16302 ins_cost(5 * INSN_COST);
16303 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16304
16305 ins_encode %{
16306 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16307 %}
16308
16309 ins_pipe(pipe_serial);
16310 %}
16311
16312 // ============================================================================
16313 // Safepoint Instructions
16314
16315 // TODO
16316 // provide a near and far version of this code
16317
16318 instruct safePoint(rFlagsReg cr, iRegP poll)
16319 %{
16320 match(SafePoint poll);
16321 effect(KILL cr);
16322
16323 format %{
16324 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16325 %}
16326 ins_encode %{
16327 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16328 %}
16329 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16330 %}
16331
16332
16333 // ============================================================================
16334 // Procedure Call/Return Instructions
16335
16336 // Call Java Static Instruction
16337
16338 instruct CallStaticJavaDirect(method meth)
16339 %{
16340 match(CallStaticJava);
16341
16342 effect(USE meth);
16343
16344 ins_cost(CALL_COST);
16345
16346 format %{ "call,static $meth \t// ==> " %}
16347
16348 ins_encode(aarch64_enc_java_static_call(meth),
16349 aarch64_enc_call_epilog);
16350
16351 ins_pipe(pipe_class_call);
16352 %}
16353
16354 // TO HERE
16355
16356 // Call Java Dynamic Instruction
16357 instruct CallDynamicJavaDirect(method meth)
16358 %{
16359 match(CallDynamicJava);
16360
16361 effect(USE meth);
16362
16363 ins_cost(CALL_COST);
16364
16365 format %{ "CALL,dynamic $meth \t// ==> " %}
16366
16367 ins_encode(aarch64_enc_java_dynamic_call(meth),
16368 aarch64_enc_call_epilog);
16369
16370 ins_pipe(pipe_class_call);
16371 %}
16372
16373 // Call Runtime Instruction
16374
16375 instruct CallRuntimeDirect(method meth)
16376 %{
16377 match(CallRuntime);
16378
16379 effect(USE meth);
16380
16381 ins_cost(CALL_COST);
16382
16383 format %{ "CALL, runtime $meth" %}
16384
16385 ins_encode( aarch64_enc_java_to_runtime(meth) );
16386
16387 ins_pipe(pipe_class_call);
16388 %}
16389
16390 // Call Runtime Instruction
16391
16392 instruct CallLeafDirect(method meth)
16393 %{
16394 match(CallLeaf);
16395
16396 effect(USE meth);
16397
16398 ins_cost(CALL_COST);
16399
16400 format %{ "CALL, runtime leaf $meth" %}
16401
16402 ins_encode( aarch64_enc_java_to_runtime(meth) );
16403
16404 ins_pipe(pipe_class_call);
16405 %}
16406
16407 // Call Runtime Instruction without safepoint and with vector arguments
16408 instruct CallLeafDirectVector(method meth)
16409 %{
16410 match(CallLeafVector);
16411
16412 effect(USE meth);
16413
16414 ins_cost(CALL_COST);
16415
16416 format %{ "CALL, runtime leaf vector $meth" %}
16417
16418 ins_encode(aarch64_enc_java_to_runtime(meth));
16419
16420 ins_pipe(pipe_class_call);
16421 %}
16422
16423 // Call Runtime Instruction
16424
16425 // entry point is null, target holds the address to call
16426 instruct CallLeafNoFPIndirect(iRegP target)
16427 %{
16428 predicate(n->as_Call()->entry_point() == nullptr);
16429
16430 match(CallLeafNoFP target);
16431
16432 ins_cost(CALL_COST);
16433
16434 format %{ "CALL, runtime leaf nofp indirect $target" %}
16435
16436 ins_encode %{
16437 __ blr($target$$Register);
16438 %}
16439
16440 ins_pipe(pipe_class_call);
16441 %}
16442
16443 instruct CallLeafNoFPDirect(method meth)
16444 %{
16445 predicate(n->as_Call()->entry_point() != nullptr);
16446
16447 match(CallLeafNoFP);
16448
16449 effect(USE meth);
16450
16451 ins_cost(CALL_COST);
16452
16453 format %{ "CALL, runtime leaf nofp $meth" %}
16454
16455 ins_encode( aarch64_enc_java_to_runtime(meth) );
16456
16457 ins_pipe(pipe_class_call);
16458 %}
16459
16460 // Tail Call; Jump from runtime stub to Java code.
16461 // Also known as an 'interprocedural jump'.
16462 // Target of jump will eventually return to caller.
16463 // TailJump below removes the return address.
16464 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16465 // emitted just above the TailCall which has reset rfp to the caller state.
16466 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16467 %{
16468 match(TailCall jump_target method_ptr);
16469
16470 ins_cost(CALL_COST);
16471
16472 format %{ "br $jump_target\t# $method_ptr holds method" %}
16473
16474 ins_encode(aarch64_enc_tail_call(jump_target));
16475
16476 ins_pipe(pipe_class_call);
16477 %}
16478
16479 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16480 %{
16481 match(TailJump jump_target ex_oop);
16482
16483 ins_cost(CALL_COST);
16484
16485 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16486
16487 ins_encode(aarch64_enc_tail_jmp(jump_target));
16488
16489 ins_pipe(pipe_class_call);
16490 %}
16491
16492 // Forward exception.
16493 instruct ForwardExceptionjmp()
16494 %{
16495 match(ForwardException);
16496 ins_cost(CALL_COST);
16497
16498 format %{ "b forward_exception_stub" %}
16499 ins_encode %{
16500 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16501 %}
16502 ins_pipe(pipe_class_call);
16503 %}
16504
16505 // Create exception oop: created by stack-crawling runtime code.
16506 // Created exception is now available to this handler, and is setup
16507 // just prior to jumping to this handler. No code emitted.
16508 // TODO check
16509 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16510 instruct CreateException(iRegP_R0 ex_oop)
16511 %{
16512 match(Set ex_oop (CreateEx));
16513
16514 format %{ " -- \t// exception oop; no code emitted" %}
16515
16516 size(0);
16517
16518 ins_encode( /*empty*/ );
16519
16520 ins_pipe(pipe_class_empty);
16521 %}
16522
16523 // Rethrow exception: The exception oop will come in the first
16524 // argument position. Then JUMP (not call) to the rethrow stub code.
16525 instruct RethrowException() %{
16526 match(Rethrow);
16527 ins_cost(CALL_COST);
16528
16529 format %{ "b rethrow_stub" %}
16530
16531 ins_encode( aarch64_enc_rethrow() );
16532
16533 ins_pipe(pipe_class_call);
16534 %}
16535
16536
16537 // Return Instruction
16538 // epilog node loads ret address into lr as part of frame pop
16539 instruct Ret()
16540 %{
16541 match(Return);
16542
16543 format %{ "ret\t// return register" %}
16544
16545 ins_encode( aarch64_enc_ret() );
16546
16547 ins_pipe(pipe_branch);
16548 %}
16549
16550 // Die now.
16551 instruct ShouldNotReachHere() %{
16552 match(Halt);
16553
16554 ins_cost(CALL_COST);
16555 format %{ "ShouldNotReachHere" %}
16556
16557 ins_encode %{
16558 if (is_reachable()) {
16559 const char* str = __ code_string(_halt_reason);
16560 __ stop(str);
16561 }
16562 %}
16563
16564 ins_pipe(pipe_class_default);
16565 %}
16566
16567 // ============================================================================
16568 // Partial Subtype Check
16569 //
16570 // superklass array for an instance of the superklass. Set a hidden
16571 // internal cache on a hit (cache is checked with exposed code in
16572 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16573 // encoding ALSO sets flags.
16574
16575 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16576 %{
16577 match(Set result (PartialSubtypeCheck sub super));
16578 predicate(!UseSecondarySupersTable);
16579 effect(KILL cr, KILL temp);
16580
16581 ins_cost(20 * INSN_COST); // slightly larger than the next version
16582 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16583
16584 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16585
16586 opcode(0x1); // Force zero of result reg on hit
16587
16588 ins_pipe(pipe_class_memory);
16589 %}
16590
16591 // Two versions of partialSubtypeCheck, both used when we need to
16592 // search for a super class in the secondary supers array. The first
16593 // is used when we don't know _a priori_ the class being searched
16594 // for. The second, far more common, is used when we do know: this is
16595 // used for instanceof, checkcast, and any case where C2 can determine
16596 // it by constant propagation.
16597
16598 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16599 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16600 rFlagsReg cr)
16601 %{
16602 match(Set result (PartialSubtypeCheck sub super));
16603 predicate(UseSecondarySupersTable);
16604 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16605
16606 ins_cost(10 * INSN_COST); // slightly larger than the next version
16607 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16608
16609 ins_encode %{
16610 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16611 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16612 $vtemp$$FloatRegister,
16613 $result$$Register, /*L_success*/nullptr);
16614 %}
16615
16616 ins_pipe(pipe_class_memory);
16617 %}
16618
16619 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16620 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16621 rFlagsReg cr)
16622 %{
16623 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16624 predicate(UseSecondarySupersTable);
16625 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16626
16627 ins_cost(5 * INSN_COST); // smaller than the next version
16628 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16629
16630 ins_encode %{
16631 bool success = false;
16632 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16633 if (InlineSecondarySupersTest) {
16634 success =
16635 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16636 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16637 $vtemp$$FloatRegister,
16638 $result$$Register,
16639 super_klass_slot);
16640 } else {
16641 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16642 success = (call != nullptr);
16643 }
16644 if (!success) {
16645 ciEnv::current()->record_failure("CodeCache is full");
16646 return;
16647 }
16648 %}
16649
16650 ins_pipe(pipe_class_memory);
16651 %}
16652
16653 // Intrisics for String.compareTo()
16654
16655 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16656 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16657 %{
16658 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16659 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16660 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16661
16662 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16663 ins_encode %{
16664 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16665 __ string_compare($str1$$Register, $str2$$Register,
16666 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16667 $tmp1$$Register, $tmp2$$Register,
16668 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16669 %}
16670 ins_pipe(pipe_class_memory);
16671 %}
16672
16673 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16674 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16675 %{
16676 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16677 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16678 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16679
16680 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16681 ins_encode %{
16682 __ string_compare($str1$$Register, $str2$$Register,
16683 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16684 $tmp1$$Register, $tmp2$$Register,
16685 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16686 %}
16687 ins_pipe(pipe_class_memory);
16688 %}
16689
16690 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16691 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16692 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16693 %{
16694 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16695 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16696 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16697 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16698
16699 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16700 ins_encode %{
16701 __ string_compare($str1$$Register, $str2$$Register,
16702 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16703 $tmp1$$Register, $tmp2$$Register,
16704 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16705 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16706 %}
16707 ins_pipe(pipe_class_memory);
16708 %}
16709
16710 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16711 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16712 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16713 %{
16714 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16715 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16716 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16717 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16718
16719 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16720 ins_encode %{
16721 __ string_compare($str1$$Register, $str2$$Register,
16722 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16723 $tmp1$$Register, $tmp2$$Register,
16724 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16725 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16726 %}
16727 ins_pipe(pipe_class_memory);
16728 %}
16729
16730 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16731 // these string_compare variants as NEON register type for convenience so that the prototype of
16732 // string_compare can be shared with all variants.
16733
16734 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16735 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16736 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16737 pRegGov_P1 pgtmp2, rFlagsReg cr)
16738 %{
16739 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16740 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16741 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16742 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16743
16744 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16745 ins_encode %{
16746 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16747 __ string_compare($str1$$Register, $str2$$Register,
16748 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16749 $tmp1$$Register, $tmp2$$Register,
16750 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16751 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16752 StrIntrinsicNode::LL);
16753 %}
16754 ins_pipe(pipe_class_memory);
16755 %}
16756
16757 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16758 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16759 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16760 pRegGov_P1 pgtmp2, rFlagsReg cr)
16761 %{
16762 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16763 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16764 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16765 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16766
16767 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16768 ins_encode %{
16769 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16770 __ string_compare($str1$$Register, $str2$$Register,
16771 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16772 $tmp1$$Register, $tmp2$$Register,
16773 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16774 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16775 StrIntrinsicNode::LU);
16776 %}
16777 ins_pipe(pipe_class_memory);
16778 %}
16779
16780 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16781 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16782 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16783 pRegGov_P1 pgtmp2, rFlagsReg cr)
16784 %{
16785 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16786 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16787 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16788 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16789
16790 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16791 ins_encode %{
16792 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16793 __ string_compare($str1$$Register, $str2$$Register,
16794 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16795 $tmp1$$Register, $tmp2$$Register,
16796 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16797 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16798 StrIntrinsicNode::UL);
16799 %}
16800 ins_pipe(pipe_class_memory);
16801 %}
16802
16803 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16804 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16805 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16806 pRegGov_P1 pgtmp2, rFlagsReg cr)
16807 %{
16808 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16809 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16810 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16811 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16812
16813 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16814 ins_encode %{
16815 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16816 __ string_compare($str1$$Register, $str2$$Register,
16817 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16818 $tmp1$$Register, $tmp2$$Register,
16819 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16820 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16821 StrIntrinsicNode::UU);
16822 %}
16823 ins_pipe(pipe_class_memory);
16824 %}
16825
16826 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16827 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16828 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16829 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16830 %{
16831 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16832 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16833 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16834 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16835 TEMP vtmp0, TEMP vtmp1, KILL cr);
16836 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16837 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16838
16839 ins_encode %{
16840 __ string_indexof($str1$$Register, $str2$$Register,
16841 $cnt1$$Register, $cnt2$$Register,
16842 $tmp1$$Register, $tmp2$$Register,
16843 $tmp3$$Register, $tmp4$$Register,
16844 $tmp5$$Register, $tmp6$$Register,
16845 -1, $result$$Register, StrIntrinsicNode::UU);
16846 %}
16847 ins_pipe(pipe_class_memory);
16848 %}
16849
16850 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16851 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16852 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16853 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16854 %{
16855 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16856 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16857 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16858 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16859 TEMP vtmp0, TEMP vtmp1, KILL cr);
16860 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16861 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16862
16863 ins_encode %{
16864 __ string_indexof($str1$$Register, $str2$$Register,
16865 $cnt1$$Register, $cnt2$$Register,
16866 $tmp1$$Register, $tmp2$$Register,
16867 $tmp3$$Register, $tmp4$$Register,
16868 $tmp5$$Register, $tmp6$$Register,
16869 -1, $result$$Register, StrIntrinsicNode::LL);
16870 %}
16871 ins_pipe(pipe_class_memory);
16872 %}
16873
16874 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16875 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16876 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16877 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16878 %{
16879 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16880 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16881 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16882 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16883 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16884 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16885 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16886
16887 ins_encode %{
16888 __ string_indexof($str1$$Register, $str2$$Register,
16889 $cnt1$$Register, $cnt2$$Register,
16890 $tmp1$$Register, $tmp2$$Register,
16891 $tmp3$$Register, $tmp4$$Register,
16892 $tmp5$$Register, $tmp6$$Register,
16893 -1, $result$$Register, StrIntrinsicNode::UL);
16894 %}
16895 ins_pipe(pipe_class_memory);
16896 %}
16897
16898 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16899 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16900 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16901 %{
16902 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16903 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16904 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16905 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16906 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16907 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16908
16909 ins_encode %{
16910 int icnt2 = (int)$int_cnt2$$constant;
16911 __ string_indexof($str1$$Register, $str2$$Register,
16912 $cnt1$$Register, zr,
16913 $tmp1$$Register, $tmp2$$Register,
16914 $tmp3$$Register, $tmp4$$Register, zr, zr,
16915 icnt2, $result$$Register, StrIntrinsicNode::UU);
16916 %}
16917 ins_pipe(pipe_class_memory);
16918 %}
16919
16920 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16921 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16922 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16923 %{
16924 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16925 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16926 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16927 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16928 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16929 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16930
16931 ins_encode %{
16932 int icnt2 = (int)$int_cnt2$$constant;
16933 __ string_indexof($str1$$Register, $str2$$Register,
16934 $cnt1$$Register, zr,
16935 $tmp1$$Register, $tmp2$$Register,
16936 $tmp3$$Register, $tmp4$$Register, zr, zr,
16937 icnt2, $result$$Register, StrIntrinsicNode::LL);
16938 %}
16939 ins_pipe(pipe_class_memory);
16940 %}
16941
16942 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16943 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16944 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16945 %{
16946 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16947 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16948 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16949 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16950 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16951 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16952
16953 ins_encode %{
16954 int icnt2 = (int)$int_cnt2$$constant;
16955 __ string_indexof($str1$$Register, $str2$$Register,
16956 $cnt1$$Register, zr,
16957 $tmp1$$Register, $tmp2$$Register,
16958 $tmp3$$Register, $tmp4$$Register, zr, zr,
16959 icnt2, $result$$Register, StrIntrinsicNode::UL);
16960 %}
16961 ins_pipe(pipe_class_memory);
16962 %}
16963
16964 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16965 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16966 iRegINoSp tmp3, rFlagsReg cr)
16967 %{
16968 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16969 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16970 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16971 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16972
16973 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16974
16975 ins_encode %{
16976 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16977 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16978 $tmp3$$Register);
16979 %}
16980 ins_pipe(pipe_class_memory);
16981 %}
16982
16983 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16984 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16985 iRegINoSp tmp3, rFlagsReg cr)
16986 %{
16987 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16988 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16989 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16990 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16991
16992 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16993
16994 ins_encode %{
16995 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16996 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16997 $tmp3$$Register);
16998 %}
16999 ins_pipe(pipe_class_memory);
17000 %}
17001
17002 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17003 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17004 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17005 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17006 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17007 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17008 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17009 ins_encode %{
17010 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17011 $result$$Register, $ztmp1$$FloatRegister,
17012 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17013 $ptmp$$PRegister, true /* isL */);
17014 %}
17015 ins_pipe(pipe_class_memory);
17016 %}
17017
17018 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17019 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17020 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17021 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17022 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17023 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17024 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17025 ins_encode %{
17026 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17027 $result$$Register, $ztmp1$$FloatRegister,
17028 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17029 $ptmp$$PRegister, false /* isL */);
17030 %}
17031 ins_pipe(pipe_class_memory);
17032 %}
17033
17034 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17035 iRegI_R0 result, rFlagsReg cr)
17036 %{
17037 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17038 match(Set result (StrEquals (Binary str1 str2) cnt));
17039 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17040
17041 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17042 ins_encode %{
17043 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17044 __ string_equals($str1$$Register, $str2$$Register,
17045 $result$$Register, $cnt$$Register);
17046 %}
17047 ins_pipe(pipe_class_memory);
17048 %}
17049
17050 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17051 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17052 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17053 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17054 iRegP_R10 tmp, rFlagsReg cr)
17055 %{
17056 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17057 match(Set result (AryEq ary1 ary2));
17058 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17059 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17060 TEMP vtmp6, TEMP vtmp7, KILL cr);
17061
17062 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17063 ins_encode %{
17064 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17065 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17066 $result$$Register, $tmp$$Register, 1);
17067 if (tpc == nullptr) {
17068 ciEnv::current()->record_failure("CodeCache is full");
17069 return;
17070 }
17071 %}
17072 ins_pipe(pipe_class_memory);
17073 %}
17074
17075 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17076 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17077 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17078 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17079 iRegP_R10 tmp, rFlagsReg cr)
17080 %{
17081 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17082 match(Set result (AryEq ary1 ary2));
17083 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17084 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17085 TEMP vtmp6, TEMP vtmp7, KILL cr);
17086
17087 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17088 ins_encode %{
17089 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17090 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17091 $result$$Register, $tmp$$Register, 2);
17092 if (tpc == nullptr) {
17093 ciEnv::current()->record_failure("CodeCache is full");
17094 return;
17095 }
17096 %}
17097 ins_pipe(pipe_class_memory);
17098 %}
17099
17100 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17101 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17102 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17103 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17104 %{
17105 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17106 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17107 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17108
17109 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
17110 ins_encode %{
17111 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17112 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17113 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17114 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17115 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17116 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17117 (BasicType)$basic_type$$constant);
17118 if (tpc == nullptr) {
17119 ciEnv::current()->record_failure("CodeCache is full");
17120 return;
17121 }
17122 %}
17123 ins_pipe(pipe_class_memory);
17124 %}
17125
17126 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17127 %{
17128 match(Set result (CountPositives ary1 len));
17129 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17130 format %{ "count positives byte[] $ary1,$len -> $result" %}
17131 ins_encode %{
17132 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17133 if (tpc == nullptr) {
17134 ciEnv::current()->record_failure("CodeCache is full");
17135 return;
17136 }
17137 %}
17138 ins_pipe( pipe_slow );
17139 %}
17140
17141 // fast char[] to byte[] compression
17142 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17143 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17144 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17145 iRegI_R0 result, rFlagsReg cr)
17146 %{
17147 match(Set result (StrCompressedCopy src (Binary dst len)));
17148 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17149 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17150
17151 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17152 ins_encode %{
17153 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17154 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17155 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17156 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17157 %}
17158 ins_pipe(pipe_slow);
17159 %}
17160
17161 // fast byte[] to char[] inflation
17162 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17163 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17164 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17165 %{
17166 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17167 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17168 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17169 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17170
17171 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17172 ins_encode %{
17173 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17174 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17175 $vtmp2$$FloatRegister, $tmp$$Register);
17176 if (tpc == nullptr) {
17177 ciEnv::current()->record_failure("CodeCache is full");
17178 return;
17179 }
17180 %}
17181 ins_pipe(pipe_class_memory);
17182 %}
17183
17184 // encode char[] to byte[] in ISO_8859_1
17185 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17186 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17187 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17188 iRegI_R0 result, rFlagsReg cr)
17189 %{
17190 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17191 match(Set result (EncodeISOArray src (Binary dst len)));
17192 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17193 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17194
17195 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17196 ins_encode %{
17197 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17198 $result$$Register, false,
17199 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17200 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17201 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17202 %}
17203 ins_pipe(pipe_class_memory);
17204 %}
17205
17206 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17207 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17208 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17209 iRegI_R0 result, rFlagsReg cr)
17210 %{
17211 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17212 match(Set result (EncodeISOArray src (Binary dst len)));
17213 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17214 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17215
17216 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17217 ins_encode %{
17218 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17219 $result$$Register, true,
17220 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17221 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17222 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17223 %}
17224 ins_pipe(pipe_class_memory);
17225 %}
17226
17227 //----------------------------- CompressBits/ExpandBits ------------------------
17228
17229 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17230 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17231 match(Set dst (CompressBits src mask));
17232 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17233 format %{ "mov $tsrc, $src\n\t"
17234 "mov $tmask, $mask\n\t"
17235 "bext $tdst, $tsrc, $tmask\n\t"
17236 "mov $dst, $tdst"
17237 %}
17238 ins_encode %{
17239 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17240 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17241 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17242 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17243 %}
17244 ins_pipe(pipe_slow);
17245 %}
17246
17247 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17248 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17249 match(Set dst (CompressBits (LoadI mem) mask));
17250 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17251 format %{ "ldrs $tsrc, $mem\n\t"
17252 "ldrs $tmask, $mask\n\t"
17253 "bext $tdst, $tsrc, $tmask\n\t"
17254 "mov $dst, $tdst"
17255 %}
17256 ins_encode %{
17257 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17258 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17259 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17260 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17261 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17262 %}
17263 ins_pipe(pipe_slow);
17264 %}
17265
17266 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17267 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17268 match(Set dst (CompressBits src mask));
17269 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17270 format %{ "mov $tsrc, $src\n\t"
17271 "mov $tmask, $mask\n\t"
17272 "bext $tdst, $tsrc, $tmask\n\t"
17273 "mov $dst, $tdst"
17274 %}
17275 ins_encode %{
17276 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17277 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17278 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17279 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17280 %}
17281 ins_pipe(pipe_slow);
17282 %}
17283
17284 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17285 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17286 match(Set dst (CompressBits (LoadL mem) mask));
17287 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17288 format %{ "ldrd $tsrc, $mem\n\t"
17289 "ldrd $tmask, $mask\n\t"
17290 "bext $tdst, $tsrc, $tmask\n\t"
17291 "mov $dst, $tdst"
17292 %}
17293 ins_encode %{
17294 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17295 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17296 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17297 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17298 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17299 %}
17300 ins_pipe(pipe_slow);
17301 %}
17302
17303 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17304 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17305 match(Set dst (ExpandBits src mask));
17306 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17307 format %{ "mov $tsrc, $src\n\t"
17308 "mov $tmask, $mask\n\t"
17309 "bdep $tdst, $tsrc, $tmask\n\t"
17310 "mov $dst, $tdst"
17311 %}
17312 ins_encode %{
17313 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17314 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17315 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17316 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17317 %}
17318 ins_pipe(pipe_slow);
17319 %}
17320
17321 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17322 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17323 match(Set dst (ExpandBits (LoadI mem) mask));
17324 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17325 format %{ "ldrs $tsrc, $mem\n\t"
17326 "ldrs $tmask, $mask\n\t"
17327 "bdep $tdst, $tsrc, $tmask\n\t"
17328 "mov $dst, $tdst"
17329 %}
17330 ins_encode %{
17331 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17332 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17333 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17334 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17335 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17336 %}
17337 ins_pipe(pipe_slow);
17338 %}
17339
17340 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17341 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17342 match(Set dst (ExpandBits src mask));
17343 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17344 format %{ "mov $tsrc, $src\n\t"
17345 "mov $tmask, $mask\n\t"
17346 "bdep $tdst, $tsrc, $tmask\n\t"
17347 "mov $dst, $tdst"
17348 %}
17349 ins_encode %{
17350 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17351 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17352 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17353 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17354 %}
17355 ins_pipe(pipe_slow);
17356 %}
17357
17358
17359 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17360 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17361 match(Set dst (ExpandBits (LoadL mem) mask));
17362 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17363 format %{ "ldrd $tsrc, $mem\n\t"
17364 "ldrd $tmask, $mask\n\t"
17365 "bdep $tdst, $tsrc, $tmask\n\t"
17366 "mov $dst, $tdst"
17367 %}
17368 ins_encode %{
17369 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17370 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17371 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17372 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17373 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17374 %}
17375 ins_pipe(pipe_slow);
17376 %}
17377
17378 //----------------------------- Reinterpret ----------------------------------
17379 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17380 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17381 match(Set dst (ReinterpretHF2S src));
17382 format %{ "reinterpretHF2S $dst, $src" %}
17383 ins_encode %{
17384 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17385 %}
17386 ins_pipe(pipe_slow);
17387 %}
17388
17389 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17390 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17391 match(Set dst (ReinterpretS2HF src));
17392 format %{ "reinterpretS2HF $dst, $src" %}
17393 ins_encode %{
17394 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17395 %}
17396 ins_pipe(pipe_slow);
17397 %}
17398
17399 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17400 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17401 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
17402 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
17403 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
17404 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17405 // can be omitted in this pattern, resulting in -
17406 // fcvt $dst, $src // Convert float to half-precision float
17407 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17408 %{
17409 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17410 format %{ "convF2HFAndS2HF $dst, $src" %}
17411 ins_encode %{
17412 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17413 %}
17414 ins_pipe(pipe_slow);
17415 %}
17416
17417 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17418 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17419 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
17420 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17421 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
17422 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17423 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17424 // resulting in -
17425 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
17426 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17427 %{
17428 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17429 format %{ "convHF2SAndHF2F $dst, $src" %}
17430 ins_encode %{
17431 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17432 %}
17433 ins_pipe(pipe_slow);
17434 %}
17435
17436 // ============================================================================
17437 // This name is KNOWN by the ADLC and cannot be changed.
17438 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17439 // for this guy.
17440 instruct tlsLoadP(thread_RegP dst)
17441 %{
17442 match(Set dst (ThreadLocal));
17443
17444 ins_cost(0);
17445
17446 format %{ " -- \t// $dst=Thread::current(), empty" %}
17447
17448 size(0);
17449
17450 ins_encode( /*empty*/ );
17451
17452 ins_pipe(pipe_class_empty);
17453 %}
17454
17455 //----------PEEPHOLE RULES-----------------------------------------------------
17456 // These must follow all instruction definitions as they use the names
17457 // defined in the instructions definitions.
17458 //
17459 // peepmatch ( root_instr_name [preceding_instruction]* );
17460 //
17461 // peepconstraint %{
17462 // (instruction_number.operand_name relational_op instruction_number.operand_name
17463 // [, ...] );
17464 // // instruction numbers are zero-based using left to right order in peepmatch
17465 //
17466 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17467 // // provide an instruction_number.operand_name for each operand that appears
17468 // // in the replacement instruction's match rule
17469 //
17470 // ---------VM FLAGS---------------------------------------------------------
17471 //
17472 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17473 //
17474 // Each peephole rule is given an identifying number starting with zero and
17475 // increasing by one in the order seen by the parser. An individual peephole
17476 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17477 // on the command-line.
17478 //
17479 // ---------CURRENT LIMITATIONS----------------------------------------------
17480 //
17481 // Only match adjacent instructions in same basic block
17482 // Only equality constraints
17483 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17484 // Only one replacement instruction
17485 //
17486 // ---------EXAMPLE----------------------------------------------------------
17487 //
17488 // // pertinent parts of existing instructions in architecture description
17489 // instruct movI(iRegINoSp dst, iRegI src)
17490 // %{
17491 // match(Set dst (CopyI src));
17492 // %}
17493 //
17494 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17495 // %{
17496 // match(Set dst (AddI dst src));
17497 // effect(KILL cr);
17498 // %}
17499 //
17500 // // Change (inc mov) to lea
17501 // peephole %{
17502 // // increment preceded by register-register move
17503 // peepmatch ( incI_iReg movI );
17504 // // require that the destination register of the increment
17505 // // match the destination register of the move
17506 // peepconstraint ( 0.dst == 1.dst );
17507 // // construct a replacement instruction that sets
17508 // // the destination to ( move's source register + one )
17509 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17510 // %}
17511 //
17512
17513 // Implementation no longer uses movX instructions since
17514 // machine-independent system no longer uses CopyX nodes.
17515 //
17516 // peephole
17517 // %{
17518 // peepmatch (incI_iReg movI);
17519 // peepconstraint (0.dst == 1.dst);
17520 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17521 // %}
17522
17523 // peephole
17524 // %{
17525 // peepmatch (decI_iReg movI);
17526 // peepconstraint (0.dst == 1.dst);
17527 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17528 // %}
17529
17530 // peephole
17531 // %{
17532 // peepmatch (addI_iReg_imm movI);
17533 // peepconstraint (0.dst == 1.dst);
17534 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17535 // %}
17536
17537 // peephole
17538 // %{
17539 // peepmatch (incL_iReg movL);
17540 // peepconstraint (0.dst == 1.dst);
17541 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17542 // %}
17543
17544 // peephole
17545 // %{
17546 // peepmatch (decL_iReg movL);
17547 // peepconstraint (0.dst == 1.dst);
17548 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17549 // %}
17550
17551 // peephole
17552 // %{
17553 // peepmatch (addL_iReg_imm movL);
17554 // peepconstraint (0.dst == 1.dst);
17555 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17556 // %}
17557
17558 // peephole
17559 // %{
17560 // peepmatch (addP_iReg_imm movP);
17561 // peepconstraint (0.dst == 1.dst);
17562 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17563 // %}
17564
17565 // // Change load of spilled value to only a spill
17566 // instruct storeI(memory mem, iRegI src)
17567 // %{
17568 // match(Set mem (StoreI mem src));
17569 // %}
17570 //
17571 // instruct loadI(iRegINoSp dst, memory mem)
17572 // %{
17573 // match(Set dst (LoadI mem));
17574 // %}
17575 //
17576
17577 //----------SMARTSPILL RULES---------------------------------------------------
17578 // These must follow all instruction definitions as they use the names
17579 // defined in the instructions definitions.
17580
17581 // Local Variables:
17582 // mode: c++
17583 // End: