1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for vector register V10
885 reg_class v10_veca_reg(
886 V10, V10_H, V10_J, V10_K
887 );
888
889 // Class for vector register V11
890 reg_class v11_veca_reg(
891 V11, V11_H, V11_J, V11_K
892 );
893
894 // Class for vector register V12
895 reg_class v12_veca_reg(
896 V12, V12_H, V12_J, V12_K
897 );
898
899 // Class for vector register V13
900 reg_class v13_veca_reg(
901 V13, V13_H, V13_J, V13_K
902 );
903
904 // Class for vector register V17
905 reg_class v17_veca_reg(
906 V17, V17_H, V17_J, V17_K
907 );
908
909 // Class for vector register V18
910 reg_class v18_veca_reg(
911 V18, V18_H, V18_J, V18_K
912 );
913
914 // Class for vector register V23
915 reg_class v23_veca_reg(
916 V23, V23_H, V23_J, V23_K
917 );
918
919 // Class for vector register V24
920 reg_class v24_veca_reg(
921 V24, V24_H, V24_J, V24_K
922 );
923
924 // Class for 128 bit register v0
925 reg_class v0_reg(
926 V0, V0_H
927 );
928
929 // Class for 128 bit register v1
930 reg_class v1_reg(
931 V1, V1_H
932 );
933
934 // Class for 128 bit register v2
935 reg_class v2_reg(
936 V2, V2_H
937 );
938
939 // Class for 128 bit register v3
940 reg_class v3_reg(
941 V3, V3_H
942 );
943
944 // Class for 128 bit register v4
945 reg_class v4_reg(
946 V4, V4_H
947 );
948
949 // Class for 128 bit register v5
950 reg_class v5_reg(
951 V5, V5_H
952 );
953
954 // Class for 128 bit register v6
955 reg_class v6_reg(
956 V6, V6_H
957 );
958
959 // Class for 128 bit register v7
960 reg_class v7_reg(
961 V7, V7_H
962 );
963
964 // Class for 128 bit register v8
965 reg_class v8_reg(
966 V8, V8_H
967 );
968
969 // Class for 128 bit register v9
970 reg_class v9_reg(
971 V9, V9_H
972 );
973
974 // Class for 128 bit register v10
975 reg_class v10_reg(
976 V10, V10_H
977 );
978
979 // Class for 128 bit register v11
980 reg_class v11_reg(
981 V11, V11_H
982 );
983
984 // Class for 128 bit register v12
985 reg_class v12_reg(
986 V12, V12_H
987 );
988
989 // Class for 128 bit register v13
990 reg_class v13_reg(
991 V13, V13_H
992 );
993
994 // Class for 128 bit register v14
995 reg_class v14_reg(
996 V14, V14_H
997 );
998
999 // Class for 128 bit register v15
1000 reg_class v15_reg(
1001 V15, V15_H
1002 );
1003
1004 // Class for 128 bit register v16
1005 reg_class v16_reg(
1006 V16, V16_H
1007 );
1008
1009 // Class for 128 bit register v17
1010 reg_class v17_reg(
1011 V17, V17_H
1012 );
1013
1014 // Class for 128 bit register v18
1015 reg_class v18_reg(
1016 V18, V18_H
1017 );
1018
1019 // Class for 128 bit register v19
1020 reg_class v19_reg(
1021 V19, V19_H
1022 );
1023
1024 // Class for 128 bit register v20
1025 reg_class v20_reg(
1026 V20, V20_H
1027 );
1028
1029 // Class for 128 bit register v21
1030 reg_class v21_reg(
1031 V21, V21_H
1032 );
1033
1034 // Class for 128 bit register v22
1035 reg_class v22_reg(
1036 V22, V22_H
1037 );
1038
1039 // Class for 128 bit register v23
1040 reg_class v23_reg(
1041 V23, V23_H
1042 );
1043
1044 // Class for 128 bit register v24
1045 reg_class v24_reg(
1046 V24, V24_H
1047 );
1048
1049 // Class for 128 bit register v25
1050 reg_class v25_reg(
1051 V25, V25_H
1052 );
1053
1054 // Class for 128 bit register v26
1055 reg_class v26_reg(
1056 V26, V26_H
1057 );
1058
1059 // Class for 128 bit register v27
1060 reg_class v27_reg(
1061 V27, V27_H
1062 );
1063
1064 // Class for 128 bit register v28
1065 reg_class v28_reg(
1066 V28, V28_H
1067 );
1068
1069 // Class for 128 bit register v29
1070 reg_class v29_reg(
1071 V29, V29_H
1072 );
1073
1074 // Class for 128 bit register v30
1075 reg_class v30_reg(
1076 V30, V30_H
1077 );
1078
1079 // Class for 128 bit register v31
1080 reg_class v31_reg(
1081 V31, V31_H
1082 );
1083
1084 // Class for all SVE predicate registers.
1085 reg_class pr_reg (
1086 P0,
1087 P1,
1088 P2,
1089 P3,
1090 P4,
1091 P5,
1092 P6,
1093 // P7, non-allocatable, preserved with all elements preset to TRUE.
1094 P8,
1095 P9,
1096 P10,
1097 P11,
1098 P12,
1099 P13,
1100 P14,
1101 P15
1102 );
1103
1104 // Class for SVE governing predicate registers, which are used
1105 // to determine the active elements of a predicated instruction.
1106 reg_class gov_pr (
1107 P0,
1108 P1,
1109 P2,
1110 P3,
1111 P4,
1112 P5,
1113 P6,
1114 // P7, non-allocatable, preserved with all elements preset to TRUE.
1115 );
1116
1117 reg_class p0_reg(P0);
1118 reg_class p1_reg(P1);
1119
1120 // Singleton class for condition codes
1121 reg_class int_flags(RFLAGS);
1122
1123 %}
1124
1125 //----------DEFINITION BLOCK---------------------------------------------------
1126 // Define name --> value mappings to inform the ADLC of an integer valued name
1127 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1128 // Format:
1129 // int_def <name> ( <int_value>, <expression>);
1130 // Generated Code in ad_<arch>.hpp
1131 // #define <name> (<expression>)
1132 // // value == <int_value>
1133 // Generated code in ad_<arch>.cpp adlc_verification()
1134 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1135 //
1136
1137 // we follow the ppc-aix port in using a simple cost model which ranks
1138 // register operations as cheap, memory ops as more expensive and
1139 // branches as most expensive. the first two have a low as well as a
1140 // normal cost. huge cost appears to be a way of saying don't do
1141 // something
1142
1143 definitions %{
1144 // The default cost (of a register move instruction).
1145 int_def INSN_COST ( 100, 100);
1146 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1147 int_def CALL_COST ( 200, 2 * INSN_COST);
1148 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1149 %}
1150
1151
1152 //----------SOURCE BLOCK-------------------------------------------------------
1153 // This is a block of C++ code which provides values, functions, and
1154 // definitions necessary in the rest of the architecture description
1155
1156 source_hpp %{
1157
1158 #include "asm/macroAssembler.hpp"
1159 #include "gc/shared/barrierSetAssembler.hpp"
1160 #include "gc/shared/cardTable.hpp"
1161 #include "gc/shared/cardTableBarrierSet.hpp"
1162 #include "gc/shared/collectedHeap.hpp"
1163 #include "opto/addnode.hpp"
1164 #include "opto/convertnode.hpp"
1165 #include "runtime/objectMonitor.hpp"
1166
1167 extern RegMask _ANY_REG32_mask;
1168 extern RegMask _ANY_REG_mask;
1169 extern RegMask _PTR_REG_mask;
1170 extern RegMask _NO_SPECIAL_REG32_mask;
1171 extern RegMask _NO_SPECIAL_REG_mask;
1172 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1173 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1174
1175 class CallStubImpl {
1176
1177 //--------------------------------------------------------------
1178 //---< Used for optimization in Compile::shorten_branches >---
1179 //--------------------------------------------------------------
1180
1181 public:
1182 // Size of call trampoline stub.
1183 static uint size_call_trampoline() {
1184 return MacroAssembler::max_trampoline_stub_size(); // no call trampolines on this platform
1185 }
1186
1187 // number of relocations needed by a call trampoline stub
1188 static uint reloc_call_trampoline() {
1189 return 0; // no call trampolines on this platform
1190 }
1191 };
1192
1193 class HandlerImpl {
1194
1195 public:
1196
1197 static int emit_exception_handler(C2_MacroAssembler *masm);
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_exception_handler() {
1201 return MacroAssembler::far_codestub_branch_size();
1202 }
1203
1204 static uint size_deopt_handler() {
1205 // count one adr and one far branch instruction
1206 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1207 }
1208 };
1209
1210 class Node::PD {
1211 public:
1212 enum NodeFlags {
1213 _last_flag = Node::_last_flag
1214 };
1215 };
1216
1217 bool is_CAS(int opcode, bool maybe_volatile);
1218
1219 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1220
1221 bool unnecessary_acquire(const Node *barrier);
1222 bool needs_acquiring_load(const Node *load);
1223
1224 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1225
1226 bool unnecessary_release(const Node *barrier);
1227 bool unnecessary_volatile(const Node *barrier);
1228 bool needs_releasing_store(const Node *store);
1229
1230 // predicate controlling translation of CompareAndSwapX
1231 bool needs_acquiring_load_exclusive(const Node *load);
1232
1233 // predicate controlling addressing modes
1234 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1235
1236 // Convert BootTest condition to Assembler condition.
1237 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1238 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1239 %}
1240
1241 source %{
1242
1243 // Derived RegMask with conditionally allocatable registers
1244
1245 void PhaseOutput::pd_perform_mach_node_analysis() {
1246 }
1247
1248 int MachNode::pd_alignment_required() const {
1249 return 1;
1250 }
1251
1252 int MachNode::compute_padding(int current_offset) const {
1253 return 0;
1254 }
1255
1256 RegMask _ANY_REG32_mask;
1257 RegMask _ANY_REG_mask;
1258 RegMask _PTR_REG_mask;
1259 RegMask _NO_SPECIAL_REG32_mask;
1260 RegMask _NO_SPECIAL_REG_mask;
1261 RegMask _NO_SPECIAL_PTR_REG_mask;
1262 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1263
1264 void reg_mask_init() {
1265 // We derive below RegMask(s) from the ones which are auto-generated from
1266 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1267 // registers conditionally reserved.
1268
1269 _ANY_REG32_mask = _ALL_REG32_mask;
1270 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1271
1272 _ANY_REG_mask = _ALL_REG_mask;
1273
1274 _PTR_REG_mask = _ALL_REG_mask;
1275
1276 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1277 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1278
1279 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1280 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1281
1282 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1283 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1284
1285 // r27 is not allocatable when compressed oops is on and heapbase is not
1286 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1287 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1288 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1289 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1290 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1291 }
1292
1293 // r29 is not allocatable when PreserveFramePointer is on
1294 if (PreserveFramePointer) {
1295 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1296 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1297 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1301 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1302 }
1303
1304 // Optimizaton of volatile gets and puts
1305 // -------------------------------------
1306 //
1307 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1308 // use to implement volatile reads and writes. For a volatile read
1309 // we simply need
1310 //
1311 // ldar<x>
1312 //
1313 // and for a volatile write we need
1314 //
1315 // stlr<x>
1316 //
1317 // Alternatively, we can implement them by pairing a normal
1318 // load/store with a memory barrier. For a volatile read we need
1319 //
1320 // ldr<x>
1321 // dmb ishld
1322 //
1323 // for a volatile write
1324 //
1325 // dmb ish
1326 // str<x>
1327 // dmb ish
1328 //
1329 // We can also use ldaxr and stlxr to implement compare and swap CAS
1330 // sequences. These are normally translated to an instruction
1331 // sequence like the following
1332 //
1333 // dmb ish
1334 // retry:
1335 // ldxr<x> rval raddr
1336 // cmp rval rold
1337 // b.ne done
1338 // stlxr<x> rval, rnew, rold
1339 // cbnz rval retry
1340 // done:
1341 // cset r0, eq
1342 // dmb ishld
1343 //
1344 // Note that the exclusive store is already using an stlxr
1345 // instruction. That is required to ensure visibility to other
1346 // threads of the exclusive write (assuming it succeeds) before that
1347 // of any subsequent writes.
1348 //
1349 // The following instruction sequence is an improvement on the above
1350 //
1351 // retry:
1352 // ldaxr<x> rval raddr
1353 // cmp rval rold
1354 // b.ne done
1355 // stlxr<x> rval, rnew, rold
1356 // cbnz rval retry
1357 // done:
1358 // cset r0, eq
1359 //
1360 // We don't need the leading dmb ish since the stlxr guarantees
1361 // visibility of prior writes in the case that the swap is
1362 // successful. Crucially we don't have to worry about the case where
1363 // the swap is not successful since no valid program should be
1364 // relying on visibility of prior changes by the attempting thread
1365 // in the case where the CAS fails.
1366 //
1367 // Similarly, we don't need the trailing dmb ishld if we substitute
1368 // an ldaxr instruction since that will provide all the guarantees we
1369 // require regarding observation of changes made by other threads
1370 // before any change to the CAS address observed by the load.
1371 //
1372 // In order to generate the desired instruction sequence we need to
1373 // be able to identify specific 'signature' ideal graph node
1374 // sequences which i) occur as a translation of a volatile reads or
1375 // writes or CAS operations and ii) do not occur through any other
1376 // translation or graph transformation. We can then provide
1377 // alternative aldc matching rules which translate these node
1378 // sequences to the desired machine code sequences. Selection of the
1379 // alternative rules can be implemented by predicates which identify
1380 // the relevant node sequences.
1381 //
1382 // The ideal graph generator translates a volatile read to the node
1383 // sequence
1384 //
1385 // LoadX[mo_acquire]
1386 // MemBarAcquire
1387 //
1388 // As a special case when using the compressed oops optimization we
1389 // may also see this variant
1390 //
1391 // LoadN[mo_acquire]
1392 // DecodeN
1393 // MemBarAcquire
1394 //
1395 // A volatile write is translated to the node sequence
1396 //
1397 // MemBarRelease
1398 // StoreX[mo_release] {CardMark}-optional
1399 // MemBarVolatile
1400 //
1401 // n.b. the above node patterns are generated with a strict
1402 // 'signature' configuration of input and output dependencies (see
1403 // the predicates below for exact details). The card mark may be as
1404 // simple as a few extra nodes or, in a few GC configurations, may
1405 // include more complex control flow between the leading and
1406 // trailing memory barriers. However, whatever the card mark
1407 // configuration these signatures are unique to translated volatile
1408 // reads/stores -- they will not appear as a result of any other
1409 // bytecode translation or inlining nor as a consequence of
1410 // optimizing transforms.
1411 //
1412 // We also want to catch inlined unsafe volatile gets and puts and
1413 // be able to implement them using either ldar<x>/stlr<x> or some
1414 // combination of ldr<x>/stlr<x> and dmb instructions.
1415 //
1416 // Inlined unsafe volatiles puts manifest as a minor variant of the
1417 // normal volatile put node sequence containing an extra cpuorder
1418 // membar
1419 //
1420 // MemBarRelease
1421 // MemBarCPUOrder
1422 // StoreX[mo_release] {CardMark}-optional
1423 // MemBarCPUOrder
1424 // MemBarVolatile
1425 //
1426 // n.b. as an aside, a cpuorder membar is not itself subject to
1427 // matching and translation by adlc rules. However, the rule
1428 // predicates need to detect its presence in order to correctly
1429 // select the desired adlc rules.
1430 //
1431 // Inlined unsafe volatile gets manifest as a slightly different
1432 // node sequence to a normal volatile get because of the
1433 // introduction of some CPUOrder memory barriers to bracket the
1434 // Load. However, but the same basic skeleton of a LoadX feeding a
1435 // MemBarAcquire, possibly through an optional DecodeN, is still
1436 // present
1437 //
1438 // MemBarCPUOrder
1439 // || \\
1440 // MemBarCPUOrder LoadX[mo_acquire]
1441 // || |
1442 // || {DecodeN} optional
1443 // || /
1444 // MemBarAcquire
1445 //
1446 // In this case the acquire membar does not directly depend on the
1447 // load. However, we can be sure that the load is generated from an
1448 // inlined unsafe volatile get if we see it dependent on this unique
1449 // sequence of membar nodes. Similarly, given an acquire membar we
1450 // can know that it was added because of an inlined unsafe volatile
1451 // get if it is fed and feeds a cpuorder membar and if its feed
1452 // membar also feeds an acquiring load.
1453 //
1454 // Finally an inlined (Unsafe) CAS operation is translated to the
1455 // following ideal graph
1456 //
1457 // MemBarRelease
1458 // MemBarCPUOrder
1459 // CompareAndSwapX {CardMark}-optional
1460 // MemBarCPUOrder
1461 // MemBarAcquire
1462 //
1463 // So, where we can identify these volatile read and write
1464 // signatures we can choose to plant either of the above two code
1465 // sequences. For a volatile read we can simply plant a normal
1466 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1467 // also choose to inhibit translation of the MemBarAcquire and
1468 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1469 //
1470 // When we recognise a volatile store signature we can choose to
1471 // plant at a dmb ish as a translation for the MemBarRelease, a
1472 // normal str<x> and then a dmb ish for the MemBarVolatile.
1473 // Alternatively, we can inhibit translation of the MemBarRelease
1474 // and MemBarVolatile and instead plant a simple stlr<x>
1475 // instruction.
1476 //
1477 // when we recognise a CAS signature we can choose to plant a dmb
1478 // ish as a translation for the MemBarRelease, the conventional
1479 // macro-instruction sequence for the CompareAndSwap node (which
1480 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1481 // Alternatively, we can elide generation of the dmb instructions
1482 // and plant the alternative CompareAndSwap macro-instruction
1483 // sequence (which uses ldaxr<x>).
1484 //
1485 // Of course, the above only applies when we see these signature
1486 // configurations. We still want to plant dmb instructions in any
1487 // other cases where we may see a MemBarAcquire, MemBarRelease or
1488 // MemBarVolatile. For example, at the end of a constructor which
1489 // writes final/volatile fields we will see a MemBarRelease
1490 // instruction and this needs a 'dmb ish' lest we risk the
1491 // constructed object being visible without making the
1492 // final/volatile field writes visible.
1493 //
1494 // n.b. the translation rules below which rely on detection of the
1495 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1496 // If we see anything other than the signature configurations we
1497 // always just translate the loads and stores to ldr<x> and str<x>
1498 // and translate acquire, release and volatile membars to the
1499 // relevant dmb instructions.
1500 //
1501
1502 // is_CAS(int opcode, bool maybe_volatile)
1503 //
1504 // return true if opcode is one of the possible CompareAndSwapX
1505 // values otherwise false.
1506
1507 bool is_CAS(int opcode, bool maybe_volatile)
1508 {
1509 switch(opcode) {
1510 // We handle these
1511 case Op_CompareAndSwapI:
1512 case Op_CompareAndSwapL:
1513 case Op_CompareAndSwapP:
1514 case Op_CompareAndSwapN:
1515 case Op_ShenandoahCompareAndSwapP:
1516 case Op_ShenandoahCompareAndSwapN:
1517 case Op_CompareAndSwapB:
1518 case Op_CompareAndSwapS:
1519 case Op_GetAndSetI:
1520 case Op_GetAndSetL:
1521 case Op_GetAndSetP:
1522 case Op_GetAndSetN:
1523 case Op_GetAndAddI:
1524 case Op_GetAndAddL:
1525 return true;
1526 case Op_CompareAndExchangeI:
1527 case Op_CompareAndExchangeN:
1528 case Op_CompareAndExchangeB:
1529 case Op_CompareAndExchangeS:
1530 case Op_CompareAndExchangeL:
1531 case Op_CompareAndExchangeP:
1532 case Op_WeakCompareAndSwapB:
1533 case Op_WeakCompareAndSwapS:
1534 case Op_WeakCompareAndSwapI:
1535 case Op_WeakCompareAndSwapL:
1536 case Op_WeakCompareAndSwapP:
1537 case Op_WeakCompareAndSwapN:
1538 case Op_ShenandoahWeakCompareAndSwapP:
1539 case Op_ShenandoahWeakCompareAndSwapN:
1540 case Op_ShenandoahCompareAndExchangeP:
1541 case Op_ShenandoahCompareAndExchangeN:
1542 return maybe_volatile;
1543 default:
1544 return false;
1545 }
1546 }
1547
1548 // helper to determine the maximum number of Phi nodes we may need to
1549 // traverse when searching from a card mark membar for the merge mem
1550 // feeding a trailing membar or vice versa
1551
1552 // predicates controlling emit of ldr<x>/ldar<x>
1553
1554 bool unnecessary_acquire(const Node *barrier)
1555 {
1556 assert(barrier->is_MemBar(), "expecting a membar");
1557
1558 MemBarNode* mb = barrier->as_MemBar();
1559
1560 if (mb->trailing_load()) {
1561 return true;
1562 }
1563
1564 if (mb->trailing_load_store()) {
1565 Node* load_store = mb->in(MemBarNode::Precedent);
1566 assert(load_store->is_LoadStore(), "unexpected graph shape");
1567 return is_CAS(load_store->Opcode(), true);
1568 }
1569
1570 return false;
1571 }
1572
1573 bool needs_acquiring_load(const Node *n)
1574 {
1575 assert(n->is_Load(), "expecting a load");
1576 LoadNode *ld = n->as_Load();
1577 return ld->is_acquire();
1578 }
1579
1580 bool unnecessary_release(const Node *n)
1581 {
1582 assert((n->is_MemBar() &&
1583 n->Opcode() == Op_MemBarRelease),
1584 "expecting a release membar");
1585
1586 MemBarNode *barrier = n->as_MemBar();
1587 if (!barrier->leading()) {
1588 return false;
1589 } else {
1590 Node* trailing = barrier->trailing_membar();
1591 MemBarNode* trailing_mb = trailing->as_MemBar();
1592 assert(trailing_mb->trailing(), "Not a trailing membar?");
1593 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1594
1595 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1596 if (mem->is_Store()) {
1597 assert(mem->as_Store()->is_release(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1599 return true;
1600 } else {
1601 assert(mem->is_LoadStore(), "");
1602 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1603 return is_CAS(mem->Opcode(), true);
1604 }
1605 }
1606 return false;
1607 }
1608
1609 bool unnecessary_volatile(const Node *n)
1610 {
1611 // assert n->is_MemBar();
1612 MemBarNode *mbvol = n->as_MemBar();
1613
1614 bool release = mbvol->trailing_store();
1615 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1616 #ifdef ASSERT
1617 if (release) {
1618 Node* leading = mbvol->leading_membar();
1619 assert(leading->Opcode() == Op_MemBarRelease, "");
1620 assert(leading->as_MemBar()->leading_store(), "");
1621 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1622 }
1623 #endif
1624
1625 return release;
1626 }
1627
1628 // predicates controlling emit of str<x>/stlr<x>
1629
1630 bool needs_releasing_store(const Node *n)
1631 {
1632 // assert n->is_Store();
1633 StoreNode *st = n->as_Store();
1634 return st->trailing_membar() != nullptr;
1635 }
1636
1637 // predicate controlling translation of CAS
1638 //
1639 // returns true if CAS needs to use an acquiring load otherwise false
1640
1641 bool needs_acquiring_load_exclusive(const Node *n)
1642 {
1643 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1644 LoadStoreNode* ldst = n->as_LoadStore();
1645 if (is_CAS(n->Opcode(), false)) {
1646 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1647 } else {
1648 return ldst->trailing_membar() != nullptr;
1649 }
1650
1651 // so we can just return true here
1652 return true;
1653 }
1654
1655 #define __ masm->
1656
1657 // advance declarations for helper functions to convert register
1658 // indices to register objects
1659
1660 // the ad file has to provide implementations of certain methods
1661 // expected by the generic code
1662 //
1663 // REQUIRED FUNCTIONALITY
1664
1665 //=============================================================================
1666
1667 // !!!!! Special hack to get all types of calls to specify the byte offset
1668 // from the start of the call to the point where the return address
1669 // will point.
1670
1671 int MachCallStaticJavaNode::ret_addr_offset()
1672 {
1673 // call should be a simple bl
1674 int off = 4;
1675 return off;
1676 }
1677
1678 int MachCallDynamicJavaNode::ret_addr_offset()
1679 {
1680 return 16; // movz, movk, movk, bl
1681 }
1682
1683 int MachCallRuntimeNode::ret_addr_offset() {
1684 // for generated stubs the call will be
1685 // bl(addr)
1686 // or with far branches
1687 // bl(trampoline_stub)
1688 // for real runtime callouts it will be six instructions
1689 // see aarch64_enc_java_to_runtime
1690 // adr(rscratch2, retaddr)
1691 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1692 // lea(rscratch1, RuntimeAddress(addr)
1693 // blr(rscratch1)
1694 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1695 if (cb) {
1696 return 1 * NativeInstruction::instruction_size;
1697 } else {
1698 return 6 * NativeInstruction::instruction_size;
1699 }
1700 }
1701
1702 //=============================================================================
1703
1704 #ifndef PRODUCT
1705 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1706 st->print("BREAKPOINT");
1707 }
1708 #endif
1709
1710 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1711 __ brk(0);
1712 }
1713
1714 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1715 return MachNode::size(ra_);
1716 }
1717
1718 //=============================================================================
1719
1720 #ifndef PRODUCT
1721 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1722 st->print("nop \t# %d bytes pad for loops and calls", _count);
1723 }
1724 #endif
1725
1726 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1727 for (int i = 0; i < _count; i++) {
1728 __ nop();
1729 }
1730 }
1731
1732 uint MachNopNode::size(PhaseRegAlloc*) const {
1733 return _count * NativeInstruction::instruction_size;
1734 }
1735
1736 //=============================================================================
1737 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1738
1739 int ConstantTable::calculate_table_base_offset() const {
1740 return 0; // absolute addressing, no offset
1741 }
1742
1743 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1744 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1745 ShouldNotReachHere();
1746 }
1747
1748 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1749 // Empty encoding
1750 }
1751
1752 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1753 return 0;
1754 }
1755
1756 #ifndef PRODUCT
1757 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1758 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1759 }
1760 #endif
1761
1762 #ifndef PRODUCT
1763 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1764 Compile* C = ra_->C;
1765
1766 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1767
1768 if (C->output()->need_stack_bang(framesize))
1769 st->print("# stack bang size=%d\n\t", framesize);
1770
1771 if (VM_Version::use_rop_protection()) {
1772 st->print("ldr zr, [lr]\n\t");
1773 st->print("paciaz\n\t");
1774 }
1775 if (framesize < ((1 << 9) + 2 * wordSize)) {
1776 st->print("sub sp, sp, #%d\n\t", framesize);
1777 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1778 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1779 } else {
1780 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1781 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1782 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1783 st->print("sub sp, sp, rscratch1");
1784 }
1785 if (C->stub_function() == nullptr) {
1786 st->print("\n\t");
1787 st->print("ldr rscratch1, [guard]\n\t");
1788 st->print("dmb ishld\n\t");
1789 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1790 st->print("cmp rscratch1, rscratch2\n\t");
1791 st->print("b.eq skip");
1792 st->print("\n\t");
1793 st->print("blr #nmethod_entry_barrier_stub\n\t");
1794 st->print("b skip\n\t");
1795 st->print("guard: int\n\t");
1796 st->print("\n\t");
1797 st->print("skip:\n\t");
1798 }
1799 }
1800 #endif
1801
1802 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1803 Compile* C = ra_->C;
1804
1805 // n.b. frame size includes space for return pc and rfp
1806 const int framesize = C->output()->frame_size_in_bytes();
1807
1808 if (C->clinit_barrier_on_entry()) {
1809 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1810
1811 Label L_skip_barrier;
1812
1813 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1814 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1815 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1816 __ bind(L_skip_barrier);
1817 }
1818
1819 if (C->max_vector_size() > 0) {
1820 __ reinitialize_ptrue();
1821 }
1822
1823 int bangsize = C->output()->bang_size_in_bytes();
1824 if (C->output()->need_stack_bang(bangsize))
1825 __ generate_stack_overflow_check(bangsize);
1826
1827 __ build_frame(framesize);
1828
1829 if (C->stub_function() == nullptr) {
1830 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1831 // Dummy labels for just measuring the code size
1832 Label dummy_slow_path;
1833 Label dummy_continuation;
1834 Label dummy_guard;
1835 Label* slow_path = &dummy_slow_path;
1836 Label* continuation = &dummy_continuation;
1837 Label* guard = &dummy_guard;
1838 if (!Compile::current()->output()->in_scratch_emit_size()) {
1839 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1840 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1841 Compile::current()->output()->add_stub(stub);
1842 slow_path = &stub->entry();
1843 continuation = &stub->continuation();
1844 guard = &stub->guard();
1845 }
1846 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1847 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1848 }
1849
1850 if (VerifyStackAtCalls) {
1851 Unimplemented();
1852 }
1853
1854 C->output()->set_frame_complete(__ offset());
1855
1856 if (C->has_mach_constant_base_node()) {
1857 // NOTE: We set the table base offset here because users might be
1858 // emitted before MachConstantBaseNode.
1859 ConstantTable& constant_table = C->output()->constant_table();
1860 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1861 }
1862 }
1863
1864 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1865 {
1866 return MachNode::size(ra_); // too many variables; just compute it
1867 // the hard way
1868 }
1869
1870 int MachPrologNode::reloc() const
1871 {
1872 return 0;
1873 }
1874
1875 //=============================================================================
1876
1877 #ifndef PRODUCT
1878 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1879 Compile* C = ra_->C;
1880 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1881
1882 st->print("# pop frame %d\n\t",framesize);
1883
1884 if (framesize == 0) {
1885 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1886 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1887 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1888 st->print("add sp, sp, #%d\n\t", framesize);
1889 } else {
1890 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1891 st->print("add sp, sp, rscratch1\n\t");
1892 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1893 }
1894 if (VM_Version::use_rop_protection()) {
1895 st->print("autiaz\n\t");
1896 st->print("ldr zr, [lr]\n\t");
1897 }
1898
1899 if (do_polling() && C->is_method_compilation()) {
1900 st->print("# test polling word\n\t");
1901 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1902 st->print("cmp sp, rscratch1\n\t");
1903 st->print("bhi #slow_path");
1904 }
1905 }
1906 #endif
1907
1908 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1909 Compile* C = ra_->C;
1910 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1911
1912 __ remove_frame(framesize);
1913
1914 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1915 __ reserved_stack_check();
1916 }
1917
1918 if (do_polling() && C->is_method_compilation()) {
1919 Label dummy_label;
1920 Label* code_stub = &dummy_label;
1921 if (!C->output()->in_scratch_emit_size()) {
1922 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1923 C->output()->add_stub(stub);
1924 code_stub = &stub->entry();
1925 }
1926 __ relocate(relocInfo::poll_return_type);
1927 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1928 }
1929 }
1930
1931 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1932 // Variable size. Determine dynamically.
1933 return MachNode::size(ra_);
1934 }
1935
1936 int MachEpilogNode::reloc() const {
1937 // Return number of relocatable values contained in this instruction.
1938 return 1; // 1 for polling page.
1939 }
1940
1941 const Pipeline * MachEpilogNode::pipeline() const {
1942 return MachNode::pipeline_class();
1943 }
1944
1945 //=============================================================================
1946
1947 static enum RC rc_class(OptoReg::Name reg) {
1948
1949 if (reg == OptoReg::Bad) {
1950 return rc_bad;
1951 }
1952
1953 // we have 32 int registers * 2 halves
1954 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1955
1956 if (reg < slots_of_int_registers) {
1957 return rc_int;
1958 }
1959
1960 // we have 32 float register * 8 halves
1961 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1962 if (reg < slots_of_int_registers + slots_of_float_registers) {
1963 return rc_float;
1964 }
1965
1966 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1967 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1968 return rc_predicate;
1969 }
1970
1971 // Between predicate regs & stack is the flags.
1972 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1973
1974 return rc_stack;
1975 }
1976
1977 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1978 Compile* C = ra_->C;
1979
1980 // Get registers to move.
1981 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1982 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1983 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1984 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1985
1986 enum RC src_hi_rc = rc_class(src_hi);
1987 enum RC src_lo_rc = rc_class(src_lo);
1988 enum RC dst_hi_rc = rc_class(dst_hi);
1989 enum RC dst_lo_rc = rc_class(dst_lo);
1990
1991 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1992
1993 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1994 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1995 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1996 "expected aligned-adjacent pairs");
1997 }
1998
1999 if (src_lo == dst_lo && src_hi == dst_hi) {
2000 return 0; // Self copy, no move.
2001 }
2002
2003 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2004 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2005 int src_offset = ra_->reg2offset(src_lo);
2006 int dst_offset = ra_->reg2offset(dst_lo);
2007
2008 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2009 uint ireg = ideal_reg();
2010 if (ireg == Op_VecA && masm) {
2011 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2012 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2013 // stack->stack
2014 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2015 sve_vector_reg_size_in_bytes);
2016 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2017 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2018 sve_vector_reg_size_in_bytes);
2019 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2020 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2021 sve_vector_reg_size_in_bytes);
2022 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2023 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2024 as_FloatRegister(Matcher::_regEncode[src_lo]),
2025 as_FloatRegister(Matcher::_regEncode[src_lo]));
2026 } else {
2027 ShouldNotReachHere();
2028 }
2029 } else if (masm) {
2030 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2031 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2032 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2033 // stack->stack
2034 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2035 if (ireg == Op_VecD) {
2036 __ unspill(rscratch1, true, src_offset);
2037 __ spill(rscratch1, true, dst_offset);
2038 } else {
2039 __ spill_copy128(src_offset, dst_offset);
2040 }
2041 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2042 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2043 ireg == Op_VecD ? __ T8B : __ T16B,
2044 as_FloatRegister(Matcher::_regEncode[src_lo]));
2045 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2046 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2047 ireg == Op_VecD ? __ D : __ Q,
2048 ra_->reg2offset(dst_lo));
2049 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2050 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2051 ireg == Op_VecD ? __ D : __ Q,
2052 ra_->reg2offset(src_lo));
2053 } else {
2054 ShouldNotReachHere();
2055 }
2056 }
2057 } else if (masm) {
2058 switch (src_lo_rc) {
2059 case rc_int:
2060 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2061 if (is64) {
2062 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2063 as_Register(Matcher::_regEncode[src_lo]));
2064 } else {
2065 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2066 as_Register(Matcher::_regEncode[src_lo]));
2067 }
2068 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2069 if (is64) {
2070 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2071 as_Register(Matcher::_regEncode[src_lo]));
2072 } else {
2073 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2074 as_Register(Matcher::_regEncode[src_lo]));
2075 }
2076 } else { // gpr --> stack spill
2077 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2078 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2079 }
2080 break;
2081 case rc_float:
2082 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2083 if (is64) {
2084 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2085 as_FloatRegister(Matcher::_regEncode[src_lo]));
2086 } else {
2087 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2088 as_FloatRegister(Matcher::_regEncode[src_lo]));
2089 }
2090 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2091 if (is64) {
2092 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2093 as_FloatRegister(Matcher::_regEncode[src_lo]));
2094 } else {
2095 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2096 as_FloatRegister(Matcher::_regEncode[src_lo]));
2097 }
2098 } else { // fpr --> stack spill
2099 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2100 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2101 is64 ? __ D : __ S, dst_offset);
2102 }
2103 break;
2104 case rc_stack:
2105 if (dst_lo_rc == rc_int) { // stack --> gpr load
2106 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2107 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2108 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2109 is64 ? __ D : __ S, src_offset);
2110 } else if (dst_lo_rc == rc_predicate) {
2111 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2112 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2113 } else { // stack --> stack copy
2114 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2115 if (ideal_reg() == Op_RegVectMask) {
2116 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2117 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2118 } else {
2119 __ unspill(rscratch1, is64, src_offset);
2120 __ spill(rscratch1, is64, dst_offset);
2121 }
2122 }
2123 break;
2124 case rc_predicate:
2125 if (dst_lo_rc == rc_predicate) {
2126 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2127 } else if (dst_lo_rc == rc_stack) {
2128 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2129 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2130 } else {
2131 assert(false, "bad src and dst rc_class combination.");
2132 ShouldNotReachHere();
2133 }
2134 break;
2135 default:
2136 assert(false, "bad rc_class for spill");
2137 ShouldNotReachHere();
2138 }
2139 }
2140
2141 if (st) {
2142 st->print("spill ");
2143 if (src_lo_rc == rc_stack) {
2144 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2145 } else {
2146 st->print("%s -> ", Matcher::regName[src_lo]);
2147 }
2148 if (dst_lo_rc == rc_stack) {
2149 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2150 } else {
2151 st->print("%s", Matcher::regName[dst_lo]);
2152 }
2153 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2154 int vsize = 0;
2155 switch (ideal_reg()) {
2156 case Op_VecD:
2157 vsize = 64;
2158 break;
2159 case Op_VecX:
2160 vsize = 128;
2161 break;
2162 case Op_VecA:
2163 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2164 break;
2165 default:
2166 assert(false, "bad register type for spill");
2167 ShouldNotReachHere();
2168 }
2169 st->print("\t# vector spill size = %d", vsize);
2170 } else if (ideal_reg() == Op_RegVectMask) {
2171 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2172 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2173 st->print("\t# predicate spill size = %d", vsize);
2174 } else {
2175 st->print("\t# spill size = %d", is64 ? 64 : 32);
2176 }
2177 }
2178
2179 return 0;
2180
2181 }
2182
2183 #ifndef PRODUCT
2184 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2185 if (!ra_)
2186 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2187 else
2188 implementation(nullptr, ra_, false, st);
2189 }
2190 #endif
2191
2192 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2193 implementation(masm, ra_, false, nullptr);
2194 }
2195
2196 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2197 return MachNode::size(ra_);
2198 }
2199
2200 //=============================================================================
2201
2202 #ifndef PRODUCT
2203 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2204 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2205 int reg = ra_->get_reg_first(this);
2206 st->print("add %s, rsp, #%d]\t# box lock",
2207 Matcher::regName[reg], offset);
2208 }
2209 #endif
2210
2211 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2212 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2213 int reg = ra_->get_encode(this);
2214
2215 // This add will handle any 24-bit signed offset. 24 bits allows an
2216 // 8 megabyte stack frame.
2217 __ add(as_Register(reg), sp, offset);
2218 }
2219
2220 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2221 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2222 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2223
2224 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2225 return NativeInstruction::instruction_size;
2226 } else {
2227 return 2 * NativeInstruction::instruction_size;
2228 }
2229 }
2230
2231 //=============================================================================
2232
2233 #ifndef PRODUCT
2234 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2235 {
2236 st->print_cr("# MachUEPNode");
2237 if (UseCompressedClassPointers) {
2238 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2239 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2240 st->print_cr("\tcmpw rscratch1, r10");
2241 } else {
2242 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2243 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2244 st->print_cr("\tcmp rscratch1, r10");
2245 }
2246 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2247 }
2248 #endif
2249
2250 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2251 {
2252 __ ic_check(InteriorEntryAlignment);
2253 }
2254
2255 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2256 {
2257 return MachNode::size(ra_);
2258 }
2259
2260 // REQUIRED EMIT CODE
2261
2262 //=============================================================================
2263
2264 // Emit exception handler code.
2265 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2266 {
2267 // mov rscratch1 #exception_blob_entry_point
2268 // br rscratch1
2269 // Note that the code buffer's insts_mark is always relative to insts.
2270 // That's why we must use the macroassembler to generate a handler.
2271 address base = __ start_a_stub(size_exception_handler());
2272 if (base == nullptr) {
2273 ciEnv::current()->record_failure("CodeCache is full");
2274 return 0; // CodeBuffer::expand failed
2275 }
2276 int offset = __ offset();
2277 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2278 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2279 __ end_a_stub();
2280 return offset;
2281 }
2282
2283 // Emit deopt handler code.
2284 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2285 {
2286 // Note that the code buffer's insts_mark is always relative to insts.
2287 // That's why we must use the macroassembler to generate a handler.
2288 address base = __ start_a_stub(size_deopt_handler());
2289 if (base == nullptr) {
2290 ciEnv::current()->record_failure("CodeCache is full");
2291 return 0; // CodeBuffer::expand failed
2292 }
2293 int offset = __ offset();
2294
2295 __ adr(lr, __ pc());
2296 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2297
2298 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2299 __ end_a_stub();
2300 return offset;
2301 }
2302
2303 // REQUIRED MATCHER CODE
2304
2305 //=============================================================================
2306
2307 bool Matcher::match_rule_supported(int opcode) {
2308 if (!has_match_rule(opcode))
2309 return false;
2310
2311 switch (opcode) {
2312 case Op_OnSpinWait:
2313 return VM_Version::supports_on_spin_wait();
2314 case Op_CacheWB:
2315 case Op_CacheWBPreSync:
2316 case Op_CacheWBPostSync:
2317 if (!VM_Version::supports_data_cache_line_flush()) {
2318 return false;
2319 }
2320 break;
2321 case Op_ExpandBits:
2322 case Op_CompressBits:
2323 if (!VM_Version::supports_svebitperm()) {
2324 return false;
2325 }
2326 break;
2327 case Op_FmaF:
2328 case Op_FmaD:
2329 case Op_FmaVF:
2330 case Op_FmaVD:
2331 if (!UseFMA) {
2332 return false;
2333 }
2334 break;
2335 case Op_FmaHF:
2336 // UseFMA flag also needs to be checked along with FEAT_FP16
2337 if (!UseFMA || !is_feat_fp16_supported()) {
2338 return false;
2339 }
2340 break;
2341 case Op_AddHF:
2342 case Op_SubHF:
2343 case Op_MulHF:
2344 case Op_DivHF:
2345 case Op_MinHF:
2346 case Op_MaxHF:
2347 case Op_SqrtHF:
2348 // Half-precision floating point scalar operations require FEAT_FP16
2349 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2350 // features are supported.
2351 if (!is_feat_fp16_supported()) {
2352 return false;
2353 }
2354 break;
2355 }
2356
2357 return true; // Per default match rules are supported.
2358 }
2359
2360 const RegMask* Matcher::predicate_reg_mask(void) {
2361 return &_PR_REG_mask;
2362 }
2363
2364 bool Matcher::supports_vector_calling_convention(void) {
2365 return EnableVectorSupport;
2366 }
2367
2368 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2369 assert(EnableVectorSupport, "sanity");
2370 int lo = V0_num;
2371 int hi = V0_H_num;
2372 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2373 hi = V0_K_num;
2374 }
2375 return OptoRegPair(hi, lo);
2376 }
2377
2378 // Is this branch offset short enough that a short branch can be used?
2379 //
2380 // NOTE: If the platform does not provide any short branch variants, then
2381 // this method should return false for offset 0.
2382 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2383 // The passed offset is relative to address of the branch.
2384
2385 return (-32768 <= offset && offset < 32768);
2386 }
2387
2388 // Vector width in bytes.
2389 int Matcher::vector_width_in_bytes(BasicType bt) {
2390 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2391 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2392 // Minimum 2 values in vector
2393 if (size < 2*type2aelembytes(bt)) size = 0;
2394 // But never < 4
2395 if (size < 4) size = 0;
2396 return size;
2397 }
2398
2399 // Limits on vector size (number of elements) loaded into vector.
2400 int Matcher::max_vector_size(const BasicType bt) {
2401 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2402 }
2403
2404 int Matcher::min_vector_size(const BasicType bt) {
2405 // Usually, the shortest vector length supported by AArch64 ISA and
2406 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2407 // vectors in a few special cases.
2408 int size;
2409 switch(bt) {
2410 case T_BOOLEAN:
2411 // Load/store a vector mask with only 2 elements for vector types
2412 // such as "2I/2F/2L/2D".
2413 size = 2;
2414 break;
2415 case T_BYTE:
2416 // Generate a "4B" vector, to support vector cast between "8B/16B"
2417 // and "4S/4I/4L/4F/4D".
2418 size = 4;
2419 break;
2420 case T_SHORT:
2421 // Generate a "2S" vector, to support vector cast between "4S/8S"
2422 // and "2I/2L/2F/2D".
2423 size = 2;
2424 break;
2425 default:
2426 // Limit the min vector length to 64-bit.
2427 size = 8 / type2aelembytes(bt);
2428 // The number of elements in a vector should be at least 2.
2429 size = MAX2(size, 2);
2430 }
2431
2432 int max_size = max_vector_size(bt);
2433 return MIN2(size, max_size);
2434 }
2435
2436 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2437 return Matcher::max_vector_size(bt);
2438 }
2439
2440 // Actual max scalable vector register length.
2441 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2442 return Matcher::max_vector_size(bt);
2443 }
2444
2445 // Vector ideal reg.
2446 uint Matcher::vector_ideal_reg(int len) {
2447 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2448 return Op_VecA;
2449 }
2450 switch(len) {
2451 // For 16-bit/32-bit mask vector, reuse VecD.
2452 case 2:
2453 case 4:
2454 case 8: return Op_VecD;
2455 case 16: return Op_VecX;
2456 }
2457 ShouldNotReachHere();
2458 return 0;
2459 }
2460
2461 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2462 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2463 switch (ideal_reg) {
2464 case Op_VecA: return new vecAOper();
2465 case Op_VecD: return new vecDOper();
2466 case Op_VecX: return new vecXOper();
2467 }
2468 ShouldNotReachHere();
2469 return nullptr;
2470 }
2471
2472 bool Matcher::is_reg2reg_move(MachNode* m) {
2473 return false;
2474 }
2475
2476 bool Matcher::is_generic_vector(MachOper* opnd) {
2477 return opnd->opcode() == VREG;
2478 }
2479
2480 // Return whether or not this register is ever used as an argument.
2481 // This function is used on startup to build the trampoline stubs in
2482 // generateOptoStub. Registers not mentioned will be killed by the VM
2483 // call in the trampoline, and arguments in those registers not be
2484 // available to the callee.
2485 bool Matcher::can_be_java_arg(int reg)
2486 {
2487 return
2488 reg == R0_num || reg == R0_H_num ||
2489 reg == R1_num || reg == R1_H_num ||
2490 reg == R2_num || reg == R2_H_num ||
2491 reg == R3_num || reg == R3_H_num ||
2492 reg == R4_num || reg == R4_H_num ||
2493 reg == R5_num || reg == R5_H_num ||
2494 reg == R6_num || reg == R6_H_num ||
2495 reg == R7_num || reg == R7_H_num ||
2496 reg == V0_num || reg == V0_H_num ||
2497 reg == V1_num || reg == V1_H_num ||
2498 reg == V2_num || reg == V2_H_num ||
2499 reg == V3_num || reg == V3_H_num ||
2500 reg == V4_num || reg == V4_H_num ||
2501 reg == V5_num || reg == V5_H_num ||
2502 reg == V6_num || reg == V6_H_num ||
2503 reg == V7_num || reg == V7_H_num;
2504 }
2505
2506 bool Matcher::is_spillable_arg(int reg)
2507 {
2508 return can_be_java_arg(reg);
2509 }
2510
2511 uint Matcher::int_pressure_limit()
2512 {
2513 // JDK-8183543: When taking the number of available registers as int
2514 // register pressure threshold, the jtreg test:
2515 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2516 // failed due to C2 compilation failure with
2517 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2518 //
2519 // A derived pointer is live at CallNode and then is flagged by RA
2520 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2521 // derived pointers and lastly fail to spill after reaching maximum
2522 // number of iterations. Lowering the default pressure threshold to
2523 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2524 // a high register pressure area of the code so that split_DEF can
2525 // generate DefinitionSpillCopy for the derived pointer.
2526 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2527 if (!PreserveFramePointer) {
2528 // When PreserveFramePointer is off, frame pointer is allocatable,
2529 // but different from other SOC registers, it is excluded from
2530 // fatproj's mask because its save type is No-Save. Decrease 1 to
2531 // ensure high pressure at fatproj when PreserveFramePointer is off.
2532 // See check_pressure_at_fatproj().
2533 default_int_pressure_threshold--;
2534 }
2535 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2536 }
2537
2538 uint Matcher::float_pressure_limit()
2539 {
2540 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2541 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2542 }
2543
2544 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2545 return false;
2546 }
2547
2548 RegMask Matcher::divI_proj_mask() {
2549 ShouldNotReachHere();
2550 return RegMask();
2551 }
2552
2553 // Register for MODI projection of divmodI.
2554 RegMask Matcher::modI_proj_mask() {
2555 ShouldNotReachHere();
2556 return RegMask();
2557 }
2558
2559 // Register for DIVL projection of divmodL.
2560 RegMask Matcher::divL_proj_mask() {
2561 ShouldNotReachHere();
2562 return RegMask();
2563 }
2564
2565 // Register for MODL projection of divmodL.
2566 RegMask Matcher::modL_proj_mask() {
2567 ShouldNotReachHere();
2568 return RegMask();
2569 }
2570
2571 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2572 return FP_REG_mask();
2573 }
2574
2575 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2576 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2577 Node* u = addp->fast_out(i);
2578 if (u->is_LoadStore()) {
2579 // On AArch64, LoadStoreNodes (i.e. compare and swap
2580 // instructions) only take register indirect as an operand, so
2581 // any attempt to use an AddPNode as an input to a LoadStoreNode
2582 // must fail.
2583 return false;
2584 }
2585 if (u->is_Mem()) {
2586 int opsize = u->as_Mem()->memory_size();
2587 assert(opsize > 0, "unexpected memory operand size");
2588 if (u->as_Mem()->memory_size() != (1<<shift)) {
2589 return false;
2590 }
2591 }
2592 }
2593 return true;
2594 }
2595
2596 // Convert BootTest condition to Assembler condition.
2597 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2598 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2599 Assembler::Condition result;
2600 switch(cond) {
2601 case BoolTest::eq:
2602 result = Assembler::EQ; break;
2603 case BoolTest::ne:
2604 result = Assembler::NE; break;
2605 case BoolTest::le:
2606 result = Assembler::LE; break;
2607 case BoolTest::ge:
2608 result = Assembler::GE; break;
2609 case BoolTest::lt:
2610 result = Assembler::LT; break;
2611 case BoolTest::gt:
2612 result = Assembler::GT; break;
2613 case BoolTest::ule:
2614 result = Assembler::LS; break;
2615 case BoolTest::uge:
2616 result = Assembler::HS; break;
2617 case BoolTest::ult:
2618 result = Assembler::LO; break;
2619 case BoolTest::ugt:
2620 result = Assembler::HI; break;
2621 case BoolTest::overflow:
2622 result = Assembler::VS; break;
2623 case BoolTest::no_overflow:
2624 result = Assembler::VC; break;
2625 default:
2626 ShouldNotReachHere();
2627 return Assembler::Condition(-1);
2628 }
2629
2630 // Check conversion
2631 if (cond & BoolTest::unsigned_compare) {
2632 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2633 } else {
2634 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2635 }
2636
2637 return result;
2638 }
2639
2640 // Binary src (Replicate con)
2641 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2642 if (n == nullptr || m == nullptr) {
2643 return false;
2644 }
2645
2646 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2647 return false;
2648 }
2649
2650 Node* imm_node = m->in(1);
2651 if (!imm_node->is_Con()) {
2652 return false;
2653 }
2654
2655 const Type* t = imm_node->bottom_type();
2656 if (!(t->isa_int() || t->isa_long())) {
2657 return false;
2658 }
2659
2660 switch (n->Opcode()) {
2661 case Op_AndV:
2662 case Op_OrV:
2663 case Op_XorV: {
2664 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2665 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2666 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2667 }
2668 case Op_AddVB:
2669 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2670 case Op_AddVS:
2671 case Op_AddVI:
2672 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2673 case Op_AddVL:
2674 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2675 default:
2676 return false;
2677 }
2678 }
2679
2680 // (XorV src (Replicate m1))
2681 // (XorVMask src (MaskAll m1))
2682 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2683 if (n != nullptr && m != nullptr) {
2684 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2685 VectorNode::is_all_ones_vector(m);
2686 }
2687 return false;
2688 }
2689
2690 // Should the matcher clone input 'm' of node 'n'?
2691 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2692 if (is_vshift_con_pattern(n, m) ||
2693 is_vector_bitwise_not_pattern(n, m) ||
2694 is_valid_sve_arith_imm_pattern(n, m) ||
2695 is_encode_and_store_pattern(n, m)) {
2696 mstack.push(m, Visit);
2697 return true;
2698 }
2699 return false;
2700 }
2701
2702 // Should the Matcher clone shifts on addressing modes, expecting them
2703 // to be subsumed into complex addressing expressions or compute them
2704 // into registers?
2705 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2706
2707 // Loads and stores with indirect memory input (e.g., volatile loads and
2708 // stores) do not subsume the input into complex addressing expressions. If
2709 // the addressing expression is input to at least one such load or store, do
2710 // not clone the addressing expression. Query needs_acquiring_load and
2711 // needs_releasing_store as a proxy for indirect memory input, as it is not
2712 // possible to directly query for indirect memory input at this stage.
2713 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2714 Node* n = m->fast_out(i);
2715 if (n->is_Load() && needs_acquiring_load(n)) {
2716 return false;
2717 }
2718 if (n->is_Store() && needs_releasing_store(n)) {
2719 return false;
2720 }
2721 }
2722
2723 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2724 return true;
2725 }
2726
2727 Node *off = m->in(AddPNode::Offset);
2728 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2729 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2730 // Are there other uses besides address expressions?
2731 !is_visited(off)) {
2732 address_visited.set(off->_idx); // Flag as address_visited
2733 mstack.push(off->in(2), Visit);
2734 Node *conv = off->in(1);
2735 if (conv->Opcode() == Op_ConvI2L &&
2736 // Are there other uses besides address expressions?
2737 !is_visited(conv)) {
2738 address_visited.set(conv->_idx); // Flag as address_visited
2739 mstack.push(conv->in(1), Pre_Visit);
2740 } else {
2741 mstack.push(conv, Pre_Visit);
2742 }
2743 address_visited.test_set(m->_idx); // Flag as address_visited
2744 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2745 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2746 return true;
2747 } else if (off->Opcode() == Op_ConvI2L &&
2748 // Are there other uses besides address expressions?
2749 !is_visited(off)) {
2750 address_visited.test_set(m->_idx); // Flag as address_visited
2751 address_visited.set(off->_idx); // Flag as address_visited
2752 mstack.push(off->in(1), Pre_Visit);
2753 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2754 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2755 return true;
2756 }
2757 return false;
2758 }
2759
2760 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2761 { \
2762 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2763 guarantee(DISP == 0, "mode not permitted for volatile"); \
2764 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2765 __ INSN(REG, as_Register(BASE)); \
2766 }
2767
2768
2769 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2770 {
2771 Address::extend scale;
2772
2773 // Hooboy, this is fugly. We need a way to communicate to the
2774 // encoder that the index needs to be sign extended, so we have to
2775 // enumerate all the cases.
2776 switch (opcode) {
2777 case INDINDEXSCALEDI2L:
2778 case INDINDEXSCALEDI2LN:
2779 case INDINDEXI2L:
2780 case INDINDEXI2LN:
2781 scale = Address::sxtw(size);
2782 break;
2783 default:
2784 scale = Address::lsl(size);
2785 }
2786
2787 if (index == -1) {
2788 return Address(base, disp);
2789 } else {
2790 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2791 return Address(base, as_Register(index), scale);
2792 }
2793 }
2794
2795
2796 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2797 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2798 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2799 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2800 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2801
2802 // Used for all non-volatile memory accesses. The use of
2803 // $mem->opcode() to discover whether this pattern uses sign-extended
2804 // offsets is something of a kludge.
2805 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2806 Register reg, int opcode,
2807 Register base, int index, int scale, int disp,
2808 int size_in_memory)
2809 {
2810 Address addr = mem2address(opcode, base, index, scale, disp);
2811 if (addr.getMode() == Address::base_plus_offset) {
2812 /* Fix up any out-of-range offsets. */
2813 assert_different_registers(rscratch1, base);
2814 assert_different_registers(rscratch1, reg);
2815 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2816 }
2817 (masm->*insn)(reg, addr);
2818 }
2819
2820 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2821 FloatRegister reg, int opcode,
2822 Register base, int index, int size, int disp,
2823 int size_in_memory)
2824 {
2825 Address::extend scale;
2826
2827 switch (opcode) {
2828 case INDINDEXSCALEDI2L:
2829 case INDINDEXSCALEDI2LN:
2830 scale = Address::sxtw(size);
2831 break;
2832 default:
2833 scale = Address::lsl(size);
2834 }
2835
2836 if (index == -1) {
2837 // Fix up any out-of-range offsets.
2838 assert_different_registers(rscratch1, base);
2839 Address addr = Address(base, disp);
2840 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2841 (masm->*insn)(reg, addr);
2842 } else {
2843 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2844 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2845 }
2846 }
2847
2848 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2849 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2850 int opcode, Register base, int index, int size, int disp)
2851 {
2852 if (index == -1) {
2853 (masm->*insn)(reg, T, Address(base, disp));
2854 } else {
2855 assert(disp == 0, "unsupported address mode");
2856 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2857 }
2858 }
2859
2860 %}
2861
2862
2863
2864 //----------ENCODING BLOCK-----------------------------------------------------
2865 // This block specifies the encoding classes used by the compiler to
2866 // output byte streams. Encoding classes are parameterized macros
2867 // used by Machine Instruction Nodes in order to generate the bit
2868 // encoding of the instruction. Operands specify their base encoding
2869 // interface with the interface keyword. There are currently
2870 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2871 // COND_INTER. REG_INTER causes an operand to generate a function
2872 // which returns its register number when queried. CONST_INTER causes
2873 // an operand to generate a function which returns the value of the
2874 // constant when queried. MEMORY_INTER causes an operand to generate
2875 // four functions which return the Base Register, the Index Register,
2876 // the Scale Value, and the Offset Value of the operand when queried.
2877 // COND_INTER causes an operand to generate six functions which return
2878 // the encoding code (ie - encoding bits for the instruction)
2879 // associated with each basic boolean condition for a conditional
2880 // instruction.
2881 //
2882 // Instructions specify two basic values for encoding. Again, a
2883 // function is available to check if the constant displacement is an
2884 // oop. They use the ins_encode keyword to specify their encoding
2885 // classes (which must be a sequence of enc_class names, and their
2886 // parameters, specified in the encoding block), and they use the
2887 // opcode keyword to specify, in order, their primary, secondary, and
2888 // tertiary opcode. Only the opcode sections which a particular
2889 // instruction needs for encoding need to be specified.
2890 encode %{
2891 // Build emit functions for each basic byte or larger field in the
2892 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2893 // from C++ code in the enc_class source block. Emit functions will
2894 // live in the main source block for now. In future, we can
2895 // generalize this by adding a syntax that specifies the sizes of
2896 // fields in an order, so that the adlc can build the emit functions
2897 // automagically
2898
2899 // catch all for unimplemented encodings
2900 enc_class enc_unimplemented %{
2901 __ unimplemented("C2 catch all");
2902 %}
2903
2904 // BEGIN Non-volatile memory access
2905
2906 // This encoding class is generated automatically from ad_encode.m4.
2907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2908 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2909 Register dst_reg = as_Register($dst$$reg);
2910 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2911 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2912 %}
2913
2914 // This encoding class is generated automatically from ad_encode.m4.
2915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2916 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2917 Register dst_reg = as_Register($dst$$reg);
2918 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2919 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2920 %}
2921
2922 // This encoding class is generated automatically from ad_encode.m4.
2923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2924 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2925 Register dst_reg = as_Register($dst$$reg);
2926 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2927 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2928 %}
2929
2930 // This encoding class is generated automatically from ad_encode.m4.
2931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2932 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2933 Register dst_reg = as_Register($dst$$reg);
2934 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2935 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2936 %}
2937
2938 // This encoding class is generated automatically from ad_encode.m4.
2939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2940 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2941 Register dst_reg = as_Register($dst$$reg);
2942 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2943 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2944 %}
2945
2946 // This encoding class is generated automatically from ad_encode.m4.
2947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2948 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2949 Register dst_reg = as_Register($dst$$reg);
2950 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2951 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2952 %}
2953
2954 // This encoding class is generated automatically from ad_encode.m4.
2955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2956 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2957 Register dst_reg = as_Register($dst$$reg);
2958 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2959 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2960 %}
2961
2962 // This encoding class is generated automatically from ad_encode.m4.
2963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2964 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2965 Register dst_reg = as_Register($dst$$reg);
2966 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2967 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2968 %}
2969
2970 // This encoding class is generated automatically from ad_encode.m4.
2971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2972 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2973 Register dst_reg = as_Register($dst$$reg);
2974 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2975 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2976 %}
2977
2978 // This encoding class is generated automatically from ad_encode.m4.
2979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2980 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2981 Register dst_reg = as_Register($dst$$reg);
2982 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2983 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2984 %}
2985
2986 // This encoding class is generated automatically from ad_encode.m4.
2987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2988 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2989 Register dst_reg = as_Register($dst$$reg);
2990 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2991 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2992 %}
2993
2994 // This encoding class is generated automatically from ad_encode.m4.
2995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2996 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2997 Register dst_reg = as_Register($dst$$reg);
2998 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2999 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3000 %}
3001
3002 // This encoding class is generated automatically from ad_encode.m4.
3003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3004 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
3005 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3006 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
3007 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3008 %}
3009
3010 // This encoding class is generated automatically from ad_encode.m4.
3011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3012 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
3013 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3014 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
3015 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3016 %}
3017
3018 // This encoding class is generated automatically from ad_encode.m4.
3019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3020 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3021 Register src_reg = as_Register($src$$reg);
3022 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3023 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3024 %}
3025
3026 // This encoding class is generated automatically from ad_encode.m4.
3027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3028 enc_class aarch64_enc_strb0(memory1 mem) %{
3029 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3030 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3031 %}
3032
3033 // This encoding class is generated automatically from ad_encode.m4.
3034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3035 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3036 Register src_reg = as_Register($src$$reg);
3037 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3038 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3039 %}
3040
3041 // This encoding class is generated automatically from ad_encode.m4.
3042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3043 enc_class aarch64_enc_strh0(memory2 mem) %{
3044 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3045 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3046 %}
3047
3048 // This encoding class is generated automatically from ad_encode.m4.
3049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3050 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3051 Register src_reg = as_Register($src$$reg);
3052 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3053 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3054 %}
3055
3056 // This encoding class is generated automatically from ad_encode.m4.
3057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3058 enc_class aarch64_enc_strw0(memory4 mem) %{
3059 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3060 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3061 %}
3062
3063 // This encoding class is generated automatically from ad_encode.m4.
3064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3065 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3066 Register src_reg = as_Register($src$$reg);
3067 // we sometimes get asked to store the stack pointer into the
3068 // current thread -- we cannot do that directly on AArch64
3069 if (src_reg == r31_sp) {
3070 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3071 __ mov(rscratch2, sp);
3072 src_reg = rscratch2;
3073 }
3074 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3075 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3076 %}
3077
3078 // This encoding class is generated automatically from ad_encode.m4.
3079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3080 enc_class aarch64_enc_str0(memory8 mem) %{
3081 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3082 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3083 %}
3084
3085 // This encoding class is generated automatically from ad_encode.m4.
3086 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3087 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3088 FloatRegister src_reg = as_FloatRegister($src$$reg);
3089 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3090 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3091 %}
3092
3093 // This encoding class is generated automatically from ad_encode.m4.
3094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3095 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3096 FloatRegister src_reg = as_FloatRegister($src$$reg);
3097 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3098 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3099 %}
3100
3101 // This encoding class is generated automatically from ad_encode.m4.
3102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3103 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3104 __ membar(Assembler::StoreStore);
3105 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3106 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3107 %}
3108
3109 // END Non-volatile memory access
3110
3111 // Vector loads and stores
3112 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3113 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3114 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3115 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3116 %}
3117
3118 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3119 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3120 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3121 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3122 %}
3123
3124 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3125 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3126 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3127 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3128 %}
3129
3130 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3131 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3132 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3133 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3134 %}
3135
3136 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3137 FloatRegister src_reg = as_FloatRegister($src$$reg);
3138 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3139 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3140 %}
3141
3142 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3143 FloatRegister src_reg = as_FloatRegister($src$$reg);
3144 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3145 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3146 %}
3147
3148 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3149 FloatRegister src_reg = as_FloatRegister($src$$reg);
3150 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3151 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3152 %}
3153
3154 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3155 FloatRegister src_reg = as_FloatRegister($src$$reg);
3156 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3157 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3158 %}
3159
3160 // volatile loads and stores
3161
3162 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3163 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3164 rscratch1, stlrb);
3165 %}
3166
3167 enc_class aarch64_enc_stlrb0(memory mem) %{
3168 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3169 rscratch1, stlrb);
3170 %}
3171
3172 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3173 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3174 rscratch1, stlrh);
3175 %}
3176
3177 enc_class aarch64_enc_stlrh0(memory mem) %{
3178 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3179 rscratch1, stlrh);
3180 %}
3181
3182 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3183 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3184 rscratch1, stlrw);
3185 %}
3186
3187 enc_class aarch64_enc_stlrw0(memory mem) %{
3188 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3189 rscratch1, stlrw);
3190 %}
3191
3192 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3193 Register dst_reg = as_Register($dst$$reg);
3194 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3195 rscratch1, ldarb);
3196 __ sxtbw(dst_reg, dst_reg);
3197 %}
3198
3199 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3200 Register dst_reg = as_Register($dst$$reg);
3201 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3202 rscratch1, ldarb);
3203 __ sxtb(dst_reg, dst_reg);
3204 %}
3205
3206 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3207 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3208 rscratch1, ldarb);
3209 %}
3210
3211 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3212 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3213 rscratch1, ldarb);
3214 %}
3215
3216 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3217 Register dst_reg = as_Register($dst$$reg);
3218 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3219 rscratch1, ldarh);
3220 __ sxthw(dst_reg, dst_reg);
3221 %}
3222
3223 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3224 Register dst_reg = as_Register($dst$$reg);
3225 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3226 rscratch1, ldarh);
3227 __ sxth(dst_reg, dst_reg);
3228 %}
3229
3230 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3231 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3232 rscratch1, ldarh);
3233 %}
3234
3235 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3236 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3237 rscratch1, ldarh);
3238 %}
3239
3240 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3241 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3242 rscratch1, ldarw);
3243 %}
3244
3245 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3246 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3247 rscratch1, ldarw);
3248 %}
3249
3250 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3251 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3252 rscratch1, ldar);
3253 %}
3254
3255 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3256 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3257 rscratch1, ldarw);
3258 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3259 %}
3260
3261 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3262 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3263 rscratch1, ldar);
3264 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3265 %}
3266
3267 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3268 Register src_reg = as_Register($src$$reg);
3269 // we sometimes get asked to store the stack pointer into the
3270 // current thread -- we cannot do that directly on AArch64
3271 if (src_reg == r31_sp) {
3272 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3273 __ mov(rscratch2, sp);
3274 src_reg = rscratch2;
3275 }
3276 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3277 rscratch1, stlr);
3278 %}
3279
3280 enc_class aarch64_enc_stlr0(memory mem) %{
3281 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3282 rscratch1, stlr);
3283 %}
3284
3285 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3286 {
3287 FloatRegister src_reg = as_FloatRegister($src$$reg);
3288 __ fmovs(rscratch2, src_reg);
3289 }
3290 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3291 rscratch1, stlrw);
3292 %}
3293
3294 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3295 {
3296 FloatRegister src_reg = as_FloatRegister($src$$reg);
3297 __ fmovd(rscratch2, src_reg);
3298 }
3299 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3300 rscratch1, stlr);
3301 %}
3302
3303 // synchronized read/update encodings
3304
3305 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3306 Register dst_reg = as_Register($dst$$reg);
3307 Register base = as_Register($mem$$base);
3308 int index = $mem$$index;
3309 int scale = $mem$$scale;
3310 int disp = $mem$$disp;
3311 if (index == -1) {
3312 if (disp != 0) {
3313 __ lea(rscratch1, Address(base, disp));
3314 __ ldaxr(dst_reg, rscratch1);
3315 } else {
3316 // TODO
3317 // should we ever get anything other than this case?
3318 __ ldaxr(dst_reg, base);
3319 }
3320 } else {
3321 Register index_reg = as_Register(index);
3322 if (disp == 0) {
3323 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3324 __ ldaxr(dst_reg, rscratch1);
3325 } else {
3326 __ lea(rscratch1, Address(base, disp));
3327 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3328 __ ldaxr(dst_reg, rscratch1);
3329 }
3330 }
3331 %}
3332
3333 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3334 Register src_reg = as_Register($src$$reg);
3335 Register base = as_Register($mem$$base);
3336 int index = $mem$$index;
3337 int scale = $mem$$scale;
3338 int disp = $mem$$disp;
3339 if (index == -1) {
3340 if (disp != 0) {
3341 __ lea(rscratch2, Address(base, disp));
3342 __ stlxr(rscratch1, src_reg, rscratch2);
3343 } else {
3344 // TODO
3345 // should we ever get anything other than this case?
3346 __ stlxr(rscratch1, src_reg, base);
3347 }
3348 } else {
3349 Register index_reg = as_Register(index);
3350 if (disp == 0) {
3351 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3352 __ stlxr(rscratch1, src_reg, rscratch2);
3353 } else {
3354 __ lea(rscratch2, Address(base, disp));
3355 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3356 __ stlxr(rscratch1, src_reg, rscratch2);
3357 }
3358 }
3359 __ cmpw(rscratch1, zr);
3360 %}
3361
3362 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3363 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3364 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3365 Assembler::xword, /*acquire*/ false, /*release*/ true,
3366 /*weak*/ false, noreg);
3367 %}
3368
3369 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3370 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3371 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3372 Assembler::word, /*acquire*/ false, /*release*/ true,
3373 /*weak*/ false, noreg);
3374 %}
3375
3376 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3377 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3378 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3379 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3380 /*weak*/ false, noreg);
3381 %}
3382
3383 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3384 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3385 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3386 Assembler::byte, /*acquire*/ false, /*release*/ true,
3387 /*weak*/ false, noreg);
3388 %}
3389
3390
3391 // The only difference between aarch64_enc_cmpxchg and
3392 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3393 // CompareAndSwap sequence to serve as a barrier on acquiring a
3394 // lock.
3395 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3396 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3397 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3398 Assembler::xword, /*acquire*/ true, /*release*/ true,
3399 /*weak*/ false, noreg);
3400 %}
3401
3402 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3403 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3404 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3405 Assembler::word, /*acquire*/ true, /*release*/ true,
3406 /*weak*/ false, noreg);
3407 %}
3408
3409 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3410 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3411 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3412 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3413 /*weak*/ false, noreg);
3414 %}
3415
3416 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3417 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3418 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3419 Assembler::byte, /*acquire*/ true, /*release*/ true,
3420 /*weak*/ false, noreg);
3421 %}
3422
3423 // auxiliary used for CompareAndSwapX to set result register
3424 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3425 Register res_reg = as_Register($res$$reg);
3426 __ cset(res_reg, Assembler::EQ);
3427 %}
3428
3429 // prefetch encodings
3430
3431 enc_class aarch64_enc_prefetchw(memory mem) %{
3432 Register base = as_Register($mem$$base);
3433 int index = $mem$$index;
3434 int scale = $mem$$scale;
3435 int disp = $mem$$disp;
3436 if (index == -1) {
3437 // Fix up any out-of-range offsets.
3438 assert_different_registers(rscratch1, base);
3439 Address addr = Address(base, disp);
3440 addr = __ legitimize_address(addr, 8, rscratch1);
3441 __ prfm(addr, PSTL1KEEP);
3442 } else {
3443 Register index_reg = as_Register(index);
3444 if (disp == 0) {
3445 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3446 } else {
3447 __ lea(rscratch1, Address(base, disp));
3448 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3449 }
3450 }
3451 %}
3452
3453 // mov encodings
3454
3455 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3456 uint32_t con = (uint32_t)$src$$constant;
3457 Register dst_reg = as_Register($dst$$reg);
3458 if (con == 0) {
3459 __ movw(dst_reg, zr);
3460 } else {
3461 __ movw(dst_reg, con);
3462 }
3463 %}
3464
3465 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3466 Register dst_reg = as_Register($dst$$reg);
3467 uint64_t con = (uint64_t)$src$$constant;
3468 if (con == 0) {
3469 __ mov(dst_reg, zr);
3470 } else {
3471 __ mov(dst_reg, con);
3472 }
3473 %}
3474
3475 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3476 Register dst_reg = as_Register($dst$$reg);
3477 address con = (address)$src$$constant;
3478 if (con == nullptr || con == (address)1) {
3479 ShouldNotReachHere();
3480 } else {
3481 relocInfo::relocType rtype = $src->constant_reloc();
3482 if (rtype == relocInfo::oop_type) {
3483 __ movoop(dst_reg, (jobject)con);
3484 } else if (rtype == relocInfo::metadata_type) {
3485 __ mov_metadata(dst_reg, (Metadata*)con);
3486 } else {
3487 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3488 if (! __ is_valid_AArch64_address(con) ||
3489 con < (address)(uintptr_t)os::vm_page_size()) {
3490 __ mov(dst_reg, con);
3491 } else {
3492 uint64_t offset;
3493 __ adrp(dst_reg, con, offset);
3494 __ add(dst_reg, dst_reg, offset);
3495 }
3496 }
3497 }
3498 %}
3499
3500 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3501 Register dst_reg = as_Register($dst$$reg);
3502 __ mov(dst_reg, zr);
3503 %}
3504
3505 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3506 Register dst_reg = as_Register($dst$$reg);
3507 __ mov(dst_reg, (uint64_t)1);
3508 %}
3509
3510 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3511 Register dst_reg = as_Register($dst$$reg);
3512 address con = (address)$src$$constant;
3513 if (con == nullptr) {
3514 ShouldNotReachHere();
3515 } else {
3516 relocInfo::relocType rtype = $src->constant_reloc();
3517 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3518 __ set_narrow_oop(dst_reg, (jobject)con);
3519 }
3520 %}
3521
3522 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3523 Register dst_reg = as_Register($dst$$reg);
3524 __ mov(dst_reg, zr);
3525 %}
3526
3527 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3528 Register dst_reg = as_Register($dst$$reg);
3529 address con = (address)$src$$constant;
3530 if (con == nullptr) {
3531 ShouldNotReachHere();
3532 } else {
3533 relocInfo::relocType rtype = $src->constant_reloc();
3534 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3535 __ set_narrow_klass(dst_reg, (Klass *)con);
3536 }
3537 %}
3538
3539 // arithmetic encodings
3540
3541 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3542 Register dst_reg = as_Register($dst$$reg);
3543 Register src_reg = as_Register($src1$$reg);
3544 int32_t con = (int32_t)$src2$$constant;
3545 // add has primary == 0, subtract has primary == 1
3546 if ($primary) { con = -con; }
3547 if (con < 0) {
3548 __ subw(dst_reg, src_reg, -con);
3549 } else {
3550 __ addw(dst_reg, src_reg, con);
3551 }
3552 %}
3553
3554 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3555 Register dst_reg = as_Register($dst$$reg);
3556 Register src_reg = as_Register($src1$$reg);
3557 int32_t con = (int32_t)$src2$$constant;
3558 // add has primary == 0, subtract has primary == 1
3559 if ($primary) { con = -con; }
3560 if (con < 0) {
3561 __ sub(dst_reg, src_reg, -con);
3562 } else {
3563 __ add(dst_reg, src_reg, con);
3564 }
3565 %}
3566
3567 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3568 Register dst_reg = as_Register($dst$$reg);
3569 Register src1_reg = as_Register($src1$$reg);
3570 Register src2_reg = as_Register($src2$$reg);
3571 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3572 %}
3573
3574 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3575 Register dst_reg = as_Register($dst$$reg);
3576 Register src1_reg = as_Register($src1$$reg);
3577 Register src2_reg = as_Register($src2$$reg);
3578 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3579 %}
3580
3581 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3582 Register dst_reg = as_Register($dst$$reg);
3583 Register src1_reg = as_Register($src1$$reg);
3584 Register src2_reg = as_Register($src2$$reg);
3585 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3586 %}
3587
3588 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3589 Register dst_reg = as_Register($dst$$reg);
3590 Register src1_reg = as_Register($src1$$reg);
3591 Register src2_reg = as_Register($src2$$reg);
3592 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3593 %}
3594
3595 // compare instruction encodings
3596
3597 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3598 Register reg1 = as_Register($src1$$reg);
3599 Register reg2 = as_Register($src2$$reg);
3600 __ cmpw(reg1, reg2);
3601 %}
3602
3603 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3604 Register reg = as_Register($src1$$reg);
3605 int32_t val = $src2$$constant;
3606 if (val >= 0) {
3607 __ subsw(zr, reg, val);
3608 } else {
3609 __ addsw(zr, reg, -val);
3610 }
3611 %}
3612
3613 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3614 Register reg1 = as_Register($src1$$reg);
3615 uint32_t val = (uint32_t)$src2$$constant;
3616 __ movw(rscratch1, val);
3617 __ cmpw(reg1, rscratch1);
3618 %}
3619
3620 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3621 Register reg1 = as_Register($src1$$reg);
3622 Register reg2 = as_Register($src2$$reg);
3623 __ cmp(reg1, reg2);
3624 %}
3625
3626 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3627 Register reg = as_Register($src1$$reg);
3628 int64_t val = $src2$$constant;
3629 if (val >= 0) {
3630 __ subs(zr, reg, val);
3631 } else if (val != -val) {
3632 __ adds(zr, reg, -val);
3633 } else {
3634 // aargh, Long.MIN_VALUE is a special case
3635 __ orr(rscratch1, zr, (uint64_t)val);
3636 __ subs(zr, reg, rscratch1);
3637 }
3638 %}
3639
3640 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3641 Register reg1 = as_Register($src1$$reg);
3642 uint64_t val = (uint64_t)$src2$$constant;
3643 __ mov(rscratch1, val);
3644 __ cmp(reg1, rscratch1);
3645 %}
3646
3647 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3648 Register reg1 = as_Register($src1$$reg);
3649 Register reg2 = as_Register($src2$$reg);
3650 __ cmp(reg1, reg2);
3651 %}
3652
3653 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3654 Register reg1 = as_Register($src1$$reg);
3655 Register reg2 = as_Register($src2$$reg);
3656 __ cmpw(reg1, reg2);
3657 %}
3658
3659 enc_class aarch64_enc_testp(iRegP src) %{
3660 Register reg = as_Register($src$$reg);
3661 __ cmp(reg, zr);
3662 %}
3663
3664 enc_class aarch64_enc_testn(iRegN src) %{
3665 Register reg = as_Register($src$$reg);
3666 __ cmpw(reg, zr);
3667 %}
3668
3669 enc_class aarch64_enc_b(label lbl) %{
3670 Label *L = $lbl$$label;
3671 __ b(*L);
3672 %}
3673
3674 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3675 Label *L = $lbl$$label;
3676 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3677 %}
3678
3679 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3680 Label *L = $lbl$$label;
3681 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3682 %}
3683
3684 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3685 %{
3686 Register sub_reg = as_Register($sub$$reg);
3687 Register super_reg = as_Register($super$$reg);
3688 Register temp_reg = as_Register($temp$$reg);
3689 Register result_reg = as_Register($result$$reg);
3690
3691 Label miss;
3692 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3693 nullptr, &miss,
3694 /*set_cond_codes:*/ true);
3695 if ($primary) {
3696 __ mov(result_reg, zr);
3697 }
3698 __ bind(miss);
3699 %}
3700
3701 enc_class aarch64_enc_java_static_call(method meth) %{
3702 address addr = (address)$meth$$method;
3703 address call;
3704 if (!_method) {
3705 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3706 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3707 if (call == nullptr) {
3708 ciEnv::current()->record_failure("CodeCache is full");
3709 return;
3710 }
3711 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3712 // The NOP here is purely to ensure that eliding a call to
3713 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3714 __ nop();
3715 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3716 } else {
3717 int method_index = resolved_method_index(masm);
3718 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3719 : static_call_Relocation::spec(method_index);
3720 call = __ trampoline_call(Address(addr, rspec));
3721 if (call == nullptr) {
3722 ciEnv::current()->record_failure("CodeCache is full");
3723 return;
3724 }
3725 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3726 // Calls of the same statically bound method can share
3727 // a stub to the interpreter.
3728 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3729 } else {
3730 // Emit stub for static call
3731 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3732 if (stub == nullptr) {
3733 ciEnv::current()->record_failure("CodeCache is full");
3734 return;
3735 }
3736 }
3737 }
3738
3739 __ post_call_nop();
3740
3741 // Only non uncommon_trap calls need to reinitialize ptrue.
3742 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3743 __ reinitialize_ptrue();
3744 }
3745 %}
3746
3747 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3748 int method_index = resolved_method_index(masm);
3749 address call = __ ic_call((address)$meth$$method, method_index);
3750 if (call == nullptr) {
3751 ciEnv::current()->record_failure("CodeCache is full");
3752 return;
3753 }
3754 __ post_call_nop();
3755 if (Compile::current()->max_vector_size() > 0) {
3756 __ reinitialize_ptrue();
3757 }
3758 %}
3759
3760 enc_class aarch64_enc_call_epilog() %{
3761 if (VerifyStackAtCalls) {
3762 // Check that stack depth is unchanged: find majik cookie on stack
3763 __ call_Unimplemented();
3764 }
3765 %}
3766
3767 enc_class aarch64_enc_java_to_runtime(method meth) %{
3768 // some calls to generated routines (arraycopy code) are scheduled
3769 // by C2 as runtime calls. if so we can call them using a br (they
3770 // will be in a reachable segment) otherwise we have to use a blr
3771 // which loads the absolute address into a register.
3772 address entry = (address)$meth$$method;
3773 CodeBlob *cb = CodeCache::find_blob(entry);
3774 if (cb) {
3775 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3776 if (call == nullptr) {
3777 ciEnv::current()->record_failure("CodeCache is full");
3778 return;
3779 }
3780 __ post_call_nop();
3781 } else {
3782 Label retaddr;
3783 // Make the anchor frame walkable
3784 __ adr(rscratch2, retaddr);
3785 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3786 __ lea(rscratch1, RuntimeAddress(entry));
3787 __ blr(rscratch1);
3788 __ bind(retaddr);
3789 __ post_call_nop();
3790 }
3791 if (Compile::current()->max_vector_size() > 0) {
3792 __ reinitialize_ptrue();
3793 }
3794 %}
3795
3796 enc_class aarch64_enc_rethrow() %{
3797 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3798 %}
3799
3800 enc_class aarch64_enc_ret() %{
3801 #ifdef ASSERT
3802 if (Compile::current()->max_vector_size() > 0) {
3803 __ verify_ptrue();
3804 }
3805 #endif
3806 __ ret(lr);
3807 %}
3808
3809 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3810 Register target_reg = as_Register($jump_target$$reg);
3811 __ br(target_reg);
3812 %}
3813
3814 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3815 Register target_reg = as_Register($jump_target$$reg);
3816 // exception oop should be in r0
3817 // ret addr has been popped into lr
3818 // callee expects it in r3
3819 __ mov(r3, lr);
3820 __ br(target_reg);
3821 %}
3822
3823 %}
3824
3825 //----------FRAME--------------------------------------------------------------
3826 // Definition of frame structure and management information.
3827 //
3828 // S T A C K L A Y O U T Allocators stack-slot number
3829 // | (to get allocators register number
3830 // G Owned by | | v add OptoReg::stack0())
3831 // r CALLER | |
3832 // o | +--------+ pad to even-align allocators stack-slot
3833 // w V | pad0 | numbers; owned by CALLER
3834 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3835 // h ^ | in | 5
3836 // | | args | 4 Holes in incoming args owned by SELF
3837 // | | | | 3
3838 // | | +--------+
3839 // V | | old out| Empty on Intel, window on Sparc
3840 // | old |preserve| Must be even aligned.
3841 // | SP-+--------+----> Matcher::_old_SP, even aligned
3842 // | | in | 3 area for Intel ret address
3843 // Owned by |preserve| Empty on Sparc.
3844 // SELF +--------+
3845 // | | pad2 | 2 pad to align old SP
3846 // | +--------+ 1
3847 // | | locks | 0
3848 // | +--------+----> OptoReg::stack0(), even aligned
3849 // | | pad1 | 11 pad to align new SP
3850 // | +--------+
3851 // | | | 10
3852 // | | spills | 9 spills
3853 // V | | 8 (pad0 slot for callee)
3854 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3855 // ^ | out | 7
3856 // | | args | 6 Holes in outgoing args owned by CALLEE
3857 // Owned by +--------+
3858 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3859 // | new |preserve| Must be even-aligned.
3860 // | SP-+--------+----> Matcher::_new_SP, even aligned
3861 // | | |
3862 //
3863 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3864 // known from SELF's arguments and the Java calling convention.
3865 // Region 6-7 is determined per call site.
3866 // Note 2: If the calling convention leaves holes in the incoming argument
3867 // area, those holes are owned by SELF. Holes in the outgoing area
3868 // are owned by the CALLEE. Holes should not be necessary in the
3869 // incoming area, as the Java calling convention is completely under
3870 // the control of the AD file. Doubles can be sorted and packed to
3871 // avoid holes. Holes in the outgoing arguments may be necessary for
3872 // varargs C calling conventions.
3873 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3874 // even aligned with pad0 as needed.
3875 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3876 // (the latter is true on Intel but is it false on AArch64?)
3877 // region 6-11 is even aligned; it may be padded out more so that
3878 // the region from SP to FP meets the minimum stack alignment.
3879 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3880 // alignment. Region 11, pad1, may be dynamically extended so that
3881 // SP meets the minimum alignment.
3882
3883 frame %{
3884 // These three registers define part of the calling convention
3885 // between compiled code and the interpreter.
3886
3887 // Inline Cache Register or Method for I2C.
3888 inline_cache_reg(R12);
3889
3890 // Number of stack slots consumed by locking an object
3891 sync_stack_slots(2);
3892
3893 // Compiled code's Frame Pointer
3894 frame_pointer(R31);
3895
3896 // Interpreter stores its frame pointer in a register which is
3897 // stored to the stack by I2CAdaptors.
3898 // I2CAdaptors convert from interpreted java to compiled java.
3899 interpreter_frame_pointer(R29);
3900
3901 // Stack alignment requirement
3902 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3903
3904 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3905 // for calls to C. Supports the var-args backing area for register parms.
3906 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3907
3908 // The after-PROLOG location of the return address. Location of
3909 // return address specifies a type (REG or STACK) and a number
3910 // representing the register number (i.e. - use a register name) or
3911 // stack slot.
3912 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3913 // Otherwise, it is above the locks and verification slot and alignment word
3914 // TODO this may well be correct but need to check why that - 2 is there
3915 // ppc port uses 0 but we definitely need to allow for fixed_slots
3916 // which folds in the space used for monitors
3917 return_addr(STACK - 2 +
3918 align_up((Compile::current()->in_preserve_stack_slots() +
3919 Compile::current()->fixed_slots()),
3920 stack_alignment_in_slots()));
3921
3922 // Location of compiled Java return values. Same as C for now.
3923 return_value
3924 %{
3925 // TODO do we allow ideal_reg == Op_RegN???
3926 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3927 "only return normal values");
3928
3929 static const int lo[Op_RegL + 1] = { // enum name
3930 0, // Op_Node
3931 0, // Op_Set
3932 R0_num, // Op_RegN
3933 R0_num, // Op_RegI
3934 R0_num, // Op_RegP
3935 V0_num, // Op_RegF
3936 V0_num, // Op_RegD
3937 R0_num // Op_RegL
3938 };
3939
3940 static const int hi[Op_RegL + 1] = { // enum name
3941 0, // Op_Node
3942 0, // Op_Set
3943 OptoReg::Bad, // Op_RegN
3944 OptoReg::Bad, // Op_RegI
3945 R0_H_num, // Op_RegP
3946 OptoReg::Bad, // Op_RegF
3947 V0_H_num, // Op_RegD
3948 R0_H_num // Op_RegL
3949 };
3950
3951 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3952 %}
3953 %}
3954
3955 //----------ATTRIBUTES---------------------------------------------------------
3956 //----------Operand Attributes-------------------------------------------------
3957 op_attrib op_cost(1); // Required cost attribute
3958
3959 //----------Instruction Attributes---------------------------------------------
3960 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3961 ins_attrib ins_size(32); // Required size attribute (in bits)
3962 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3963 // a non-matching short branch variant
3964 // of some long branch?
3965 ins_attrib ins_alignment(4); // Required alignment attribute (must
3966 // be a power of 2) specifies the
3967 // alignment that some part of the
3968 // instruction (not necessarily the
3969 // start) requires. If > 1, a
3970 // compute_padding() function must be
3971 // provided for the instruction
3972
3973 // Whether this node is expanded during code emission into a sequence of
3974 // instructions and the first instruction can perform an implicit null check.
3975 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3976
3977 //----------OPERANDS-----------------------------------------------------------
3978 // Operand definitions must precede instruction definitions for correct parsing
3979 // in the ADLC because operands constitute user defined types which are used in
3980 // instruction definitions.
3981
3982 //----------Simple Operands----------------------------------------------------
3983
3984 // Integer operands 32 bit
3985 // 32 bit immediate
3986 operand immI()
3987 %{
3988 match(ConI);
3989
3990 op_cost(0);
3991 format %{ %}
3992 interface(CONST_INTER);
3993 %}
3994
3995 // 32 bit zero
3996 operand immI0()
3997 %{
3998 predicate(n->get_int() == 0);
3999 match(ConI);
4000
4001 op_cost(0);
4002 format %{ %}
4003 interface(CONST_INTER);
4004 %}
4005
4006 // 32 bit unit increment
4007 operand immI_1()
4008 %{
4009 predicate(n->get_int() == 1);
4010 match(ConI);
4011
4012 op_cost(0);
4013 format %{ %}
4014 interface(CONST_INTER);
4015 %}
4016
4017 // 32 bit unit decrement
4018 operand immI_M1()
4019 %{
4020 predicate(n->get_int() == -1);
4021 match(ConI);
4022
4023 op_cost(0);
4024 format %{ %}
4025 interface(CONST_INTER);
4026 %}
4027
4028 // Shift values for add/sub extension shift
4029 operand immIExt()
4030 %{
4031 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4032 match(ConI);
4033
4034 op_cost(0);
4035 format %{ %}
4036 interface(CONST_INTER);
4037 %}
4038
4039 operand immI_gt_1()
4040 %{
4041 predicate(n->get_int() > 1);
4042 match(ConI);
4043
4044 op_cost(0);
4045 format %{ %}
4046 interface(CONST_INTER);
4047 %}
4048
4049 operand immI_le_4()
4050 %{
4051 predicate(n->get_int() <= 4);
4052 match(ConI);
4053
4054 op_cost(0);
4055 format %{ %}
4056 interface(CONST_INTER);
4057 %}
4058
4059 operand immI_16()
4060 %{
4061 predicate(n->get_int() == 16);
4062 match(ConI);
4063
4064 op_cost(0);
4065 format %{ %}
4066 interface(CONST_INTER);
4067 %}
4068
4069 operand immI_24()
4070 %{
4071 predicate(n->get_int() == 24);
4072 match(ConI);
4073
4074 op_cost(0);
4075 format %{ %}
4076 interface(CONST_INTER);
4077 %}
4078
4079 operand immI_32()
4080 %{
4081 predicate(n->get_int() == 32);
4082 match(ConI);
4083
4084 op_cost(0);
4085 format %{ %}
4086 interface(CONST_INTER);
4087 %}
4088
4089 operand immI_48()
4090 %{
4091 predicate(n->get_int() == 48);
4092 match(ConI);
4093
4094 op_cost(0);
4095 format %{ %}
4096 interface(CONST_INTER);
4097 %}
4098
4099 operand immI_56()
4100 %{
4101 predicate(n->get_int() == 56);
4102 match(ConI);
4103
4104 op_cost(0);
4105 format %{ %}
4106 interface(CONST_INTER);
4107 %}
4108
4109 operand immI_255()
4110 %{
4111 predicate(n->get_int() == 255);
4112 match(ConI);
4113
4114 op_cost(0);
4115 format %{ %}
4116 interface(CONST_INTER);
4117 %}
4118
4119 operand immI_65535()
4120 %{
4121 predicate(n->get_int() == 65535);
4122 match(ConI);
4123
4124 op_cost(0);
4125 format %{ %}
4126 interface(CONST_INTER);
4127 %}
4128
4129 operand immI_positive()
4130 %{
4131 predicate(n->get_int() > 0);
4132 match(ConI);
4133
4134 op_cost(0);
4135 format %{ %}
4136 interface(CONST_INTER);
4137 %}
4138
4139 // BoolTest condition for signed compare
4140 operand immI_cmp_cond()
4141 %{
4142 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4143 match(ConI);
4144
4145 op_cost(0);
4146 format %{ %}
4147 interface(CONST_INTER);
4148 %}
4149
4150 // BoolTest condition for unsigned compare
4151 operand immI_cmpU_cond()
4152 %{
4153 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4154 match(ConI);
4155
4156 op_cost(0);
4157 format %{ %}
4158 interface(CONST_INTER);
4159 %}
4160
4161 operand immL_255()
4162 %{
4163 predicate(n->get_long() == 255L);
4164 match(ConL);
4165
4166 op_cost(0);
4167 format %{ %}
4168 interface(CONST_INTER);
4169 %}
4170
4171 operand immL_65535()
4172 %{
4173 predicate(n->get_long() == 65535L);
4174 match(ConL);
4175
4176 op_cost(0);
4177 format %{ %}
4178 interface(CONST_INTER);
4179 %}
4180
4181 operand immL_4294967295()
4182 %{
4183 predicate(n->get_long() == 4294967295L);
4184 match(ConL);
4185
4186 op_cost(0);
4187 format %{ %}
4188 interface(CONST_INTER);
4189 %}
4190
4191 operand immL_bitmask()
4192 %{
4193 predicate((n->get_long() != 0)
4194 && ((n->get_long() & 0xc000000000000000l) == 0)
4195 && is_power_of_2(n->get_long() + 1));
4196 match(ConL);
4197
4198 op_cost(0);
4199 format %{ %}
4200 interface(CONST_INTER);
4201 %}
4202
4203 operand immI_bitmask()
4204 %{
4205 predicate((n->get_int() != 0)
4206 && ((n->get_int() & 0xc0000000) == 0)
4207 && is_power_of_2(n->get_int() + 1));
4208 match(ConI);
4209
4210 op_cost(0);
4211 format %{ %}
4212 interface(CONST_INTER);
4213 %}
4214
4215 operand immL_positive_bitmaskI()
4216 %{
4217 predicate((n->get_long() != 0)
4218 && ((julong)n->get_long() < 0x80000000ULL)
4219 && is_power_of_2(n->get_long() + 1));
4220 match(ConL);
4221
4222 op_cost(0);
4223 format %{ %}
4224 interface(CONST_INTER);
4225 %}
4226
4227 // Scale values for scaled offset addressing modes (up to long but not quad)
4228 operand immIScale()
4229 %{
4230 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4231 match(ConI);
4232
4233 op_cost(0);
4234 format %{ %}
4235 interface(CONST_INTER);
4236 %}
4237
4238 // 5 bit signed integer
4239 operand immI5()
4240 %{
4241 predicate(Assembler::is_simm(n->get_int(), 5));
4242 match(ConI);
4243
4244 op_cost(0);
4245 format %{ %}
4246 interface(CONST_INTER);
4247 %}
4248
4249 // 7 bit unsigned integer
4250 operand immIU7()
4251 %{
4252 predicate(Assembler::is_uimm(n->get_int(), 7));
4253 match(ConI);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 // Offset for scaled or unscaled immediate loads and stores
4261 operand immIOffset()
4262 %{
4263 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4264 match(ConI);
4265
4266 op_cost(0);
4267 format %{ %}
4268 interface(CONST_INTER);
4269 %}
4270
4271 operand immIOffset1()
4272 %{
4273 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4274 match(ConI);
4275
4276 op_cost(0);
4277 format %{ %}
4278 interface(CONST_INTER);
4279 %}
4280
4281 operand immIOffset2()
4282 %{
4283 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4284 match(ConI);
4285
4286 op_cost(0);
4287 format %{ %}
4288 interface(CONST_INTER);
4289 %}
4290
4291 operand immIOffset4()
4292 %{
4293 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4294 match(ConI);
4295
4296 op_cost(0);
4297 format %{ %}
4298 interface(CONST_INTER);
4299 %}
4300
4301 operand immIOffset8()
4302 %{
4303 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4304 match(ConI);
4305
4306 op_cost(0);
4307 format %{ %}
4308 interface(CONST_INTER);
4309 %}
4310
4311 operand immIOffset16()
4312 %{
4313 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4314 match(ConI);
4315
4316 op_cost(0);
4317 format %{ %}
4318 interface(CONST_INTER);
4319 %}
4320
4321 operand immLOffset()
4322 %{
4323 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4324 match(ConL);
4325
4326 op_cost(0);
4327 format %{ %}
4328 interface(CONST_INTER);
4329 %}
4330
4331 operand immLoffset1()
4332 %{
4333 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4334 match(ConL);
4335
4336 op_cost(0);
4337 format %{ %}
4338 interface(CONST_INTER);
4339 %}
4340
4341 operand immLoffset2()
4342 %{
4343 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4344 match(ConL);
4345
4346 op_cost(0);
4347 format %{ %}
4348 interface(CONST_INTER);
4349 %}
4350
4351 operand immLoffset4()
4352 %{
4353 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4354 match(ConL);
4355
4356 op_cost(0);
4357 format %{ %}
4358 interface(CONST_INTER);
4359 %}
4360
4361 operand immLoffset8()
4362 %{
4363 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4364 match(ConL);
4365
4366 op_cost(0);
4367 format %{ %}
4368 interface(CONST_INTER);
4369 %}
4370
4371 operand immLoffset16()
4372 %{
4373 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4374 match(ConL);
4375
4376 op_cost(0);
4377 format %{ %}
4378 interface(CONST_INTER);
4379 %}
4380
4381 // 5 bit signed long integer
4382 operand immL5()
4383 %{
4384 predicate(Assembler::is_simm(n->get_long(), 5));
4385 match(ConL);
4386
4387 op_cost(0);
4388 format %{ %}
4389 interface(CONST_INTER);
4390 %}
4391
4392 // 7 bit unsigned long integer
4393 operand immLU7()
4394 %{
4395 predicate(Assembler::is_uimm(n->get_long(), 7));
4396 match(ConL);
4397
4398 op_cost(0);
4399 format %{ %}
4400 interface(CONST_INTER);
4401 %}
4402
4403 // 8 bit signed value.
4404 operand immI8()
4405 %{
4406 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4407 match(ConI);
4408
4409 op_cost(0);
4410 format %{ %}
4411 interface(CONST_INTER);
4412 %}
4413
4414 // 8 bit signed value (simm8), or #simm8 LSL 8.
4415 operand immI8_shift8()
4416 %{
4417 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4418 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4419 match(ConI);
4420
4421 op_cost(0);
4422 format %{ %}
4423 interface(CONST_INTER);
4424 %}
4425
4426 // 8 bit signed value (simm8), or #simm8 LSL 8.
4427 operand immL8_shift8()
4428 %{
4429 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4430 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4431 match(ConL);
4432
4433 op_cost(0);
4434 format %{ %}
4435 interface(CONST_INTER);
4436 %}
4437
4438 // 8 bit integer valid for vector add sub immediate
4439 operand immBAddSubV()
4440 %{
4441 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4442 match(ConI);
4443
4444 op_cost(0);
4445 format %{ %}
4446 interface(CONST_INTER);
4447 %}
4448
4449 // 32 bit integer valid for add sub immediate
4450 operand immIAddSub()
4451 %{
4452 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4453 match(ConI);
4454 op_cost(0);
4455 format %{ %}
4456 interface(CONST_INTER);
4457 %}
4458
4459 // 32 bit integer valid for vector add sub immediate
4460 operand immIAddSubV()
4461 %{
4462 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4463 match(ConI);
4464
4465 op_cost(0);
4466 format %{ %}
4467 interface(CONST_INTER);
4468 %}
4469
4470 // 32 bit unsigned integer valid for logical immediate
4471
4472 operand immBLog()
4473 %{
4474 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4475 match(ConI);
4476
4477 op_cost(0);
4478 format %{ %}
4479 interface(CONST_INTER);
4480 %}
4481
4482 operand immSLog()
4483 %{
4484 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4485 match(ConI);
4486
4487 op_cost(0);
4488 format %{ %}
4489 interface(CONST_INTER);
4490 %}
4491
4492 operand immILog()
4493 %{
4494 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4495 match(ConI);
4496
4497 op_cost(0);
4498 format %{ %}
4499 interface(CONST_INTER);
4500 %}
4501
4502 // Integer operands 64 bit
4503 // 64 bit immediate
4504 operand immL()
4505 %{
4506 match(ConL);
4507
4508 op_cost(0);
4509 format %{ %}
4510 interface(CONST_INTER);
4511 %}
4512
4513 // 64 bit zero
4514 operand immL0()
4515 %{
4516 predicate(n->get_long() == 0);
4517 match(ConL);
4518
4519 op_cost(0);
4520 format %{ %}
4521 interface(CONST_INTER);
4522 %}
4523
4524 // 64 bit unit decrement
4525 operand immL_M1()
4526 %{
4527 predicate(n->get_long() == -1);
4528 match(ConL);
4529
4530 op_cost(0);
4531 format %{ %}
4532 interface(CONST_INTER);
4533 %}
4534
4535 // 64 bit integer valid for add sub immediate
4536 operand immLAddSub()
4537 %{
4538 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4539 match(ConL);
4540 op_cost(0);
4541 format %{ %}
4542 interface(CONST_INTER);
4543 %}
4544
4545 // 64 bit integer valid for addv subv immediate
4546 operand immLAddSubV()
4547 %{
4548 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4549 match(ConL);
4550
4551 op_cost(0);
4552 format %{ %}
4553 interface(CONST_INTER);
4554 %}
4555
4556 // 64 bit integer valid for logical immediate
4557 operand immLLog()
4558 %{
4559 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4560 match(ConL);
4561 op_cost(0);
4562 format %{ %}
4563 interface(CONST_INTER);
4564 %}
4565
4566 // Long Immediate: low 32-bit mask
4567 operand immL_32bits()
4568 %{
4569 predicate(n->get_long() == 0xFFFFFFFFL);
4570 match(ConL);
4571 op_cost(0);
4572 format %{ %}
4573 interface(CONST_INTER);
4574 %}
4575
4576 // Pointer operands
4577 // Pointer Immediate
4578 operand immP()
4579 %{
4580 match(ConP);
4581
4582 op_cost(0);
4583 format %{ %}
4584 interface(CONST_INTER);
4585 %}
4586
4587 // nullptr Pointer Immediate
4588 operand immP0()
4589 %{
4590 predicate(n->get_ptr() == 0);
4591 match(ConP);
4592
4593 op_cost(0);
4594 format %{ %}
4595 interface(CONST_INTER);
4596 %}
4597
4598 // Pointer Immediate One
4599 // this is used in object initialization (initial object header)
4600 operand immP_1()
4601 %{
4602 predicate(n->get_ptr() == 1);
4603 match(ConP);
4604
4605 op_cost(0);
4606 format %{ %}
4607 interface(CONST_INTER);
4608 %}
4609
4610 // Card Table Byte Map Base
4611 operand immByteMapBase()
4612 %{
4613 // Get base of card map
4614 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4615 is_card_table_address((address)(n->get_ptr())));
4616 match(ConP);
4617
4618 op_cost(0);
4619 format %{ %}
4620 interface(CONST_INTER);
4621 %}
4622
4623 // AOT Runtime Constants Address
4624 operand immAOTRuntimeConstantsAddress()
4625 %{
4626 // Check if the address is in the range of AOT Runtime Constants
4627 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4628 match(ConP);
4629
4630 op_cost(0);
4631 format %{ %}
4632 interface(CONST_INTER);
4633 %}
4634
4635 // Float and Double operands
4636 // Double Immediate
4637 operand immD()
4638 %{
4639 match(ConD);
4640 op_cost(0);
4641 format %{ %}
4642 interface(CONST_INTER);
4643 %}
4644
4645 // Double Immediate: +0.0d
4646 operand immD0()
4647 %{
4648 predicate(jlong_cast(n->getd()) == 0);
4649 match(ConD);
4650
4651 op_cost(0);
4652 format %{ %}
4653 interface(CONST_INTER);
4654 %}
4655
4656 // constant 'double +0.0'.
4657 operand immDPacked()
4658 %{
4659 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4660 match(ConD);
4661 op_cost(0);
4662 format %{ %}
4663 interface(CONST_INTER);
4664 %}
4665
4666 // Float Immediate
4667 operand immF()
4668 %{
4669 match(ConF);
4670 op_cost(0);
4671 format %{ %}
4672 interface(CONST_INTER);
4673 %}
4674
4675 // Float Immediate: +0.0f.
4676 operand immF0()
4677 %{
4678 predicate(jint_cast(n->getf()) == 0);
4679 match(ConF);
4680
4681 op_cost(0);
4682 format %{ %}
4683 interface(CONST_INTER);
4684 %}
4685
4686 // Half Float (FP16) Immediate
4687 operand immH()
4688 %{
4689 match(ConH);
4690 op_cost(0);
4691 format %{ %}
4692 interface(CONST_INTER);
4693 %}
4694
4695 //
4696 operand immFPacked()
4697 %{
4698 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4699 match(ConF);
4700 op_cost(0);
4701 format %{ %}
4702 interface(CONST_INTER);
4703 %}
4704
4705 // Narrow pointer operands
4706 // Narrow Pointer Immediate
4707 operand immN()
4708 %{
4709 match(ConN);
4710
4711 op_cost(0);
4712 format %{ %}
4713 interface(CONST_INTER);
4714 %}
4715
4716 // Narrow nullptr Pointer Immediate
4717 operand immN0()
4718 %{
4719 predicate(n->get_narrowcon() == 0);
4720 match(ConN);
4721
4722 op_cost(0);
4723 format %{ %}
4724 interface(CONST_INTER);
4725 %}
4726
4727 operand immNKlass()
4728 %{
4729 match(ConNKlass);
4730
4731 op_cost(0);
4732 format %{ %}
4733 interface(CONST_INTER);
4734 %}
4735
4736 // Integer 32 bit Register Operands
4737 // Integer 32 bitRegister (excludes SP)
4738 operand iRegI()
4739 %{
4740 constraint(ALLOC_IN_RC(any_reg32));
4741 match(RegI);
4742 match(iRegINoSp);
4743 op_cost(0);
4744 format %{ %}
4745 interface(REG_INTER);
4746 %}
4747
4748 // Integer 32 bit Register not Special
4749 operand iRegINoSp()
4750 %{
4751 constraint(ALLOC_IN_RC(no_special_reg32));
4752 match(RegI);
4753 op_cost(0);
4754 format %{ %}
4755 interface(REG_INTER);
4756 %}
4757
4758 // Integer 64 bit Register Operands
4759 // Integer 64 bit Register (includes SP)
4760 operand iRegL()
4761 %{
4762 constraint(ALLOC_IN_RC(any_reg));
4763 match(RegL);
4764 match(iRegLNoSp);
4765 op_cost(0);
4766 format %{ %}
4767 interface(REG_INTER);
4768 %}
4769
4770 // Integer 64 bit Register not Special
4771 operand iRegLNoSp()
4772 %{
4773 constraint(ALLOC_IN_RC(no_special_reg));
4774 match(RegL);
4775 match(iRegL_R0);
4776 format %{ %}
4777 interface(REG_INTER);
4778 %}
4779
4780 // Pointer Register Operands
4781 // Pointer Register
4782 operand iRegP()
4783 %{
4784 constraint(ALLOC_IN_RC(ptr_reg));
4785 match(RegP);
4786 match(iRegPNoSp);
4787 match(iRegP_R0);
4788 //match(iRegP_R2);
4789 //match(iRegP_R4);
4790 match(iRegP_R5);
4791 match(thread_RegP);
4792 op_cost(0);
4793 format %{ %}
4794 interface(REG_INTER);
4795 %}
4796
4797 // Pointer 64 bit Register not Special
4798 operand iRegPNoSp()
4799 %{
4800 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4801 match(RegP);
4802 // match(iRegP);
4803 // match(iRegP_R0);
4804 // match(iRegP_R2);
4805 // match(iRegP_R4);
4806 // match(iRegP_R5);
4807 // match(thread_RegP);
4808 op_cost(0);
4809 format %{ %}
4810 interface(REG_INTER);
4811 %}
4812
4813 // This operand is not allowed to use rfp even if
4814 // rfp is not used to hold the frame pointer.
4815 operand iRegPNoSpNoRfp()
4816 %{
4817 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4818 match(RegP);
4819 match(iRegPNoSp);
4820 op_cost(0);
4821 format %{ %}
4822 interface(REG_INTER);
4823 %}
4824
4825 // Pointer 64 bit Register R0 only
4826 operand iRegP_R0()
4827 %{
4828 constraint(ALLOC_IN_RC(r0_reg));
4829 match(RegP);
4830 // match(iRegP);
4831 match(iRegPNoSp);
4832 op_cost(0);
4833 format %{ %}
4834 interface(REG_INTER);
4835 %}
4836
4837 // Pointer 64 bit Register R1 only
4838 operand iRegP_R1()
4839 %{
4840 constraint(ALLOC_IN_RC(r1_reg));
4841 match(RegP);
4842 // match(iRegP);
4843 match(iRegPNoSp);
4844 op_cost(0);
4845 format %{ %}
4846 interface(REG_INTER);
4847 %}
4848
4849 // Pointer 64 bit Register R2 only
4850 operand iRegP_R2()
4851 %{
4852 constraint(ALLOC_IN_RC(r2_reg));
4853 match(RegP);
4854 // match(iRegP);
4855 match(iRegPNoSp);
4856 op_cost(0);
4857 format %{ %}
4858 interface(REG_INTER);
4859 %}
4860
4861 // Pointer 64 bit Register R3 only
4862 operand iRegP_R3()
4863 %{
4864 constraint(ALLOC_IN_RC(r3_reg));
4865 match(RegP);
4866 // match(iRegP);
4867 match(iRegPNoSp);
4868 op_cost(0);
4869 format %{ %}
4870 interface(REG_INTER);
4871 %}
4872
4873 // Pointer 64 bit Register R4 only
4874 operand iRegP_R4()
4875 %{
4876 constraint(ALLOC_IN_RC(r4_reg));
4877 match(RegP);
4878 // match(iRegP);
4879 match(iRegPNoSp);
4880 op_cost(0);
4881 format %{ %}
4882 interface(REG_INTER);
4883 %}
4884
4885 // Pointer 64 bit Register R5 only
4886 operand iRegP_R5()
4887 %{
4888 constraint(ALLOC_IN_RC(r5_reg));
4889 match(RegP);
4890 // match(iRegP);
4891 match(iRegPNoSp);
4892 op_cost(0);
4893 format %{ %}
4894 interface(REG_INTER);
4895 %}
4896
4897 // Pointer 64 bit Register R10 only
4898 operand iRegP_R10()
4899 %{
4900 constraint(ALLOC_IN_RC(r10_reg));
4901 match(RegP);
4902 // match(iRegP);
4903 match(iRegPNoSp);
4904 op_cost(0);
4905 format %{ %}
4906 interface(REG_INTER);
4907 %}
4908
4909 // Long 64 bit Register R0 only
4910 operand iRegL_R0()
4911 %{
4912 constraint(ALLOC_IN_RC(r0_reg));
4913 match(RegL);
4914 match(iRegLNoSp);
4915 op_cost(0);
4916 format %{ %}
4917 interface(REG_INTER);
4918 %}
4919
4920 // Long 64 bit Register R11 only
4921 operand iRegL_R11()
4922 %{
4923 constraint(ALLOC_IN_RC(r11_reg));
4924 match(RegL);
4925 match(iRegLNoSp);
4926 op_cost(0);
4927 format %{ %}
4928 interface(REG_INTER);
4929 %}
4930
4931 // Register R0 only
4932 operand iRegI_R0()
4933 %{
4934 constraint(ALLOC_IN_RC(int_r0_reg));
4935 match(RegI);
4936 match(iRegINoSp);
4937 op_cost(0);
4938 format %{ %}
4939 interface(REG_INTER);
4940 %}
4941
4942 // Register R2 only
4943 operand iRegI_R2()
4944 %{
4945 constraint(ALLOC_IN_RC(int_r2_reg));
4946 match(RegI);
4947 match(iRegINoSp);
4948 op_cost(0);
4949 format %{ %}
4950 interface(REG_INTER);
4951 %}
4952
4953 // Register R3 only
4954 operand iRegI_R3()
4955 %{
4956 constraint(ALLOC_IN_RC(int_r3_reg));
4957 match(RegI);
4958 match(iRegINoSp);
4959 op_cost(0);
4960 format %{ %}
4961 interface(REG_INTER);
4962 %}
4963
4964
4965 // Register R4 only
4966 operand iRegI_R4()
4967 %{
4968 constraint(ALLOC_IN_RC(int_r4_reg));
4969 match(RegI);
4970 match(iRegINoSp);
4971 op_cost(0);
4972 format %{ %}
4973 interface(REG_INTER);
4974 %}
4975
4976
4977 // Pointer Register Operands
4978 // Narrow Pointer Register
4979 operand iRegN()
4980 %{
4981 constraint(ALLOC_IN_RC(any_reg32));
4982 match(RegN);
4983 match(iRegNNoSp);
4984 op_cost(0);
4985 format %{ %}
4986 interface(REG_INTER);
4987 %}
4988
4989 // Integer 64 bit Register not Special
4990 operand iRegNNoSp()
4991 %{
4992 constraint(ALLOC_IN_RC(no_special_reg32));
4993 match(RegN);
4994 op_cost(0);
4995 format %{ %}
4996 interface(REG_INTER);
4997 %}
4998
4999 // Float Register
5000 // Float register operands
5001 operand vRegF()
5002 %{
5003 constraint(ALLOC_IN_RC(float_reg));
5004 match(RegF);
5005
5006 op_cost(0);
5007 format %{ %}
5008 interface(REG_INTER);
5009 %}
5010
5011 // Double Register
5012 // Double register operands
5013 operand vRegD()
5014 %{
5015 constraint(ALLOC_IN_RC(double_reg));
5016 match(RegD);
5017
5018 op_cost(0);
5019 format %{ %}
5020 interface(REG_INTER);
5021 %}
5022
5023 // Generic vector class. This will be used for
5024 // all vector operands, including NEON and SVE.
5025 operand vReg()
5026 %{
5027 constraint(ALLOC_IN_RC(dynamic));
5028 match(VecA);
5029 match(VecD);
5030 match(VecX);
5031
5032 op_cost(0);
5033 format %{ %}
5034 interface(REG_INTER);
5035 %}
5036
5037 operand vReg_V10()
5038 %{
5039 constraint(ALLOC_IN_RC(v10_veca_reg));
5040 match(vReg);
5041
5042 op_cost(0);
5043 format %{ %}
5044 interface(REG_INTER);
5045 %}
5046
5047 operand vReg_V11()
5048 %{
5049 constraint(ALLOC_IN_RC(v11_veca_reg));
5050 match(vReg);
5051
5052 op_cost(0);
5053 format %{ %}
5054 interface(REG_INTER);
5055 %}
5056
5057 operand vReg_V12()
5058 %{
5059 constraint(ALLOC_IN_RC(v12_veca_reg));
5060 match(vReg);
5061
5062 op_cost(0);
5063 format %{ %}
5064 interface(REG_INTER);
5065 %}
5066
5067 operand vReg_V13()
5068 %{
5069 constraint(ALLOC_IN_RC(v13_veca_reg));
5070 match(vReg);
5071
5072 op_cost(0);
5073 format %{ %}
5074 interface(REG_INTER);
5075 %}
5076
5077 operand vReg_V17()
5078 %{
5079 constraint(ALLOC_IN_RC(v17_veca_reg));
5080 match(vReg);
5081
5082 op_cost(0);
5083 format %{ %}
5084 interface(REG_INTER);
5085 %}
5086
5087 operand vReg_V18()
5088 %{
5089 constraint(ALLOC_IN_RC(v18_veca_reg));
5090 match(vReg);
5091
5092 op_cost(0);
5093 format %{ %}
5094 interface(REG_INTER);
5095 %}
5096
5097 operand vReg_V23()
5098 %{
5099 constraint(ALLOC_IN_RC(v23_veca_reg));
5100 match(vReg);
5101
5102 op_cost(0);
5103 format %{ %}
5104 interface(REG_INTER);
5105 %}
5106
5107 operand vReg_V24()
5108 %{
5109 constraint(ALLOC_IN_RC(v24_veca_reg));
5110 match(vReg);
5111
5112 op_cost(0);
5113 format %{ %}
5114 interface(REG_INTER);
5115 %}
5116
5117 operand vecA()
5118 %{
5119 constraint(ALLOC_IN_RC(vectora_reg));
5120 match(VecA);
5121
5122 op_cost(0);
5123 format %{ %}
5124 interface(REG_INTER);
5125 %}
5126
5127 operand vecD()
5128 %{
5129 constraint(ALLOC_IN_RC(vectord_reg));
5130 match(VecD);
5131
5132 op_cost(0);
5133 format %{ %}
5134 interface(REG_INTER);
5135 %}
5136
5137 operand vecX()
5138 %{
5139 constraint(ALLOC_IN_RC(vectorx_reg));
5140 match(VecX);
5141
5142 op_cost(0);
5143 format %{ %}
5144 interface(REG_INTER);
5145 %}
5146
5147 operand vRegD_V0()
5148 %{
5149 constraint(ALLOC_IN_RC(v0_reg));
5150 match(RegD);
5151 op_cost(0);
5152 format %{ %}
5153 interface(REG_INTER);
5154 %}
5155
5156 operand vRegD_V1()
5157 %{
5158 constraint(ALLOC_IN_RC(v1_reg));
5159 match(RegD);
5160 op_cost(0);
5161 format %{ %}
5162 interface(REG_INTER);
5163 %}
5164
5165 operand vRegD_V2()
5166 %{
5167 constraint(ALLOC_IN_RC(v2_reg));
5168 match(RegD);
5169 op_cost(0);
5170 format %{ %}
5171 interface(REG_INTER);
5172 %}
5173
5174 operand vRegD_V3()
5175 %{
5176 constraint(ALLOC_IN_RC(v3_reg));
5177 match(RegD);
5178 op_cost(0);
5179 format %{ %}
5180 interface(REG_INTER);
5181 %}
5182
5183 operand vRegD_V4()
5184 %{
5185 constraint(ALLOC_IN_RC(v4_reg));
5186 match(RegD);
5187 op_cost(0);
5188 format %{ %}
5189 interface(REG_INTER);
5190 %}
5191
5192 operand vRegD_V5()
5193 %{
5194 constraint(ALLOC_IN_RC(v5_reg));
5195 match(RegD);
5196 op_cost(0);
5197 format %{ %}
5198 interface(REG_INTER);
5199 %}
5200
5201 operand vRegD_V6()
5202 %{
5203 constraint(ALLOC_IN_RC(v6_reg));
5204 match(RegD);
5205 op_cost(0);
5206 format %{ %}
5207 interface(REG_INTER);
5208 %}
5209
5210 operand vRegD_V7()
5211 %{
5212 constraint(ALLOC_IN_RC(v7_reg));
5213 match(RegD);
5214 op_cost(0);
5215 format %{ %}
5216 interface(REG_INTER);
5217 %}
5218
5219 operand vRegD_V12()
5220 %{
5221 constraint(ALLOC_IN_RC(v12_reg));
5222 match(RegD);
5223 op_cost(0);
5224 format %{ %}
5225 interface(REG_INTER);
5226 %}
5227
5228 operand vRegD_V13()
5229 %{
5230 constraint(ALLOC_IN_RC(v13_reg));
5231 match(RegD);
5232 op_cost(0);
5233 format %{ %}
5234 interface(REG_INTER);
5235 %}
5236
5237 operand pReg()
5238 %{
5239 constraint(ALLOC_IN_RC(pr_reg));
5240 match(RegVectMask);
5241 match(pRegGov);
5242 op_cost(0);
5243 format %{ %}
5244 interface(REG_INTER);
5245 %}
5246
5247 operand pRegGov()
5248 %{
5249 constraint(ALLOC_IN_RC(gov_pr));
5250 match(RegVectMask);
5251 match(pReg);
5252 op_cost(0);
5253 format %{ %}
5254 interface(REG_INTER);
5255 %}
5256
5257 operand pRegGov_P0()
5258 %{
5259 constraint(ALLOC_IN_RC(p0_reg));
5260 match(RegVectMask);
5261 op_cost(0);
5262 format %{ %}
5263 interface(REG_INTER);
5264 %}
5265
5266 operand pRegGov_P1()
5267 %{
5268 constraint(ALLOC_IN_RC(p1_reg));
5269 match(RegVectMask);
5270 op_cost(0);
5271 format %{ %}
5272 interface(REG_INTER);
5273 %}
5274
5275 // Flags register, used as output of signed compare instructions
5276
5277 // note that on AArch64 we also use this register as the output for
5278 // for floating point compare instructions (CmpF CmpD). this ensures
5279 // that ordered inequality tests use GT, GE, LT or LE none of which
5280 // pass through cases where the result is unordered i.e. one or both
5281 // inputs to the compare is a NaN. this means that the ideal code can
5282 // replace e.g. a GT with an LE and not end up capturing the NaN case
5283 // (where the comparison should always fail). EQ and NE tests are
5284 // always generated in ideal code so that unordered folds into the NE
5285 // case, matching the behaviour of AArch64 NE.
5286 //
5287 // This differs from x86 where the outputs of FP compares use a
5288 // special FP flags registers and where compares based on this
5289 // register are distinguished into ordered inequalities (cmpOpUCF) and
5290 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5291 // to explicitly handle the unordered case in branches. x86 also has
5292 // to include extra CMoveX rules to accept a cmpOpUCF input.
5293
5294 operand rFlagsReg()
5295 %{
5296 constraint(ALLOC_IN_RC(int_flags));
5297 match(RegFlags);
5298
5299 op_cost(0);
5300 format %{ "RFLAGS" %}
5301 interface(REG_INTER);
5302 %}
5303
5304 // Flags register, used as output of unsigned compare instructions
5305 operand rFlagsRegU()
5306 %{
5307 constraint(ALLOC_IN_RC(int_flags));
5308 match(RegFlags);
5309
5310 op_cost(0);
5311 format %{ "RFLAGSU" %}
5312 interface(REG_INTER);
5313 %}
5314
5315 // Special Registers
5316
5317 // Method Register
5318 operand inline_cache_RegP(iRegP reg)
5319 %{
5320 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5321 match(reg);
5322 match(iRegPNoSp);
5323 op_cost(0);
5324 format %{ %}
5325 interface(REG_INTER);
5326 %}
5327
5328 // Thread Register
5329 operand thread_RegP(iRegP reg)
5330 %{
5331 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5332 match(reg);
5333 op_cost(0);
5334 format %{ %}
5335 interface(REG_INTER);
5336 %}
5337
5338 //----------Memory Operands----------------------------------------------------
5339
5340 operand indirect(iRegP reg)
5341 %{
5342 constraint(ALLOC_IN_RC(ptr_reg));
5343 match(reg);
5344 op_cost(0);
5345 format %{ "[$reg]" %}
5346 interface(MEMORY_INTER) %{
5347 base($reg);
5348 index(0xffffffff);
5349 scale(0x0);
5350 disp(0x0);
5351 %}
5352 %}
5353
5354 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5355 %{
5356 constraint(ALLOC_IN_RC(ptr_reg));
5357 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5358 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5359 op_cost(0);
5360 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5361 interface(MEMORY_INTER) %{
5362 base($reg);
5363 index($ireg);
5364 scale($scale);
5365 disp(0x0);
5366 %}
5367 %}
5368
5369 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5370 %{
5371 constraint(ALLOC_IN_RC(ptr_reg));
5372 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5373 match(AddP reg (LShiftL lreg scale));
5374 op_cost(0);
5375 format %{ "$reg, $lreg lsl($scale)" %}
5376 interface(MEMORY_INTER) %{
5377 base($reg);
5378 index($lreg);
5379 scale($scale);
5380 disp(0x0);
5381 %}
5382 %}
5383
5384 operand indIndexI2L(iRegP reg, iRegI ireg)
5385 %{
5386 constraint(ALLOC_IN_RC(ptr_reg));
5387 match(AddP reg (ConvI2L ireg));
5388 op_cost(0);
5389 format %{ "$reg, $ireg, 0, I2L" %}
5390 interface(MEMORY_INTER) %{
5391 base($reg);
5392 index($ireg);
5393 scale(0x0);
5394 disp(0x0);
5395 %}
5396 %}
5397
5398 operand indIndex(iRegP reg, iRegL lreg)
5399 %{
5400 constraint(ALLOC_IN_RC(ptr_reg));
5401 match(AddP reg lreg);
5402 op_cost(0);
5403 format %{ "$reg, $lreg" %}
5404 interface(MEMORY_INTER) %{
5405 base($reg);
5406 index($lreg);
5407 scale(0x0);
5408 disp(0x0);
5409 %}
5410 %}
5411
5412 operand indOffI1(iRegP reg, immIOffset1 off)
5413 %{
5414 constraint(ALLOC_IN_RC(ptr_reg));
5415 match(AddP reg off);
5416 op_cost(0);
5417 format %{ "[$reg, $off]" %}
5418 interface(MEMORY_INTER) %{
5419 base($reg);
5420 index(0xffffffff);
5421 scale(0x0);
5422 disp($off);
5423 %}
5424 %}
5425
5426 operand indOffI2(iRegP reg, immIOffset2 off)
5427 %{
5428 constraint(ALLOC_IN_RC(ptr_reg));
5429 match(AddP reg off);
5430 op_cost(0);
5431 format %{ "[$reg, $off]" %}
5432 interface(MEMORY_INTER) %{
5433 base($reg);
5434 index(0xffffffff);
5435 scale(0x0);
5436 disp($off);
5437 %}
5438 %}
5439
5440 operand indOffI4(iRegP reg, immIOffset4 off)
5441 %{
5442 constraint(ALLOC_IN_RC(ptr_reg));
5443 match(AddP reg off);
5444 op_cost(0);
5445 format %{ "[$reg, $off]" %}
5446 interface(MEMORY_INTER) %{
5447 base($reg);
5448 index(0xffffffff);
5449 scale(0x0);
5450 disp($off);
5451 %}
5452 %}
5453
5454 operand indOffI8(iRegP reg, immIOffset8 off)
5455 %{
5456 constraint(ALLOC_IN_RC(ptr_reg));
5457 match(AddP reg off);
5458 op_cost(0);
5459 format %{ "[$reg, $off]" %}
5460 interface(MEMORY_INTER) %{
5461 base($reg);
5462 index(0xffffffff);
5463 scale(0x0);
5464 disp($off);
5465 %}
5466 %}
5467
5468 operand indOffI16(iRegP reg, immIOffset16 off)
5469 %{
5470 constraint(ALLOC_IN_RC(ptr_reg));
5471 match(AddP reg off);
5472 op_cost(0);
5473 format %{ "[$reg, $off]" %}
5474 interface(MEMORY_INTER) %{
5475 base($reg);
5476 index(0xffffffff);
5477 scale(0x0);
5478 disp($off);
5479 %}
5480 %}
5481
5482 operand indOffL1(iRegP reg, immLoffset1 off)
5483 %{
5484 constraint(ALLOC_IN_RC(ptr_reg));
5485 match(AddP reg off);
5486 op_cost(0);
5487 format %{ "[$reg, $off]" %}
5488 interface(MEMORY_INTER) %{
5489 base($reg);
5490 index(0xffffffff);
5491 scale(0x0);
5492 disp($off);
5493 %}
5494 %}
5495
5496 operand indOffL2(iRegP reg, immLoffset2 off)
5497 %{
5498 constraint(ALLOC_IN_RC(ptr_reg));
5499 match(AddP reg off);
5500 op_cost(0);
5501 format %{ "[$reg, $off]" %}
5502 interface(MEMORY_INTER) %{
5503 base($reg);
5504 index(0xffffffff);
5505 scale(0x0);
5506 disp($off);
5507 %}
5508 %}
5509
5510 operand indOffL4(iRegP reg, immLoffset4 off)
5511 %{
5512 constraint(ALLOC_IN_RC(ptr_reg));
5513 match(AddP reg off);
5514 op_cost(0);
5515 format %{ "[$reg, $off]" %}
5516 interface(MEMORY_INTER) %{
5517 base($reg);
5518 index(0xffffffff);
5519 scale(0x0);
5520 disp($off);
5521 %}
5522 %}
5523
5524 operand indOffL8(iRegP reg, immLoffset8 off)
5525 %{
5526 constraint(ALLOC_IN_RC(ptr_reg));
5527 match(AddP reg off);
5528 op_cost(0);
5529 format %{ "[$reg, $off]" %}
5530 interface(MEMORY_INTER) %{
5531 base($reg);
5532 index(0xffffffff);
5533 scale(0x0);
5534 disp($off);
5535 %}
5536 %}
5537
5538 operand indOffL16(iRegP reg, immLoffset16 off)
5539 %{
5540 constraint(ALLOC_IN_RC(ptr_reg));
5541 match(AddP reg off);
5542 op_cost(0);
5543 format %{ "[$reg, $off]" %}
5544 interface(MEMORY_INTER) %{
5545 base($reg);
5546 index(0xffffffff);
5547 scale(0x0);
5548 disp($off);
5549 %}
5550 %}
5551
5552 operand indirectX2P(iRegL reg)
5553 %{
5554 constraint(ALLOC_IN_RC(ptr_reg));
5555 match(CastX2P reg);
5556 op_cost(0);
5557 format %{ "[$reg]\t# long -> ptr" %}
5558 interface(MEMORY_INTER) %{
5559 base($reg);
5560 index(0xffffffff);
5561 scale(0x0);
5562 disp(0x0);
5563 %}
5564 %}
5565
5566 operand indOffX2P(iRegL reg, immLOffset off)
5567 %{
5568 constraint(ALLOC_IN_RC(ptr_reg));
5569 match(AddP (CastX2P reg) off);
5570 op_cost(0);
5571 format %{ "[$reg, $off]\t# long -> ptr" %}
5572 interface(MEMORY_INTER) %{
5573 base($reg);
5574 index(0xffffffff);
5575 scale(0x0);
5576 disp($off);
5577 %}
5578 %}
5579
5580 operand indirectN(iRegN reg)
5581 %{
5582 predicate(CompressedOops::shift() == 0);
5583 constraint(ALLOC_IN_RC(ptr_reg));
5584 match(DecodeN reg);
5585 op_cost(0);
5586 format %{ "[$reg]\t# narrow" %}
5587 interface(MEMORY_INTER) %{
5588 base($reg);
5589 index(0xffffffff);
5590 scale(0x0);
5591 disp(0x0);
5592 %}
5593 %}
5594
5595 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5596 %{
5597 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5598 constraint(ALLOC_IN_RC(ptr_reg));
5599 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5600 op_cost(0);
5601 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5602 interface(MEMORY_INTER) %{
5603 base($reg);
5604 index($ireg);
5605 scale($scale);
5606 disp(0x0);
5607 %}
5608 %}
5609
5610 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5611 %{
5612 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5613 constraint(ALLOC_IN_RC(ptr_reg));
5614 match(AddP (DecodeN reg) (LShiftL lreg scale));
5615 op_cost(0);
5616 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5617 interface(MEMORY_INTER) %{
5618 base($reg);
5619 index($lreg);
5620 scale($scale);
5621 disp(0x0);
5622 %}
5623 %}
5624
5625 operand indIndexI2LN(iRegN reg, iRegI ireg)
5626 %{
5627 predicate(CompressedOops::shift() == 0);
5628 constraint(ALLOC_IN_RC(ptr_reg));
5629 match(AddP (DecodeN reg) (ConvI2L ireg));
5630 op_cost(0);
5631 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5632 interface(MEMORY_INTER) %{
5633 base($reg);
5634 index($ireg);
5635 scale(0x0);
5636 disp(0x0);
5637 %}
5638 %}
5639
5640 operand indIndexN(iRegN reg, iRegL lreg)
5641 %{
5642 predicate(CompressedOops::shift() == 0);
5643 constraint(ALLOC_IN_RC(ptr_reg));
5644 match(AddP (DecodeN reg) lreg);
5645 op_cost(0);
5646 format %{ "$reg, $lreg\t# narrow" %}
5647 interface(MEMORY_INTER) %{
5648 base($reg);
5649 index($lreg);
5650 scale(0x0);
5651 disp(0x0);
5652 %}
5653 %}
5654
5655 operand indOffIN(iRegN reg, immIOffset off)
5656 %{
5657 predicate(CompressedOops::shift() == 0);
5658 constraint(ALLOC_IN_RC(ptr_reg));
5659 match(AddP (DecodeN reg) off);
5660 op_cost(0);
5661 format %{ "[$reg, $off]\t# narrow" %}
5662 interface(MEMORY_INTER) %{
5663 base($reg);
5664 index(0xffffffff);
5665 scale(0x0);
5666 disp($off);
5667 %}
5668 %}
5669
5670 operand indOffLN(iRegN reg, immLOffset off)
5671 %{
5672 predicate(CompressedOops::shift() == 0);
5673 constraint(ALLOC_IN_RC(ptr_reg));
5674 match(AddP (DecodeN reg) off);
5675 op_cost(0);
5676 format %{ "[$reg, $off]\t# narrow" %}
5677 interface(MEMORY_INTER) %{
5678 base($reg);
5679 index(0xffffffff);
5680 scale(0x0);
5681 disp($off);
5682 %}
5683 %}
5684
5685
5686 //----------Special Memory Operands--------------------------------------------
5687 // Stack Slot Operand - This operand is used for loading and storing temporary
5688 // values on the stack where a match requires a value to
5689 // flow through memory.
5690 operand stackSlotP(sRegP reg)
5691 %{
5692 constraint(ALLOC_IN_RC(stack_slots));
5693 op_cost(100);
5694 // No match rule because this operand is only generated in matching
5695 // match(RegP);
5696 format %{ "[$reg]" %}
5697 interface(MEMORY_INTER) %{
5698 base(0x1e); // RSP
5699 index(0x0); // No Index
5700 scale(0x0); // No Scale
5701 disp($reg); // Stack Offset
5702 %}
5703 %}
5704
5705 operand stackSlotI(sRegI reg)
5706 %{
5707 constraint(ALLOC_IN_RC(stack_slots));
5708 // No match rule because this operand is only generated in matching
5709 // match(RegI);
5710 format %{ "[$reg]" %}
5711 interface(MEMORY_INTER) %{
5712 base(0x1e); // RSP
5713 index(0x0); // No Index
5714 scale(0x0); // No Scale
5715 disp($reg); // Stack Offset
5716 %}
5717 %}
5718
5719 operand stackSlotF(sRegF reg)
5720 %{
5721 constraint(ALLOC_IN_RC(stack_slots));
5722 // No match rule because this operand is only generated in matching
5723 // match(RegF);
5724 format %{ "[$reg]" %}
5725 interface(MEMORY_INTER) %{
5726 base(0x1e); // RSP
5727 index(0x0); // No Index
5728 scale(0x0); // No Scale
5729 disp($reg); // Stack Offset
5730 %}
5731 %}
5732
5733 operand stackSlotD(sRegD reg)
5734 %{
5735 constraint(ALLOC_IN_RC(stack_slots));
5736 // No match rule because this operand is only generated in matching
5737 // match(RegD);
5738 format %{ "[$reg]" %}
5739 interface(MEMORY_INTER) %{
5740 base(0x1e); // RSP
5741 index(0x0); // No Index
5742 scale(0x0); // No Scale
5743 disp($reg); // Stack Offset
5744 %}
5745 %}
5746
5747 operand stackSlotL(sRegL reg)
5748 %{
5749 constraint(ALLOC_IN_RC(stack_slots));
5750 // No match rule because this operand is only generated in matching
5751 // match(RegL);
5752 format %{ "[$reg]" %}
5753 interface(MEMORY_INTER) %{
5754 base(0x1e); // RSP
5755 index(0x0); // No Index
5756 scale(0x0); // No Scale
5757 disp($reg); // Stack Offset
5758 %}
5759 %}
5760
5761 // Operands for expressing Control Flow
5762 // NOTE: Label is a predefined operand which should not be redefined in
5763 // the AD file. It is generically handled within the ADLC.
5764
5765 //----------Conditional Branch Operands----------------------------------------
5766 // Comparison Op - This is the operation of the comparison, and is limited to
5767 // the following set of codes:
5768 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5769 //
5770 // Other attributes of the comparison, such as unsignedness, are specified
5771 // by the comparison instruction that sets a condition code flags register.
5772 // That result is represented by a flags operand whose subtype is appropriate
5773 // to the unsignedness (etc.) of the comparison.
5774 //
5775 // Later, the instruction which matches both the Comparison Op (a Bool) and
5776 // the flags (produced by the Cmp) specifies the coding of the comparison op
5777 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5778
5779 // used for signed integral comparisons and fp comparisons
5780
5781 operand cmpOp()
5782 %{
5783 match(Bool);
5784
5785 format %{ "" %}
5786 interface(COND_INTER) %{
5787 equal(0x0, "eq");
5788 not_equal(0x1, "ne");
5789 less(0xb, "lt");
5790 greater_equal(0xa, "ge");
5791 less_equal(0xd, "le");
5792 greater(0xc, "gt");
5793 overflow(0x6, "vs");
5794 no_overflow(0x7, "vc");
5795 %}
5796 %}
5797
5798 // used for unsigned integral comparisons
5799
5800 operand cmpOpU()
5801 %{
5802 match(Bool);
5803
5804 format %{ "" %}
5805 interface(COND_INTER) %{
5806 equal(0x0, "eq");
5807 not_equal(0x1, "ne");
5808 less(0x3, "lo");
5809 greater_equal(0x2, "hs");
5810 less_equal(0x9, "ls");
5811 greater(0x8, "hi");
5812 overflow(0x6, "vs");
5813 no_overflow(0x7, "vc");
5814 %}
5815 %}
5816
5817 // used for certain integral comparisons which can be
5818 // converted to cbxx or tbxx instructions
5819
5820 operand cmpOpEqNe()
5821 %{
5822 match(Bool);
5823 op_cost(0);
5824 predicate(n->as_Bool()->_test._test == BoolTest::ne
5825 || n->as_Bool()->_test._test == BoolTest::eq);
5826
5827 format %{ "" %}
5828 interface(COND_INTER) %{
5829 equal(0x0, "eq");
5830 not_equal(0x1, "ne");
5831 less(0xb, "lt");
5832 greater_equal(0xa, "ge");
5833 less_equal(0xd, "le");
5834 greater(0xc, "gt");
5835 overflow(0x6, "vs");
5836 no_overflow(0x7, "vc");
5837 %}
5838 %}
5839
5840 // used for certain integral comparisons which can be
5841 // converted to cbxx or tbxx instructions
5842
5843 operand cmpOpLtGe()
5844 %{
5845 match(Bool);
5846 op_cost(0);
5847
5848 predicate(n->as_Bool()->_test._test == BoolTest::lt
5849 || n->as_Bool()->_test._test == BoolTest::ge);
5850
5851 format %{ "" %}
5852 interface(COND_INTER) %{
5853 equal(0x0, "eq");
5854 not_equal(0x1, "ne");
5855 less(0xb, "lt");
5856 greater_equal(0xa, "ge");
5857 less_equal(0xd, "le");
5858 greater(0xc, "gt");
5859 overflow(0x6, "vs");
5860 no_overflow(0x7, "vc");
5861 %}
5862 %}
5863
5864 // used for certain unsigned integral comparisons which can be
5865 // converted to cbxx or tbxx instructions
5866
5867 operand cmpOpUEqNeLeGt()
5868 %{
5869 match(Bool);
5870 op_cost(0);
5871
5872 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5873 n->as_Bool()->_test._test == BoolTest::ne ||
5874 n->as_Bool()->_test._test == BoolTest::le ||
5875 n->as_Bool()->_test._test == BoolTest::gt);
5876
5877 format %{ "" %}
5878 interface(COND_INTER) %{
5879 equal(0x0, "eq");
5880 not_equal(0x1, "ne");
5881 less(0x3, "lo");
5882 greater_equal(0x2, "hs");
5883 less_equal(0x9, "ls");
5884 greater(0x8, "hi");
5885 overflow(0x6, "vs");
5886 no_overflow(0x7, "vc");
5887 %}
5888 %}
5889
5890 // Special operand allowing long args to int ops to be truncated for free
5891
5892 operand iRegL2I(iRegL reg) %{
5893
5894 op_cost(0);
5895
5896 match(ConvL2I reg);
5897
5898 format %{ "l2i($reg)" %}
5899
5900 interface(REG_INTER)
5901 %}
5902
5903 operand iRegL2P(iRegL reg) %{
5904
5905 op_cost(0);
5906
5907 match(CastX2P reg);
5908
5909 format %{ "l2p($reg)" %}
5910
5911 interface(REG_INTER)
5912 %}
5913
5914 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5915 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5916 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5917 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5918
5919 //----------OPERAND CLASSES----------------------------------------------------
5920 // Operand Classes are groups of operands that are used as to simplify
5921 // instruction definitions by not requiring the AD writer to specify
5922 // separate instructions for every form of operand when the
5923 // instruction accepts multiple operand types with the same basic
5924 // encoding and format. The classic case of this is memory operands.
5925
5926 // memory is used to define read/write location for load/store
5927 // instruction defs. we can turn a memory op into an Address
5928
5929 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5930 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5931
5932 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5933 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5934
5935 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5936 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5937
5938 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5939 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5940
5941 // All of the memory operands. For the pipeline description.
5942 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5943 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5944 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5945
5946
5947 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5948 // operations. it allows the src to be either an iRegI or a (ConvL2I
5949 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5950 // can be elided because the 32-bit instruction will just employ the
5951 // lower 32 bits anyway.
5952 //
5953 // n.b. this does not elide all L2I conversions. if the truncated
5954 // value is consumed by more than one operation then the ConvL2I
5955 // cannot be bundled into the consuming nodes so an l2i gets planted
5956 // (actually a movw $dst $src) and the downstream instructions consume
5957 // the result of the l2i as an iRegI input. That's a shame since the
5958 // movw is actually redundant but its not too costly.
5959
5960 opclass iRegIorL2I(iRegI, iRegL2I);
5961 opclass iRegPorL2P(iRegP, iRegL2P);
5962
5963 //----------PIPELINE-----------------------------------------------------------
5964 // Rules which define the behavior of the target architectures pipeline.
5965
5966 // For specific pipelines, eg A53, define the stages of that pipeline
5967 //pipe_desc(ISS, EX1, EX2, WR);
5968 #define ISS S0
5969 #define EX1 S1
5970 #define EX2 S2
5971 #define WR S3
5972
5973 // Integer ALU reg operation
5974 pipeline %{
5975
5976 attributes %{
5977 // ARM instructions are of fixed length
5978 fixed_size_instructions; // Fixed size instructions TODO does
5979 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5980 // ARM instructions come in 32-bit word units
5981 instruction_unit_size = 4; // An instruction is 4 bytes long
5982 instruction_fetch_unit_size = 64; // The processor fetches one line
5983 instruction_fetch_units = 1; // of 64 bytes
5984
5985 // List of nop instructions
5986 nops( MachNop );
5987 %}
5988
5989 // We don't use an actual pipeline model so don't care about resources
5990 // or description. we do use pipeline classes to introduce fixed
5991 // latencies
5992
5993 //----------RESOURCES----------------------------------------------------------
5994 // Resources are the functional units available to the machine
5995
5996 resources( INS0, INS1, INS01 = INS0 | INS1,
5997 ALU0, ALU1, ALU = ALU0 | ALU1,
5998 MAC,
5999 DIV,
6000 BRANCH,
6001 LDST,
6002 NEON_FP);
6003
6004 //----------PIPELINE DESCRIPTION-----------------------------------------------
6005 // Pipeline Description specifies the stages in the machine's pipeline
6006
6007 // Define the pipeline as a generic 6 stage pipeline
6008 pipe_desc(S0, S1, S2, S3, S4, S5);
6009
6010 //----------PIPELINE CLASSES---------------------------------------------------
6011 // Pipeline Classes describe the stages in which input and output are
6012 // referenced by the hardware pipeline.
6013
6014 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
6015 %{
6016 single_instruction;
6017 src1 : S1(read);
6018 src2 : S2(read);
6019 dst : S5(write);
6020 INS01 : ISS;
6021 NEON_FP : S5;
6022 %}
6023
6024 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6025 %{
6026 single_instruction;
6027 src1 : S1(read);
6028 src2 : S2(read);
6029 dst : S5(write);
6030 INS01 : ISS;
6031 NEON_FP : S5;
6032 %}
6033
6034 pipe_class fp_uop_s(vRegF dst, vRegF src)
6035 %{
6036 single_instruction;
6037 src : S1(read);
6038 dst : S5(write);
6039 INS01 : ISS;
6040 NEON_FP : S5;
6041 %}
6042
6043 pipe_class fp_uop_d(vRegD dst, vRegD src)
6044 %{
6045 single_instruction;
6046 src : S1(read);
6047 dst : S5(write);
6048 INS01 : ISS;
6049 NEON_FP : S5;
6050 %}
6051
6052 pipe_class fp_d2f(vRegF dst, vRegD src)
6053 %{
6054 single_instruction;
6055 src : S1(read);
6056 dst : S5(write);
6057 INS01 : ISS;
6058 NEON_FP : S5;
6059 %}
6060
6061 pipe_class fp_f2d(vRegD dst, vRegF src)
6062 %{
6063 single_instruction;
6064 src : S1(read);
6065 dst : S5(write);
6066 INS01 : ISS;
6067 NEON_FP : S5;
6068 %}
6069
6070 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6071 %{
6072 single_instruction;
6073 src : S1(read);
6074 dst : S5(write);
6075 INS01 : ISS;
6076 NEON_FP : S5;
6077 %}
6078
6079 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6080 %{
6081 single_instruction;
6082 src : S1(read);
6083 dst : S5(write);
6084 INS01 : ISS;
6085 NEON_FP : S5;
6086 %}
6087
6088 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6089 %{
6090 single_instruction;
6091 src : S1(read);
6092 dst : S5(write);
6093 INS01 : ISS;
6094 NEON_FP : S5;
6095 %}
6096
6097 pipe_class fp_l2f(vRegF dst, iRegL src)
6098 %{
6099 single_instruction;
6100 src : S1(read);
6101 dst : S5(write);
6102 INS01 : ISS;
6103 NEON_FP : S5;
6104 %}
6105
6106 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6107 %{
6108 single_instruction;
6109 src : S1(read);
6110 dst : S5(write);
6111 INS01 : ISS;
6112 NEON_FP : S5;
6113 %}
6114
6115 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6116 %{
6117 single_instruction;
6118 src : S1(read);
6119 dst : S5(write);
6120 INS01 : ISS;
6121 NEON_FP : S5;
6122 %}
6123
6124 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6125 %{
6126 single_instruction;
6127 src : S1(read);
6128 dst : S5(write);
6129 INS01 : ISS;
6130 NEON_FP : S5;
6131 %}
6132
6133 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6134 %{
6135 single_instruction;
6136 src : S1(read);
6137 dst : S5(write);
6138 INS01 : ISS;
6139 NEON_FP : S5;
6140 %}
6141
6142 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6143 %{
6144 single_instruction;
6145 src1 : S1(read);
6146 src2 : S2(read);
6147 dst : S5(write);
6148 INS0 : ISS;
6149 NEON_FP : S5;
6150 %}
6151
6152 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6153 %{
6154 single_instruction;
6155 src1 : S1(read);
6156 src2 : S2(read);
6157 dst : S5(write);
6158 INS0 : ISS;
6159 NEON_FP : S5;
6160 %}
6161
6162 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6163 %{
6164 single_instruction;
6165 cr : S1(read);
6166 src1 : S1(read);
6167 src2 : S1(read);
6168 dst : S3(write);
6169 INS01 : ISS;
6170 NEON_FP : S3;
6171 %}
6172
6173 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6174 %{
6175 single_instruction;
6176 cr : S1(read);
6177 src1 : S1(read);
6178 src2 : S1(read);
6179 dst : S3(write);
6180 INS01 : ISS;
6181 NEON_FP : S3;
6182 %}
6183
6184 pipe_class fp_imm_s(vRegF dst)
6185 %{
6186 single_instruction;
6187 dst : S3(write);
6188 INS01 : ISS;
6189 NEON_FP : S3;
6190 %}
6191
6192 pipe_class fp_imm_d(vRegD dst)
6193 %{
6194 single_instruction;
6195 dst : S3(write);
6196 INS01 : ISS;
6197 NEON_FP : S3;
6198 %}
6199
6200 pipe_class fp_load_constant_s(vRegF dst)
6201 %{
6202 single_instruction;
6203 dst : S4(write);
6204 INS01 : ISS;
6205 NEON_FP : S4;
6206 %}
6207
6208 pipe_class fp_load_constant_d(vRegD dst)
6209 %{
6210 single_instruction;
6211 dst : S4(write);
6212 INS01 : ISS;
6213 NEON_FP : S4;
6214 %}
6215
6216 //------- Integer ALU operations --------------------------
6217
6218 // Integer ALU reg-reg operation
6219 // Operands needed in EX1, result generated in EX2
6220 // Eg. ADD x0, x1, x2
6221 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6222 %{
6223 single_instruction;
6224 dst : EX2(write);
6225 src1 : EX1(read);
6226 src2 : EX1(read);
6227 INS01 : ISS; // Dual issue as instruction 0 or 1
6228 ALU : EX2;
6229 %}
6230
6231 // Integer ALU reg-reg operation with constant shift
6232 // Shifted register must be available in LATE_ISS instead of EX1
6233 // Eg. ADD x0, x1, x2, LSL #2
6234 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6235 %{
6236 single_instruction;
6237 dst : EX2(write);
6238 src1 : EX1(read);
6239 src2 : ISS(read);
6240 INS01 : ISS;
6241 ALU : EX2;
6242 %}
6243
6244 // Integer ALU reg operation with constant shift
6245 // Eg. LSL x0, x1, #shift
6246 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6247 %{
6248 single_instruction;
6249 dst : EX2(write);
6250 src1 : ISS(read);
6251 INS01 : ISS;
6252 ALU : EX2;
6253 %}
6254
6255 // Integer ALU reg-reg operation with variable shift
6256 // Both operands must be available in LATE_ISS instead of EX1
6257 // Result is available in EX1 instead of EX2
6258 // Eg. LSLV x0, x1, x2
6259 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6260 %{
6261 single_instruction;
6262 dst : EX1(write);
6263 src1 : ISS(read);
6264 src2 : ISS(read);
6265 INS01 : ISS;
6266 ALU : EX1;
6267 %}
6268
6269 // Integer ALU reg-reg operation with extract
6270 // As for _vshift above, but result generated in EX2
6271 // Eg. EXTR x0, x1, x2, #N
6272 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6273 %{
6274 single_instruction;
6275 dst : EX2(write);
6276 src1 : ISS(read);
6277 src2 : ISS(read);
6278 INS1 : ISS; // Can only dual issue as Instruction 1
6279 ALU : EX1;
6280 %}
6281
6282 // Integer ALU reg operation
6283 // Eg. NEG x0, x1
6284 pipe_class ialu_reg(iRegI dst, iRegI src)
6285 %{
6286 single_instruction;
6287 dst : EX2(write);
6288 src : EX1(read);
6289 INS01 : ISS;
6290 ALU : EX2;
6291 %}
6292
6293 // Integer ALU reg mmediate operation
6294 // Eg. ADD x0, x1, #N
6295 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6296 %{
6297 single_instruction;
6298 dst : EX2(write);
6299 src1 : EX1(read);
6300 INS01 : ISS;
6301 ALU : EX2;
6302 %}
6303
6304 // Integer ALU immediate operation (no source operands)
6305 // Eg. MOV x0, #N
6306 pipe_class ialu_imm(iRegI dst)
6307 %{
6308 single_instruction;
6309 dst : EX1(write);
6310 INS01 : ISS;
6311 ALU : EX1;
6312 %}
6313
6314 //------- Compare operation -------------------------------
6315
6316 // Compare reg-reg
6317 // Eg. CMP x0, x1
6318 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6319 %{
6320 single_instruction;
6321 // fixed_latency(16);
6322 cr : EX2(write);
6323 op1 : EX1(read);
6324 op2 : EX1(read);
6325 INS01 : ISS;
6326 ALU : EX2;
6327 %}
6328
6329 // Compare reg-reg
6330 // Eg. CMP x0, #N
6331 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6332 %{
6333 single_instruction;
6334 // fixed_latency(16);
6335 cr : EX2(write);
6336 op1 : EX1(read);
6337 INS01 : ISS;
6338 ALU : EX2;
6339 %}
6340
6341 //------- Conditional instructions ------------------------
6342
6343 // Conditional no operands
6344 // Eg. CSINC x0, zr, zr, <cond>
6345 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6346 %{
6347 single_instruction;
6348 cr : EX1(read);
6349 dst : EX2(write);
6350 INS01 : ISS;
6351 ALU : EX2;
6352 %}
6353
6354 // Conditional 2 operand
6355 // EG. CSEL X0, X1, X2, <cond>
6356 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6357 %{
6358 single_instruction;
6359 cr : EX1(read);
6360 src1 : EX1(read);
6361 src2 : EX1(read);
6362 dst : EX2(write);
6363 INS01 : ISS;
6364 ALU : EX2;
6365 %}
6366
6367 // Conditional 2 operand
6368 // EG. CSEL X0, X1, X2, <cond>
6369 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6370 %{
6371 single_instruction;
6372 cr : EX1(read);
6373 src : EX1(read);
6374 dst : EX2(write);
6375 INS01 : ISS;
6376 ALU : EX2;
6377 %}
6378
6379 //------- Multiply pipeline operations --------------------
6380
6381 // Multiply reg-reg
6382 // Eg. MUL w0, w1, w2
6383 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6384 %{
6385 single_instruction;
6386 dst : WR(write);
6387 src1 : ISS(read);
6388 src2 : ISS(read);
6389 INS01 : ISS;
6390 MAC : WR;
6391 %}
6392
6393 // Multiply accumulate
6394 // Eg. MADD w0, w1, w2, w3
6395 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6396 %{
6397 single_instruction;
6398 dst : WR(write);
6399 src1 : ISS(read);
6400 src2 : ISS(read);
6401 src3 : ISS(read);
6402 INS01 : ISS;
6403 MAC : WR;
6404 %}
6405
6406 // Eg. MUL w0, w1, w2
6407 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6408 %{
6409 single_instruction;
6410 fixed_latency(3); // Maximum latency for 64 bit mul
6411 dst : WR(write);
6412 src1 : ISS(read);
6413 src2 : ISS(read);
6414 INS01 : ISS;
6415 MAC : WR;
6416 %}
6417
6418 // Multiply accumulate
6419 // Eg. MADD w0, w1, w2, w3
6420 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6421 %{
6422 single_instruction;
6423 fixed_latency(3); // Maximum latency for 64 bit mul
6424 dst : WR(write);
6425 src1 : ISS(read);
6426 src2 : ISS(read);
6427 src3 : ISS(read);
6428 INS01 : ISS;
6429 MAC : WR;
6430 %}
6431
6432 //------- Divide pipeline operations --------------------
6433
6434 // Eg. SDIV w0, w1, w2
6435 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6436 %{
6437 single_instruction;
6438 fixed_latency(8); // Maximum latency for 32 bit divide
6439 dst : WR(write);
6440 src1 : ISS(read);
6441 src2 : ISS(read);
6442 INS0 : ISS; // Can only dual issue as instruction 0
6443 DIV : WR;
6444 %}
6445
6446 // Eg. SDIV x0, x1, x2
6447 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6448 %{
6449 single_instruction;
6450 fixed_latency(16); // Maximum latency for 64 bit divide
6451 dst : WR(write);
6452 src1 : ISS(read);
6453 src2 : ISS(read);
6454 INS0 : ISS; // Can only dual issue as instruction 0
6455 DIV : WR;
6456 %}
6457
6458 //------- Load pipeline operations ------------------------
6459
6460 // Load - prefetch
6461 // Eg. PFRM <mem>
6462 pipe_class iload_prefetch(memory mem)
6463 %{
6464 single_instruction;
6465 mem : ISS(read);
6466 INS01 : ISS;
6467 LDST : WR;
6468 %}
6469
6470 // Load - reg, mem
6471 // Eg. LDR x0, <mem>
6472 pipe_class iload_reg_mem(iRegI dst, memory mem)
6473 %{
6474 single_instruction;
6475 dst : WR(write);
6476 mem : ISS(read);
6477 INS01 : ISS;
6478 LDST : WR;
6479 %}
6480
6481 // Load - reg, reg
6482 // Eg. LDR x0, [sp, x1]
6483 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6484 %{
6485 single_instruction;
6486 dst : WR(write);
6487 src : ISS(read);
6488 INS01 : ISS;
6489 LDST : WR;
6490 %}
6491
6492 //------- Store pipeline operations -----------------------
6493
6494 // Store - zr, mem
6495 // Eg. STR zr, <mem>
6496 pipe_class istore_mem(memory mem)
6497 %{
6498 single_instruction;
6499 mem : ISS(read);
6500 INS01 : ISS;
6501 LDST : WR;
6502 %}
6503
6504 // Store - reg, mem
6505 // Eg. STR x0, <mem>
6506 pipe_class istore_reg_mem(iRegI src, memory mem)
6507 %{
6508 single_instruction;
6509 mem : ISS(read);
6510 src : EX2(read);
6511 INS01 : ISS;
6512 LDST : WR;
6513 %}
6514
6515 // Store - reg, reg
6516 // Eg. STR x0, [sp, x1]
6517 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6518 %{
6519 single_instruction;
6520 dst : ISS(read);
6521 src : EX2(read);
6522 INS01 : ISS;
6523 LDST : WR;
6524 %}
6525
6526 //------- Store pipeline operations -----------------------
6527
6528 // Branch
6529 pipe_class pipe_branch()
6530 %{
6531 single_instruction;
6532 INS01 : ISS;
6533 BRANCH : EX1;
6534 %}
6535
6536 // Conditional branch
6537 pipe_class pipe_branch_cond(rFlagsReg cr)
6538 %{
6539 single_instruction;
6540 cr : EX1(read);
6541 INS01 : ISS;
6542 BRANCH : EX1;
6543 %}
6544
6545 // Compare & Branch
6546 // EG. CBZ/CBNZ
6547 pipe_class pipe_cmp_branch(iRegI op1)
6548 %{
6549 single_instruction;
6550 op1 : EX1(read);
6551 INS01 : ISS;
6552 BRANCH : EX1;
6553 %}
6554
6555 //------- Synchronisation operations ----------------------
6556
6557 // Any operation requiring serialization.
6558 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6559 pipe_class pipe_serial()
6560 %{
6561 single_instruction;
6562 force_serialization;
6563 fixed_latency(16);
6564 INS01 : ISS(2); // Cannot dual issue with any other instruction
6565 LDST : WR;
6566 %}
6567
6568 // Generic big/slow expanded idiom - also serialized
6569 pipe_class pipe_slow()
6570 %{
6571 instruction_count(10);
6572 multiple_bundles;
6573 force_serialization;
6574 fixed_latency(16);
6575 INS01 : ISS(2); // Cannot dual issue with any other instruction
6576 LDST : WR;
6577 %}
6578
6579 // Empty pipeline class
6580 pipe_class pipe_class_empty()
6581 %{
6582 single_instruction;
6583 fixed_latency(0);
6584 %}
6585
6586 // Default pipeline class.
6587 pipe_class pipe_class_default()
6588 %{
6589 single_instruction;
6590 fixed_latency(2);
6591 %}
6592
6593 // Pipeline class for compares.
6594 pipe_class pipe_class_compare()
6595 %{
6596 single_instruction;
6597 fixed_latency(16);
6598 %}
6599
6600 // Pipeline class for memory operations.
6601 pipe_class pipe_class_memory()
6602 %{
6603 single_instruction;
6604 fixed_latency(16);
6605 %}
6606
6607 // Pipeline class for call.
6608 pipe_class pipe_class_call()
6609 %{
6610 single_instruction;
6611 fixed_latency(100);
6612 %}
6613
6614 // Define the class for the Nop node.
6615 define %{
6616 MachNop = pipe_class_empty;
6617 %}
6618
6619 %}
6620 //----------INSTRUCTIONS-------------------------------------------------------
6621 //
6622 // match -- States which machine-independent subtree may be replaced
6623 // by this instruction.
6624 // ins_cost -- The estimated cost of this instruction is used by instruction
6625 // selection to identify a minimum cost tree of machine
6626 // instructions that matches a tree of machine-independent
6627 // instructions.
6628 // format -- A string providing the disassembly for this instruction.
6629 // The value of an instruction's operand may be inserted
6630 // by referring to it with a '$' prefix.
6631 // opcode -- Three instruction opcodes may be provided. These are referred
6632 // to within an encode class as $primary, $secondary, and $tertiary
6633 // rrspectively. The primary opcode is commonly used to
6634 // indicate the type of machine instruction, while secondary
6635 // and tertiary are often used for prefix options or addressing
6636 // modes.
6637 // ins_encode -- A list of encode classes with parameters. The encode class
6638 // name must have been defined in an 'enc_class' specification
6639 // in the encode section of the architecture description.
6640
6641 // ============================================================================
6642 // Memory (Load/Store) Instructions
6643
6644 // Load Instructions
6645
6646 // Load Byte (8 bit signed)
6647 instruct loadB(iRegINoSp dst, memory1 mem)
6648 %{
6649 match(Set dst (LoadB mem));
6650 predicate(!needs_acquiring_load(n));
6651
6652 ins_cost(4 * INSN_COST);
6653 format %{ "ldrsbw $dst, $mem\t# byte" %}
6654
6655 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6656
6657 ins_pipe(iload_reg_mem);
6658 %}
6659
6660 // Load Byte (8 bit signed) into long
6661 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6662 %{
6663 match(Set dst (ConvI2L (LoadB mem)));
6664 predicate(!needs_acquiring_load(n->in(1)));
6665
6666 ins_cost(4 * INSN_COST);
6667 format %{ "ldrsb $dst, $mem\t# byte" %}
6668
6669 ins_encode(aarch64_enc_ldrsb(dst, mem));
6670
6671 ins_pipe(iload_reg_mem);
6672 %}
6673
6674 // Load Byte (8 bit unsigned)
6675 instruct loadUB(iRegINoSp dst, memory1 mem)
6676 %{
6677 match(Set dst (LoadUB mem));
6678 predicate(!needs_acquiring_load(n));
6679
6680 ins_cost(4 * INSN_COST);
6681 format %{ "ldrbw $dst, $mem\t# byte" %}
6682
6683 ins_encode(aarch64_enc_ldrb(dst, mem));
6684
6685 ins_pipe(iload_reg_mem);
6686 %}
6687
6688 // Load Byte (8 bit unsigned) into long
6689 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6690 %{
6691 match(Set dst (ConvI2L (LoadUB mem)));
6692 predicate(!needs_acquiring_load(n->in(1)));
6693
6694 ins_cost(4 * INSN_COST);
6695 format %{ "ldrb $dst, $mem\t# byte" %}
6696
6697 ins_encode(aarch64_enc_ldrb(dst, mem));
6698
6699 ins_pipe(iload_reg_mem);
6700 %}
6701
6702 // Load Short (16 bit signed)
6703 instruct loadS(iRegINoSp dst, memory2 mem)
6704 %{
6705 match(Set dst (LoadS mem));
6706 predicate(!needs_acquiring_load(n));
6707
6708 ins_cost(4 * INSN_COST);
6709 format %{ "ldrshw $dst, $mem\t# short" %}
6710
6711 ins_encode(aarch64_enc_ldrshw(dst, mem));
6712
6713 ins_pipe(iload_reg_mem);
6714 %}
6715
6716 // Load Short (16 bit signed) into long
6717 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6718 %{
6719 match(Set dst (ConvI2L (LoadS mem)));
6720 predicate(!needs_acquiring_load(n->in(1)));
6721
6722 ins_cost(4 * INSN_COST);
6723 format %{ "ldrsh $dst, $mem\t# short" %}
6724
6725 ins_encode(aarch64_enc_ldrsh(dst, mem));
6726
6727 ins_pipe(iload_reg_mem);
6728 %}
6729
6730 // Load Char (16 bit unsigned)
6731 instruct loadUS(iRegINoSp dst, memory2 mem)
6732 %{
6733 match(Set dst (LoadUS mem));
6734 predicate(!needs_acquiring_load(n));
6735
6736 ins_cost(4 * INSN_COST);
6737 format %{ "ldrh $dst, $mem\t# short" %}
6738
6739 ins_encode(aarch64_enc_ldrh(dst, mem));
6740
6741 ins_pipe(iload_reg_mem);
6742 %}
6743
6744 // Load Short/Char (16 bit unsigned) into long
6745 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6746 %{
6747 match(Set dst (ConvI2L (LoadUS mem)));
6748 predicate(!needs_acquiring_load(n->in(1)));
6749
6750 ins_cost(4 * INSN_COST);
6751 format %{ "ldrh $dst, $mem\t# short" %}
6752
6753 ins_encode(aarch64_enc_ldrh(dst, mem));
6754
6755 ins_pipe(iload_reg_mem);
6756 %}
6757
6758 // Load Integer (32 bit signed)
6759 instruct loadI(iRegINoSp dst, memory4 mem)
6760 %{
6761 match(Set dst (LoadI mem));
6762 predicate(!needs_acquiring_load(n));
6763
6764 ins_cost(4 * INSN_COST);
6765 format %{ "ldrw $dst, $mem\t# int" %}
6766
6767 ins_encode(aarch64_enc_ldrw(dst, mem));
6768
6769 ins_pipe(iload_reg_mem);
6770 %}
6771
6772 // Load Integer (32 bit signed) into long
6773 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6774 %{
6775 match(Set dst (ConvI2L (LoadI mem)));
6776 predicate(!needs_acquiring_load(n->in(1)));
6777
6778 ins_cost(4 * INSN_COST);
6779 format %{ "ldrsw $dst, $mem\t# int" %}
6780
6781 ins_encode(aarch64_enc_ldrsw(dst, mem));
6782
6783 ins_pipe(iload_reg_mem);
6784 %}
6785
6786 // Load Integer (32 bit unsigned) into long
6787 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6788 %{
6789 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6790 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6791
6792 ins_cost(4 * INSN_COST);
6793 format %{ "ldrw $dst, $mem\t# int" %}
6794
6795 ins_encode(aarch64_enc_ldrw(dst, mem));
6796
6797 ins_pipe(iload_reg_mem);
6798 %}
6799
6800 // Load Long (64 bit signed)
6801 instruct loadL(iRegLNoSp dst, memory8 mem)
6802 %{
6803 match(Set dst (LoadL mem));
6804 predicate(!needs_acquiring_load(n));
6805
6806 ins_cost(4 * INSN_COST);
6807 format %{ "ldr $dst, $mem\t# int" %}
6808
6809 ins_encode(aarch64_enc_ldr(dst, mem));
6810
6811 ins_pipe(iload_reg_mem);
6812 %}
6813
6814 // Load Range
6815 instruct loadRange(iRegINoSp dst, memory4 mem)
6816 %{
6817 match(Set dst (LoadRange mem));
6818
6819 ins_cost(4 * INSN_COST);
6820 format %{ "ldrw $dst, $mem\t# range" %}
6821
6822 ins_encode(aarch64_enc_ldrw(dst, mem));
6823
6824 ins_pipe(iload_reg_mem);
6825 %}
6826
6827 // Load Pointer
6828 instruct loadP(iRegPNoSp dst, memory8 mem)
6829 %{
6830 match(Set dst (LoadP mem));
6831 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6832
6833 ins_cost(4 * INSN_COST);
6834 format %{ "ldr $dst, $mem\t# ptr" %}
6835
6836 ins_encode(aarch64_enc_ldr(dst, mem));
6837
6838 ins_pipe(iload_reg_mem);
6839 %}
6840
6841 // Load Compressed Pointer
6842 instruct loadN(iRegNNoSp dst, memory4 mem)
6843 %{
6844 match(Set dst (LoadN mem));
6845 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6846
6847 ins_cost(4 * INSN_COST);
6848 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6849
6850 ins_encode(aarch64_enc_ldrw(dst, mem));
6851
6852 ins_pipe(iload_reg_mem);
6853 %}
6854
6855 // Load Klass Pointer
6856 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6857 %{
6858 match(Set dst (LoadKlass mem));
6859 predicate(!needs_acquiring_load(n));
6860
6861 ins_cost(4 * INSN_COST);
6862 format %{ "ldr $dst, $mem\t# class" %}
6863
6864 ins_encode(aarch64_enc_ldr(dst, mem));
6865
6866 ins_pipe(iload_reg_mem);
6867 %}
6868
6869 // Load Narrow Klass Pointer
6870 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6871 %{
6872 match(Set dst (LoadNKlass mem));
6873 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6874
6875 ins_cost(4 * INSN_COST);
6876 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6877
6878 ins_encode(aarch64_enc_ldrw(dst, mem));
6879
6880 ins_pipe(iload_reg_mem);
6881 %}
6882
6883 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6884 %{
6885 match(Set dst (LoadNKlass mem));
6886 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6887
6888 ins_cost(4 * INSN_COST);
6889 format %{
6890 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6891 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6892 %}
6893 ins_encode %{
6894 // inlined aarch64_enc_ldrw
6895 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6897 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6898 %}
6899 ins_pipe(iload_reg_mem);
6900 %}
6901
6902 // Load Float
6903 instruct loadF(vRegF dst, memory4 mem)
6904 %{
6905 match(Set dst (LoadF mem));
6906 predicate(!needs_acquiring_load(n));
6907
6908 ins_cost(4 * INSN_COST);
6909 format %{ "ldrs $dst, $mem\t# float" %}
6910
6911 ins_encode( aarch64_enc_ldrs(dst, mem) );
6912
6913 ins_pipe(pipe_class_memory);
6914 %}
6915
6916 // Load Double
6917 instruct loadD(vRegD dst, memory8 mem)
6918 %{
6919 match(Set dst (LoadD mem));
6920 predicate(!needs_acquiring_load(n));
6921
6922 ins_cost(4 * INSN_COST);
6923 format %{ "ldrd $dst, $mem\t# double" %}
6924
6925 ins_encode( aarch64_enc_ldrd(dst, mem) );
6926
6927 ins_pipe(pipe_class_memory);
6928 %}
6929
6930
6931 // Load Int Constant
6932 instruct loadConI(iRegINoSp dst, immI src)
6933 %{
6934 match(Set dst src);
6935
6936 ins_cost(INSN_COST);
6937 format %{ "mov $dst, $src\t# int" %}
6938
6939 ins_encode( aarch64_enc_movw_imm(dst, src) );
6940
6941 ins_pipe(ialu_imm);
6942 %}
6943
6944 // Load Long Constant
6945 instruct loadConL(iRegLNoSp dst, immL src)
6946 %{
6947 match(Set dst src);
6948
6949 ins_cost(INSN_COST);
6950 format %{ "mov $dst, $src\t# long" %}
6951
6952 ins_encode( aarch64_enc_mov_imm(dst, src) );
6953
6954 ins_pipe(ialu_imm);
6955 %}
6956
6957 // Load Pointer Constant
6958
6959 instruct loadConP(iRegPNoSp dst, immP con)
6960 %{
6961 match(Set dst con);
6962
6963 ins_cost(INSN_COST * 4);
6964 format %{
6965 "mov $dst, $con\t# ptr\n\t"
6966 %}
6967
6968 ins_encode(aarch64_enc_mov_p(dst, con));
6969
6970 ins_pipe(ialu_imm);
6971 %}
6972
6973 // Load Null Pointer Constant
6974
6975 instruct loadConP0(iRegPNoSp dst, immP0 con)
6976 %{
6977 match(Set dst con);
6978
6979 ins_cost(INSN_COST);
6980 format %{ "mov $dst, $con\t# nullptr ptr" %}
6981
6982 ins_encode(aarch64_enc_mov_p0(dst, con));
6983
6984 ins_pipe(ialu_imm);
6985 %}
6986
6987 // Load Pointer Constant One
6988
6989 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6990 %{
6991 match(Set dst con);
6992
6993 ins_cost(INSN_COST);
6994 format %{ "mov $dst, $con\t# nullptr ptr" %}
6995
6996 ins_encode(aarch64_enc_mov_p1(dst, con));
6997
6998 ins_pipe(ialu_imm);
6999 %}
7000
7001 // Load Byte Map Base Constant
7002
7003 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7004 %{
7005 match(Set dst con);
7006
7007 ins_cost(INSN_COST);
7008 format %{ "adr $dst, $con\t# Byte Map Base" %}
7009
7010 ins_encode %{
7011 __ load_byte_map_base($dst$$Register);
7012 %}
7013
7014 ins_pipe(ialu_imm);
7015 %}
7016
7017 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
7018 %{
7019 match(Set dst con);
7020
7021 ins_cost(INSN_COST);
7022 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
7023
7024 ins_encode %{
7025 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
7026 %}
7027
7028 ins_pipe(ialu_imm);
7029 %}
7030
7031 // Load Narrow Pointer Constant
7032
7033 instruct loadConN(iRegNNoSp dst, immN con)
7034 %{
7035 match(Set dst con);
7036
7037 ins_cost(INSN_COST * 4);
7038 format %{ "mov $dst, $con\t# compressed ptr" %}
7039
7040 ins_encode(aarch64_enc_mov_n(dst, con));
7041
7042 ins_pipe(ialu_imm);
7043 %}
7044
7045 // Load Narrow Null Pointer Constant
7046
7047 instruct loadConN0(iRegNNoSp dst, immN0 con)
7048 %{
7049 match(Set dst con);
7050
7051 ins_cost(INSN_COST);
7052 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
7053
7054 ins_encode(aarch64_enc_mov_n0(dst, con));
7055
7056 ins_pipe(ialu_imm);
7057 %}
7058
7059 // Load Narrow Klass Constant
7060
7061 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7062 %{
7063 match(Set dst con);
7064
7065 ins_cost(INSN_COST);
7066 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7067
7068 ins_encode(aarch64_enc_mov_nk(dst, con));
7069
7070 ins_pipe(ialu_imm);
7071 %}
7072
7073 // Load Packed Float Constant
7074
7075 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7076 match(Set dst con);
7077 ins_cost(INSN_COST * 4);
7078 format %{ "fmovs $dst, $con"%}
7079 ins_encode %{
7080 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7081 %}
7082
7083 ins_pipe(fp_imm_s);
7084 %}
7085
7086 // Load Float Constant
7087
7088 instruct loadConF(vRegF dst, immF con) %{
7089 match(Set dst con);
7090
7091 ins_cost(INSN_COST * 4);
7092
7093 format %{
7094 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7095 %}
7096
7097 ins_encode %{
7098 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7099 %}
7100
7101 ins_pipe(fp_load_constant_s);
7102 %}
7103
7104 // Load Packed Double Constant
7105
7106 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7107 match(Set dst con);
7108 ins_cost(INSN_COST);
7109 format %{ "fmovd $dst, $con"%}
7110 ins_encode %{
7111 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7112 %}
7113
7114 ins_pipe(fp_imm_d);
7115 %}
7116
7117 // Load Double Constant
7118
7119 instruct loadConD(vRegD dst, immD con) %{
7120 match(Set dst con);
7121
7122 ins_cost(INSN_COST * 5);
7123 format %{
7124 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7125 %}
7126
7127 ins_encode %{
7128 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7129 %}
7130
7131 ins_pipe(fp_load_constant_d);
7132 %}
7133
7134 // Load Half Float Constant
7135 // The "ldr" instruction loads a 32-bit word from the constant pool into a
7136 // 32-bit register but only the bottom half will be populated and the top
7137 // 16 bits are zero.
7138 instruct loadConH(vRegF dst, immH con) %{
7139 match(Set dst con);
7140 format %{
7141 "ldrs $dst, [$constantaddress]\t# load from constant table: half float=$con\n\t"
7142 %}
7143 ins_encode %{
7144 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7145 %}
7146 ins_pipe(fp_load_constant_s);
7147 %}
7148
7149 // Store Instructions
7150
7151 // Store Byte
7152 instruct storeB(iRegIorL2I src, memory1 mem)
7153 %{
7154 match(Set mem (StoreB mem src));
7155 predicate(!needs_releasing_store(n));
7156
7157 ins_cost(INSN_COST);
7158 format %{ "strb $src, $mem\t# byte" %}
7159
7160 ins_encode(aarch64_enc_strb(src, mem));
7161
7162 ins_pipe(istore_reg_mem);
7163 %}
7164
7165
7166 instruct storeimmB0(immI0 zero, memory1 mem)
7167 %{
7168 match(Set mem (StoreB mem zero));
7169 predicate(!needs_releasing_store(n));
7170
7171 ins_cost(INSN_COST);
7172 format %{ "strb rscractch2, $mem\t# byte" %}
7173
7174 ins_encode(aarch64_enc_strb0(mem));
7175
7176 ins_pipe(istore_mem);
7177 %}
7178
7179 // Store Char/Short
7180 instruct storeC(iRegIorL2I src, memory2 mem)
7181 %{
7182 match(Set mem (StoreC mem src));
7183 predicate(!needs_releasing_store(n));
7184
7185 ins_cost(INSN_COST);
7186 format %{ "strh $src, $mem\t# short" %}
7187
7188 ins_encode(aarch64_enc_strh(src, mem));
7189
7190 ins_pipe(istore_reg_mem);
7191 %}
7192
7193 instruct storeimmC0(immI0 zero, memory2 mem)
7194 %{
7195 match(Set mem (StoreC mem zero));
7196 predicate(!needs_releasing_store(n));
7197
7198 ins_cost(INSN_COST);
7199 format %{ "strh zr, $mem\t# short" %}
7200
7201 ins_encode(aarch64_enc_strh0(mem));
7202
7203 ins_pipe(istore_mem);
7204 %}
7205
7206 // Store Integer
7207
7208 instruct storeI(iRegIorL2I src, memory4 mem)
7209 %{
7210 match(Set mem(StoreI mem src));
7211 predicate(!needs_releasing_store(n));
7212
7213 ins_cost(INSN_COST);
7214 format %{ "strw $src, $mem\t# int" %}
7215
7216 ins_encode(aarch64_enc_strw(src, mem));
7217
7218 ins_pipe(istore_reg_mem);
7219 %}
7220
7221 instruct storeimmI0(immI0 zero, memory4 mem)
7222 %{
7223 match(Set mem(StoreI mem zero));
7224 predicate(!needs_releasing_store(n));
7225
7226 ins_cost(INSN_COST);
7227 format %{ "strw zr, $mem\t# int" %}
7228
7229 ins_encode(aarch64_enc_strw0(mem));
7230
7231 ins_pipe(istore_mem);
7232 %}
7233
7234 // Store Long (64 bit signed)
7235 instruct storeL(iRegL src, memory8 mem)
7236 %{
7237 match(Set mem (StoreL mem src));
7238 predicate(!needs_releasing_store(n));
7239
7240 ins_cost(INSN_COST);
7241 format %{ "str $src, $mem\t# int" %}
7242
7243 ins_encode(aarch64_enc_str(src, mem));
7244
7245 ins_pipe(istore_reg_mem);
7246 %}
7247
7248 // Store Long (64 bit signed)
7249 instruct storeimmL0(immL0 zero, memory8 mem)
7250 %{
7251 match(Set mem (StoreL mem zero));
7252 predicate(!needs_releasing_store(n));
7253
7254 ins_cost(INSN_COST);
7255 format %{ "str zr, $mem\t# int" %}
7256
7257 ins_encode(aarch64_enc_str0(mem));
7258
7259 ins_pipe(istore_mem);
7260 %}
7261
7262 // Store Pointer
7263 instruct storeP(iRegP src, memory8 mem)
7264 %{
7265 match(Set mem (StoreP mem src));
7266 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7267
7268 ins_cost(INSN_COST);
7269 format %{ "str $src, $mem\t# ptr" %}
7270
7271 ins_encode(aarch64_enc_str(src, mem));
7272
7273 ins_pipe(istore_reg_mem);
7274 %}
7275
7276 // Store Pointer
7277 instruct storeimmP0(immP0 zero, memory8 mem)
7278 %{
7279 match(Set mem (StoreP mem zero));
7280 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7281
7282 ins_cost(INSN_COST);
7283 format %{ "str zr, $mem\t# ptr" %}
7284
7285 ins_encode(aarch64_enc_str0(mem));
7286
7287 ins_pipe(istore_mem);
7288 %}
7289
7290 // Store Compressed Pointer
7291 instruct storeN(iRegN src, memory4 mem)
7292 %{
7293 match(Set mem (StoreN mem src));
7294 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7295
7296 ins_cost(INSN_COST);
7297 format %{ "strw $src, $mem\t# compressed ptr" %}
7298
7299 ins_encode(aarch64_enc_strw(src, mem));
7300
7301 ins_pipe(istore_reg_mem);
7302 %}
7303
7304 instruct storeImmN0(immN0 zero, memory4 mem)
7305 %{
7306 match(Set mem (StoreN mem zero));
7307 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7308
7309 ins_cost(INSN_COST);
7310 format %{ "strw zr, $mem\t# compressed ptr" %}
7311
7312 ins_encode(aarch64_enc_strw0(mem));
7313
7314 ins_pipe(istore_mem);
7315 %}
7316
7317 // Store Float
7318 instruct storeF(vRegF src, memory4 mem)
7319 %{
7320 match(Set mem (StoreF mem src));
7321 predicate(!needs_releasing_store(n));
7322
7323 ins_cost(INSN_COST);
7324 format %{ "strs $src, $mem\t# float" %}
7325
7326 ins_encode( aarch64_enc_strs(src, mem) );
7327
7328 ins_pipe(pipe_class_memory);
7329 %}
7330
7331 // TODO
7332 // implement storeImmF0 and storeFImmPacked
7333
7334 // Store Double
7335 instruct storeD(vRegD src, memory8 mem)
7336 %{
7337 match(Set mem (StoreD mem src));
7338 predicate(!needs_releasing_store(n));
7339
7340 ins_cost(INSN_COST);
7341 format %{ "strd $src, $mem\t# double" %}
7342
7343 ins_encode( aarch64_enc_strd(src, mem) );
7344
7345 ins_pipe(pipe_class_memory);
7346 %}
7347
7348 // Store Compressed Klass Pointer
7349 instruct storeNKlass(iRegN src, memory4 mem)
7350 %{
7351 predicate(!needs_releasing_store(n));
7352 match(Set mem (StoreNKlass mem src));
7353
7354 ins_cost(INSN_COST);
7355 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7356
7357 ins_encode(aarch64_enc_strw(src, mem));
7358
7359 ins_pipe(istore_reg_mem);
7360 %}
7361
7362 // TODO
7363 // implement storeImmD0 and storeDImmPacked
7364
7365 // prefetch instructions
7366 // Must be safe to execute with invalid address (cannot fault).
7367
7368 instruct prefetchalloc( memory8 mem ) %{
7369 match(PrefetchAllocation mem);
7370
7371 ins_cost(INSN_COST);
7372 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7373
7374 ins_encode( aarch64_enc_prefetchw(mem) );
7375
7376 ins_pipe(iload_prefetch);
7377 %}
7378
7379 // ---------------- volatile loads and stores ----------------
7380
7381 // Load Byte (8 bit signed)
7382 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7383 %{
7384 match(Set dst (LoadB mem));
7385
7386 ins_cost(VOLATILE_REF_COST);
7387 format %{ "ldarsb $dst, $mem\t# byte" %}
7388
7389 ins_encode(aarch64_enc_ldarsb(dst, mem));
7390
7391 ins_pipe(pipe_serial);
7392 %}
7393
7394 // Load Byte (8 bit signed) into long
7395 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7396 %{
7397 match(Set dst (ConvI2L (LoadB mem)));
7398
7399 ins_cost(VOLATILE_REF_COST);
7400 format %{ "ldarsb $dst, $mem\t# byte" %}
7401
7402 ins_encode(aarch64_enc_ldarsb(dst, mem));
7403
7404 ins_pipe(pipe_serial);
7405 %}
7406
7407 // Load Byte (8 bit unsigned)
7408 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7409 %{
7410 match(Set dst (LoadUB mem));
7411
7412 ins_cost(VOLATILE_REF_COST);
7413 format %{ "ldarb $dst, $mem\t# byte" %}
7414
7415 ins_encode(aarch64_enc_ldarb(dst, mem));
7416
7417 ins_pipe(pipe_serial);
7418 %}
7419
7420 // Load Byte (8 bit unsigned) into long
7421 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7422 %{
7423 match(Set dst (ConvI2L (LoadUB mem)));
7424
7425 ins_cost(VOLATILE_REF_COST);
7426 format %{ "ldarb $dst, $mem\t# byte" %}
7427
7428 ins_encode(aarch64_enc_ldarb(dst, mem));
7429
7430 ins_pipe(pipe_serial);
7431 %}
7432
7433 // Load Short (16 bit signed)
7434 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7435 %{
7436 match(Set dst (LoadS mem));
7437
7438 ins_cost(VOLATILE_REF_COST);
7439 format %{ "ldarshw $dst, $mem\t# short" %}
7440
7441 ins_encode(aarch64_enc_ldarshw(dst, mem));
7442
7443 ins_pipe(pipe_serial);
7444 %}
7445
7446 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7447 %{
7448 match(Set dst (LoadUS mem));
7449
7450 ins_cost(VOLATILE_REF_COST);
7451 format %{ "ldarhw $dst, $mem\t# short" %}
7452
7453 ins_encode(aarch64_enc_ldarhw(dst, mem));
7454
7455 ins_pipe(pipe_serial);
7456 %}
7457
7458 // Load Short/Char (16 bit unsigned) into long
7459 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7460 %{
7461 match(Set dst (ConvI2L (LoadUS mem)));
7462
7463 ins_cost(VOLATILE_REF_COST);
7464 format %{ "ldarh $dst, $mem\t# short" %}
7465
7466 ins_encode(aarch64_enc_ldarh(dst, mem));
7467
7468 ins_pipe(pipe_serial);
7469 %}
7470
7471 // Load Short/Char (16 bit signed) into long
7472 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7473 %{
7474 match(Set dst (ConvI2L (LoadS mem)));
7475
7476 ins_cost(VOLATILE_REF_COST);
7477 format %{ "ldarh $dst, $mem\t# short" %}
7478
7479 ins_encode(aarch64_enc_ldarsh(dst, mem));
7480
7481 ins_pipe(pipe_serial);
7482 %}
7483
7484 // Load Integer (32 bit signed)
7485 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7486 %{
7487 match(Set dst (LoadI mem));
7488
7489 ins_cost(VOLATILE_REF_COST);
7490 format %{ "ldarw $dst, $mem\t# int" %}
7491
7492 ins_encode(aarch64_enc_ldarw(dst, mem));
7493
7494 ins_pipe(pipe_serial);
7495 %}
7496
7497 // Load Integer (32 bit unsigned) into long
7498 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7499 %{
7500 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7501
7502 ins_cost(VOLATILE_REF_COST);
7503 format %{ "ldarw $dst, $mem\t# int" %}
7504
7505 ins_encode(aarch64_enc_ldarw(dst, mem));
7506
7507 ins_pipe(pipe_serial);
7508 %}
7509
7510 // Load Long (64 bit signed)
7511 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7512 %{
7513 match(Set dst (LoadL mem));
7514
7515 ins_cost(VOLATILE_REF_COST);
7516 format %{ "ldar $dst, $mem\t# int" %}
7517
7518 ins_encode(aarch64_enc_ldar(dst, mem));
7519
7520 ins_pipe(pipe_serial);
7521 %}
7522
7523 // Load Pointer
7524 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7525 %{
7526 match(Set dst (LoadP mem));
7527 predicate(n->as_Load()->barrier_data() == 0);
7528
7529 ins_cost(VOLATILE_REF_COST);
7530 format %{ "ldar $dst, $mem\t# ptr" %}
7531
7532 ins_encode(aarch64_enc_ldar(dst, mem));
7533
7534 ins_pipe(pipe_serial);
7535 %}
7536
7537 // Load Compressed Pointer
7538 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7539 %{
7540 match(Set dst (LoadN mem));
7541 predicate(n->as_Load()->barrier_data() == 0);
7542
7543 ins_cost(VOLATILE_REF_COST);
7544 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7545
7546 ins_encode(aarch64_enc_ldarw(dst, mem));
7547
7548 ins_pipe(pipe_serial);
7549 %}
7550
7551 // Load Float
7552 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7553 %{
7554 match(Set dst (LoadF mem));
7555
7556 ins_cost(VOLATILE_REF_COST);
7557 format %{ "ldars $dst, $mem\t# float" %}
7558
7559 ins_encode( aarch64_enc_fldars(dst, mem) );
7560
7561 ins_pipe(pipe_serial);
7562 %}
7563
7564 // Load Double
7565 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7566 %{
7567 match(Set dst (LoadD mem));
7568
7569 ins_cost(VOLATILE_REF_COST);
7570 format %{ "ldard $dst, $mem\t# double" %}
7571
7572 ins_encode( aarch64_enc_fldard(dst, mem) );
7573
7574 ins_pipe(pipe_serial);
7575 %}
7576
7577 // Store Byte
7578 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7579 %{
7580 match(Set mem (StoreB mem src));
7581
7582 ins_cost(VOLATILE_REF_COST);
7583 format %{ "stlrb $src, $mem\t# byte" %}
7584
7585 ins_encode(aarch64_enc_stlrb(src, mem));
7586
7587 ins_pipe(pipe_class_memory);
7588 %}
7589
7590 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7591 %{
7592 match(Set mem (StoreB mem zero));
7593
7594 ins_cost(VOLATILE_REF_COST);
7595 format %{ "stlrb zr, $mem\t# byte" %}
7596
7597 ins_encode(aarch64_enc_stlrb0(mem));
7598
7599 ins_pipe(pipe_class_memory);
7600 %}
7601
7602 // Store Char/Short
7603 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7604 %{
7605 match(Set mem (StoreC mem src));
7606
7607 ins_cost(VOLATILE_REF_COST);
7608 format %{ "stlrh $src, $mem\t# short" %}
7609
7610 ins_encode(aarch64_enc_stlrh(src, mem));
7611
7612 ins_pipe(pipe_class_memory);
7613 %}
7614
7615 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7616 %{
7617 match(Set mem (StoreC mem zero));
7618
7619 ins_cost(VOLATILE_REF_COST);
7620 format %{ "stlrh zr, $mem\t# short" %}
7621
7622 ins_encode(aarch64_enc_stlrh0(mem));
7623
7624 ins_pipe(pipe_class_memory);
7625 %}
7626
7627 // Store Integer
7628
7629 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7630 %{
7631 match(Set mem(StoreI mem src));
7632
7633 ins_cost(VOLATILE_REF_COST);
7634 format %{ "stlrw $src, $mem\t# int" %}
7635
7636 ins_encode(aarch64_enc_stlrw(src, mem));
7637
7638 ins_pipe(pipe_class_memory);
7639 %}
7640
7641 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7642 %{
7643 match(Set mem(StoreI mem zero));
7644
7645 ins_cost(VOLATILE_REF_COST);
7646 format %{ "stlrw zr, $mem\t# int" %}
7647
7648 ins_encode(aarch64_enc_stlrw0(mem));
7649
7650 ins_pipe(pipe_class_memory);
7651 %}
7652
7653 // Store Long (64 bit signed)
7654 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7655 %{
7656 match(Set mem (StoreL mem src));
7657
7658 ins_cost(VOLATILE_REF_COST);
7659 format %{ "stlr $src, $mem\t# int" %}
7660
7661 ins_encode(aarch64_enc_stlr(src, mem));
7662
7663 ins_pipe(pipe_class_memory);
7664 %}
7665
7666 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7667 %{
7668 match(Set mem (StoreL mem zero));
7669
7670 ins_cost(VOLATILE_REF_COST);
7671 format %{ "stlr zr, $mem\t# int" %}
7672
7673 ins_encode(aarch64_enc_stlr0(mem));
7674
7675 ins_pipe(pipe_class_memory);
7676 %}
7677
7678 // Store Pointer
7679 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7680 %{
7681 match(Set mem (StoreP mem src));
7682 predicate(n->as_Store()->barrier_data() == 0);
7683
7684 ins_cost(VOLATILE_REF_COST);
7685 format %{ "stlr $src, $mem\t# ptr" %}
7686
7687 ins_encode(aarch64_enc_stlr(src, mem));
7688
7689 ins_pipe(pipe_class_memory);
7690 %}
7691
7692 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7693 %{
7694 match(Set mem (StoreP mem zero));
7695 predicate(n->as_Store()->barrier_data() == 0);
7696
7697 ins_cost(VOLATILE_REF_COST);
7698 format %{ "stlr zr, $mem\t# ptr" %}
7699
7700 ins_encode(aarch64_enc_stlr0(mem));
7701
7702 ins_pipe(pipe_class_memory);
7703 %}
7704
7705 // Store Compressed Pointer
7706 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7707 %{
7708 match(Set mem (StoreN mem src));
7709 predicate(n->as_Store()->barrier_data() == 0);
7710
7711 ins_cost(VOLATILE_REF_COST);
7712 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7713
7714 ins_encode(aarch64_enc_stlrw(src, mem));
7715
7716 ins_pipe(pipe_class_memory);
7717 %}
7718
7719 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7720 %{
7721 match(Set mem (StoreN mem zero));
7722 predicate(n->as_Store()->barrier_data() == 0);
7723
7724 ins_cost(VOLATILE_REF_COST);
7725 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7726
7727 ins_encode(aarch64_enc_stlrw0(mem));
7728
7729 ins_pipe(pipe_class_memory);
7730 %}
7731
7732 // Store Float
7733 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7734 %{
7735 match(Set mem (StoreF mem src));
7736
7737 ins_cost(VOLATILE_REF_COST);
7738 format %{ "stlrs $src, $mem\t# float" %}
7739
7740 ins_encode( aarch64_enc_fstlrs(src, mem) );
7741
7742 ins_pipe(pipe_class_memory);
7743 %}
7744
7745 // TODO
7746 // implement storeImmF0 and storeFImmPacked
7747
7748 // Store Double
7749 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7750 %{
7751 match(Set mem (StoreD mem src));
7752
7753 ins_cost(VOLATILE_REF_COST);
7754 format %{ "stlrd $src, $mem\t# double" %}
7755
7756 ins_encode( aarch64_enc_fstlrd(src, mem) );
7757
7758 ins_pipe(pipe_class_memory);
7759 %}
7760
7761 // ---------------- end of volatile loads and stores ----------------
7762
7763 instruct cacheWB(indirect addr)
7764 %{
7765 predicate(VM_Version::supports_data_cache_line_flush());
7766 match(CacheWB addr);
7767
7768 ins_cost(100);
7769 format %{"cache wb $addr" %}
7770 ins_encode %{
7771 assert($addr->index_position() < 0, "should be");
7772 assert($addr$$disp == 0, "should be");
7773 __ cache_wb(Address($addr$$base$$Register, 0));
7774 %}
7775 ins_pipe(pipe_slow); // XXX
7776 %}
7777
7778 instruct cacheWBPreSync()
7779 %{
7780 predicate(VM_Version::supports_data_cache_line_flush());
7781 match(CacheWBPreSync);
7782
7783 ins_cost(100);
7784 format %{"cache wb presync" %}
7785 ins_encode %{
7786 __ cache_wbsync(true);
7787 %}
7788 ins_pipe(pipe_slow); // XXX
7789 %}
7790
7791 instruct cacheWBPostSync()
7792 %{
7793 predicate(VM_Version::supports_data_cache_line_flush());
7794 match(CacheWBPostSync);
7795
7796 ins_cost(100);
7797 format %{"cache wb postsync" %}
7798 ins_encode %{
7799 __ cache_wbsync(false);
7800 %}
7801 ins_pipe(pipe_slow); // XXX
7802 %}
7803
7804 // ============================================================================
7805 // BSWAP Instructions
7806
7807 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7808 match(Set dst (ReverseBytesI src));
7809
7810 ins_cost(INSN_COST);
7811 format %{ "revw $dst, $src" %}
7812
7813 ins_encode %{
7814 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7815 %}
7816
7817 ins_pipe(ialu_reg);
7818 %}
7819
7820 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7821 match(Set dst (ReverseBytesL src));
7822
7823 ins_cost(INSN_COST);
7824 format %{ "rev $dst, $src" %}
7825
7826 ins_encode %{
7827 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7828 %}
7829
7830 ins_pipe(ialu_reg);
7831 %}
7832
7833 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7834 match(Set dst (ReverseBytesUS src));
7835
7836 ins_cost(INSN_COST);
7837 format %{ "rev16w $dst, $src" %}
7838
7839 ins_encode %{
7840 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7841 %}
7842
7843 ins_pipe(ialu_reg);
7844 %}
7845
7846 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7847 match(Set dst (ReverseBytesS src));
7848
7849 ins_cost(INSN_COST);
7850 format %{ "rev16w $dst, $src\n\t"
7851 "sbfmw $dst, $dst, #0, #15" %}
7852
7853 ins_encode %{
7854 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7855 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7856 %}
7857
7858 ins_pipe(ialu_reg);
7859 %}
7860
7861 // ============================================================================
7862 // Zero Count Instructions
7863
7864 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7865 match(Set dst (CountLeadingZerosI src));
7866
7867 ins_cost(INSN_COST);
7868 format %{ "clzw $dst, $src" %}
7869 ins_encode %{
7870 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7871 %}
7872
7873 ins_pipe(ialu_reg);
7874 %}
7875
7876 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7877 match(Set dst (CountLeadingZerosL src));
7878
7879 ins_cost(INSN_COST);
7880 format %{ "clz $dst, $src" %}
7881 ins_encode %{
7882 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7883 %}
7884
7885 ins_pipe(ialu_reg);
7886 %}
7887
7888 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7889 match(Set dst (CountTrailingZerosI src));
7890
7891 ins_cost(INSN_COST * 2);
7892 format %{ "rbitw $dst, $src\n\t"
7893 "clzw $dst, $dst" %}
7894 ins_encode %{
7895 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7896 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7897 %}
7898
7899 ins_pipe(ialu_reg);
7900 %}
7901
7902 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7903 match(Set dst (CountTrailingZerosL src));
7904
7905 ins_cost(INSN_COST * 2);
7906 format %{ "rbit $dst, $src\n\t"
7907 "clz $dst, $dst" %}
7908 ins_encode %{
7909 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7910 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7911 %}
7912
7913 ins_pipe(ialu_reg);
7914 %}
7915
7916 //---------- Population Count Instructions -------------------------------------
7917 //
7918
7919 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7920 match(Set dst (PopCountI src));
7921 effect(TEMP tmp);
7922 ins_cost(INSN_COST * 13);
7923
7924 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7925 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7926 "addv $tmp, $tmp\t# vector (8B)\n\t"
7927 "mov $dst, $tmp\t# vector (1D)" %}
7928 ins_encode %{
7929 __ fmovs($tmp$$FloatRegister, $src$$Register);
7930 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7931 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7932 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7933 %}
7934
7935 ins_pipe(pipe_class_default);
7936 %}
7937
7938 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7939 match(Set dst (PopCountI (LoadI mem)));
7940 effect(TEMP tmp);
7941 ins_cost(INSN_COST * 13);
7942
7943 format %{ "ldrs $tmp, $mem\n\t"
7944 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7945 "addv $tmp, $tmp\t# vector (8B)\n\t"
7946 "mov $dst, $tmp\t# vector (1D)" %}
7947 ins_encode %{
7948 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7949 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7950 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7951 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7952 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7953 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7954 %}
7955
7956 ins_pipe(pipe_class_default);
7957 %}
7958
7959 // Note: Long.bitCount(long) returns an int.
7960 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7961 match(Set dst (PopCountL src));
7962 effect(TEMP tmp);
7963 ins_cost(INSN_COST * 13);
7964
7965 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7966 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7967 "addv $tmp, $tmp\t# vector (8B)\n\t"
7968 "mov $dst, $tmp\t# vector (1D)" %}
7969 ins_encode %{
7970 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7971 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7972 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7973 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7974 %}
7975
7976 ins_pipe(pipe_class_default);
7977 %}
7978
7979 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7980 match(Set dst (PopCountL (LoadL mem)));
7981 effect(TEMP tmp);
7982 ins_cost(INSN_COST * 13);
7983
7984 format %{ "ldrd $tmp, $mem\n\t"
7985 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7986 "addv $tmp, $tmp\t# vector (8B)\n\t"
7987 "mov $dst, $tmp\t# vector (1D)" %}
7988 ins_encode %{
7989 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7990 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7991 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7992 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7993 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7994 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7995 %}
7996
7997 ins_pipe(pipe_class_default);
7998 %}
7999
8000 // ============================================================================
8001 // VerifyVectorAlignment Instruction
8002
8003 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
8004 match(Set addr (VerifyVectorAlignment addr mask));
8005 effect(KILL cr);
8006 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
8007 ins_encode %{
8008 Label Lskip;
8009 // check if masked bits of addr are zero
8010 __ tst($addr$$Register, $mask$$constant);
8011 __ br(Assembler::EQ, Lskip);
8012 __ stop("verify_vector_alignment found a misaligned vector memory access");
8013 __ bind(Lskip);
8014 %}
8015 ins_pipe(pipe_slow);
8016 %}
8017
8018 // ============================================================================
8019 // MemBar Instruction
8020
8021 instruct load_fence() %{
8022 match(LoadFence);
8023 ins_cost(VOLATILE_REF_COST);
8024
8025 format %{ "load_fence" %}
8026
8027 ins_encode %{
8028 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8029 %}
8030 ins_pipe(pipe_serial);
8031 %}
8032
8033 instruct unnecessary_membar_acquire() %{
8034 predicate(unnecessary_acquire(n));
8035 match(MemBarAcquire);
8036 ins_cost(0);
8037
8038 format %{ "membar_acquire (elided)" %}
8039
8040 ins_encode %{
8041 __ block_comment("membar_acquire (elided)");
8042 %}
8043
8044 ins_pipe(pipe_class_empty);
8045 %}
8046
8047 instruct membar_acquire() %{
8048 match(MemBarAcquire);
8049 ins_cost(VOLATILE_REF_COST);
8050
8051 format %{ "membar_acquire\n\t"
8052 "dmb ishld" %}
8053
8054 ins_encode %{
8055 __ block_comment("membar_acquire");
8056 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8057 %}
8058
8059 ins_pipe(pipe_serial);
8060 %}
8061
8062
8063 instruct membar_acquire_lock() %{
8064 match(MemBarAcquireLock);
8065 ins_cost(VOLATILE_REF_COST);
8066
8067 format %{ "membar_acquire_lock (elided)" %}
8068
8069 ins_encode %{
8070 __ block_comment("membar_acquire_lock (elided)");
8071 %}
8072
8073 ins_pipe(pipe_serial);
8074 %}
8075
8076 instruct store_fence() %{
8077 match(StoreFence);
8078 ins_cost(VOLATILE_REF_COST);
8079
8080 format %{ "store_fence" %}
8081
8082 ins_encode %{
8083 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8084 %}
8085 ins_pipe(pipe_serial);
8086 %}
8087
8088 instruct unnecessary_membar_release() %{
8089 predicate(unnecessary_release(n));
8090 match(MemBarRelease);
8091 ins_cost(0);
8092
8093 format %{ "membar_release (elided)" %}
8094
8095 ins_encode %{
8096 __ block_comment("membar_release (elided)");
8097 %}
8098 ins_pipe(pipe_serial);
8099 %}
8100
8101 instruct membar_release() %{
8102 match(MemBarRelease);
8103 ins_cost(VOLATILE_REF_COST);
8104
8105 format %{ "membar_release\n\t"
8106 "dmb ishst\n\tdmb ishld" %}
8107
8108 ins_encode %{
8109 __ block_comment("membar_release");
8110 // These will be merged if AlwaysMergeDMB is enabled.
8111 __ membar(Assembler::StoreStore);
8112 __ membar(Assembler::LoadStore);
8113 %}
8114 ins_pipe(pipe_serial);
8115 %}
8116
8117 instruct membar_storestore() %{
8118 match(MemBarStoreStore);
8119 match(StoreStoreFence);
8120 ins_cost(VOLATILE_REF_COST);
8121
8122 format %{ "MEMBAR-store-store" %}
8123
8124 ins_encode %{
8125 __ membar(Assembler::StoreStore);
8126 %}
8127 ins_pipe(pipe_serial);
8128 %}
8129
8130 instruct membar_release_lock() %{
8131 match(MemBarReleaseLock);
8132 ins_cost(VOLATILE_REF_COST);
8133
8134 format %{ "membar_release_lock (elided)" %}
8135
8136 ins_encode %{
8137 __ block_comment("membar_release_lock (elided)");
8138 %}
8139
8140 ins_pipe(pipe_serial);
8141 %}
8142
8143 instruct unnecessary_membar_volatile() %{
8144 predicate(unnecessary_volatile(n));
8145 match(MemBarVolatile);
8146 ins_cost(0);
8147
8148 format %{ "membar_volatile (elided)" %}
8149
8150 ins_encode %{
8151 __ block_comment("membar_volatile (elided)");
8152 %}
8153
8154 ins_pipe(pipe_serial);
8155 %}
8156
8157 instruct membar_volatile() %{
8158 match(MemBarVolatile);
8159 ins_cost(VOLATILE_REF_COST*100);
8160
8161 format %{ "membar_volatile\n\t"
8162 "dmb ish"%}
8163
8164 ins_encode %{
8165 __ block_comment("membar_volatile");
8166 __ membar(Assembler::StoreLoad);
8167 %}
8168
8169 ins_pipe(pipe_serial);
8170 %}
8171
8172 // ============================================================================
8173 // Cast/Convert Instructions
8174
8175 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8176 match(Set dst (CastX2P src));
8177
8178 ins_cost(INSN_COST);
8179 format %{ "mov $dst, $src\t# long -> ptr" %}
8180
8181 ins_encode %{
8182 if ($dst$$reg != $src$$reg) {
8183 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8184 }
8185 %}
8186
8187 ins_pipe(ialu_reg);
8188 %}
8189
8190 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8191 match(Set dst (CastP2X src));
8192
8193 ins_cost(INSN_COST);
8194 format %{ "mov $dst, $src\t# ptr -> long" %}
8195
8196 ins_encode %{
8197 if ($dst$$reg != $src$$reg) {
8198 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8199 }
8200 %}
8201
8202 ins_pipe(ialu_reg);
8203 %}
8204
8205 // Convert oop into int for vectors alignment masking
8206 instruct convP2I(iRegINoSp dst, iRegP src) %{
8207 match(Set dst (ConvL2I (CastP2X src)));
8208
8209 ins_cost(INSN_COST);
8210 format %{ "movw $dst, $src\t# ptr -> int" %}
8211 ins_encode %{
8212 __ movw($dst$$Register, $src$$Register);
8213 %}
8214
8215 ins_pipe(ialu_reg);
8216 %}
8217
8218 // Convert compressed oop into int for vectors alignment masking
8219 // in case of 32bit oops (heap < 4Gb).
8220 instruct convN2I(iRegINoSp dst, iRegN src)
8221 %{
8222 predicate(CompressedOops::shift() == 0);
8223 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8224
8225 ins_cost(INSN_COST);
8226 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8227 ins_encode %{
8228 __ movw($dst$$Register, $src$$Register);
8229 %}
8230
8231 ins_pipe(ialu_reg);
8232 %}
8233
8234
8235 // Convert oop pointer into compressed form
8236 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8237 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8238 match(Set dst (EncodeP src));
8239 effect(KILL cr);
8240 ins_cost(INSN_COST * 3);
8241 format %{ "encode_heap_oop $dst, $src" %}
8242 ins_encode %{
8243 Register s = $src$$Register;
8244 Register d = $dst$$Register;
8245 __ encode_heap_oop(d, s);
8246 %}
8247 ins_pipe(ialu_reg);
8248 %}
8249
8250 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8251 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8252 match(Set dst (EncodeP src));
8253 ins_cost(INSN_COST * 3);
8254 format %{ "encode_heap_oop_not_null $dst, $src" %}
8255 ins_encode %{
8256 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8257 %}
8258 ins_pipe(ialu_reg);
8259 %}
8260
8261 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8262 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8263 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8264 match(Set dst (DecodeN src));
8265 ins_cost(INSN_COST * 3);
8266 format %{ "decode_heap_oop $dst, $src" %}
8267 ins_encode %{
8268 Register s = $src$$Register;
8269 Register d = $dst$$Register;
8270 __ decode_heap_oop(d, s);
8271 %}
8272 ins_pipe(ialu_reg);
8273 %}
8274
8275 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8276 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8277 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8278 match(Set dst (DecodeN src));
8279 ins_cost(INSN_COST * 3);
8280 format %{ "decode_heap_oop_not_null $dst, $src" %}
8281 ins_encode %{
8282 Register s = $src$$Register;
8283 Register d = $dst$$Register;
8284 __ decode_heap_oop_not_null(d, s);
8285 %}
8286 ins_pipe(ialu_reg);
8287 %}
8288
8289 // n.b. AArch64 implementations of encode_klass_not_null and
8290 // decode_klass_not_null do not modify the flags register so, unlike
8291 // Intel, we don't kill CR as a side effect here
8292
8293 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8294 match(Set dst (EncodePKlass src));
8295
8296 ins_cost(INSN_COST * 3);
8297 format %{ "encode_klass_not_null $dst,$src" %}
8298
8299 ins_encode %{
8300 Register src_reg = as_Register($src$$reg);
8301 Register dst_reg = as_Register($dst$$reg);
8302 __ encode_klass_not_null(dst_reg, src_reg);
8303 %}
8304
8305 ins_pipe(ialu_reg);
8306 %}
8307
8308 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8309 match(Set dst (DecodeNKlass src));
8310
8311 ins_cost(INSN_COST * 3);
8312 format %{ "decode_klass_not_null $dst,$src" %}
8313
8314 ins_encode %{
8315 Register src_reg = as_Register($src$$reg);
8316 Register dst_reg = as_Register($dst$$reg);
8317 if (dst_reg != src_reg) {
8318 __ decode_klass_not_null(dst_reg, src_reg);
8319 } else {
8320 __ decode_klass_not_null(dst_reg);
8321 }
8322 %}
8323
8324 ins_pipe(ialu_reg);
8325 %}
8326
8327 instruct checkCastPP(iRegPNoSp dst)
8328 %{
8329 match(Set dst (CheckCastPP dst));
8330
8331 size(0);
8332 format %{ "# checkcastPP of $dst" %}
8333 ins_encode(/* empty encoding */);
8334 ins_pipe(pipe_class_empty);
8335 %}
8336
8337 instruct castPP(iRegPNoSp dst)
8338 %{
8339 match(Set dst (CastPP dst));
8340
8341 size(0);
8342 format %{ "# castPP of $dst" %}
8343 ins_encode(/* empty encoding */);
8344 ins_pipe(pipe_class_empty);
8345 %}
8346
8347 instruct castII(iRegI dst)
8348 %{
8349 predicate(VerifyConstraintCasts == 0);
8350 match(Set dst (CastII dst));
8351
8352 size(0);
8353 format %{ "# castII of $dst" %}
8354 ins_encode(/* empty encoding */);
8355 ins_cost(0);
8356 ins_pipe(pipe_class_empty);
8357 %}
8358
8359 instruct castII_checked(iRegI dst, rFlagsReg cr)
8360 %{
8361 predicate(VerifyConstraintCasts > 0);
8362 match(Set dst (CastII dst));
8363 effect(KILL cr);
8364
8365 format %{ "# castII_checked of $dst" %}
8366 ins_encode %{
8367 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8368 %}
8369 ins_pipe(pipe_slow);
8370 %}
8371
8372 instruct castLL(iRegL dst)
8373 %{
8374 predicate(VerifyConstraintCasts == 0);
8375 match(Set dst (CastLL dst));
8376
8377 size(0);
8378 format %{ "# castLL of $dst" %}
8379 ins_encode(/* empty encoding */);
8380 ins_cost(0);
8381 ins_pipe(pipe_class_empty);
8382 %}
8383
8384 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8385 %{
8386 predicate(VerifyConstraintCasts > 0);
8387 match(Set dst (CastLL dst));
8388 effect(KILL cr);
8389
8390 format %{ "# castLL_checked of $dst" %}
8391 ins_encode %{
8392 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8393 %}
8394 ins_pipe(pipe_slow);
8395 %}
8396
8397 instruct castHH(vRegF dst)
8398 %{
8399 match(Set dst (CastHH dst));
8400 size(0);
8401 format %{ "# castHH of $dst" %}
8402 ins_encode(/* empty encoding */);
8403 ins_cost(0);
8404 ins_pipe(pipe_class_empty);
8405 %}
8406
8407 instruct castFF(vRegF dst)
8408 %{
8409 match(Set dst (CastFF dst));
8410
8411 size(0);
8412 format %{ "# castFF of $dst" %}
8413 ins_encode(/* empty encoding */);
8414 ins_cost(0);
8415 ins_pipe(pipe_class_empty);
8416 %}
8417
8418 instruct castDD(vRegD dst)
8419 %{
8420 match(Set dst (CastDD dst));
8421
8422 size(0);
8423 format %{ "# castDD of $dst" %}
8424 ins_encode(/* empty encoding */);
8425 ins_cost(0);
8426 ins_pipe(pipe_class_empty);
8427 %}
8428
8429 instruct castVV(vReg dst)
8430 %{
8431 match(Set dst (CastVV dst));
8432
8433 size(0);
8434 format %{ "# castVV of $dst" %}
8435 ins_encode(/* empty encoding */);
8436 ins_cost(0);
8437 ins_pipe(pipe_class_empty);
8438 %}
8439
8440 instruct castVVMask(pRegGov dst)
8441 %{
8442 match(Set dst (CastVV dst));
8443
8444 size(0);
8445 format %{ "# castVV of $dst" %}
8446 ins_encode(/* empty encoding */);
8447 ins_cost(0);
8448 ins_pipe(pipe_class_empty);
8449 %}
8450
8451 // ============================================================================
8452 // Atomic operation instructions
8453 //
8454
8455 // standard CompareAndSwapX when we are using barriers
8456 // these have higher priority than the rules selected by a predicate
8457
8458 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8459 // can't match them
8460
8461 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8462
8463 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8464 ins_cost(2 * VOLATILE_REF_COST);
8465
8466 effect(KILL cr);
8467
8468 format %{
8469 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8470 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8471 %}
8472
8473 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8474 aarch64_enc_cset_eq(res));
8475
8476 ins_pipe(pipe_slow);
8477 %}
8478
8479 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8480
8481 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8482 ins_cost(2 * VOLATILE_REF_COST);
8483
8484 effect(KILL cr);
8485
8486 format %{
8487 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8488 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8489 %}
8490
8491 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8492 aarch64_enc_cset_eq(res));
8493
8494 ins_pipe(pipe_slow);
8495 %}
8496
8497 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8498
8499 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8500 ins_cost(2 * VOLATILE_REF_COST);
8501
8502 effect(KILL cr);
8503
8504 format %{
8505 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8506 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8507 %}
8508
8509 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8510 aarch64_enc_cset_eq(res));
8511
8512 ins_pipe(pipe_slow);
8513 %}
8514
8515 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8516
8517 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8518 ins_cost(2 * VOLATILE_REF_COST);
8519
8520 effect(KILL cr);
8521
8522 format %{
8523 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8524 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8525 %}
8526
8527 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8528 aarch64_enc_cset_eq(res));
8529
8530 ins_pipe(pipe_slow);
8531 %}
8532
8533 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8534
8535 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8536 predicate(n->as_LoadStore()->barrier_data() == 0);
8537 ins_cost(2 * VOLATILE_REF_COST);
8538
8539 effect(KILL cr);
8540
8541 format %{
8542 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8543 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8544 %}
8545
8546 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8547 aarch64_enc_cset_eq(res));
8548
8549 ins_pipe(pipe_slow);
8550 %}
8551
8552 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8553
8554 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8555 predicate(n->as_LoadStore()->barrier_data() == 0);
8556 ins_cost(2 * VOLATILE_REF_COST);
8557
8558 effect(KILL cr);
8559
8560 format %{
8561 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8562 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8563 %}
8564
8565 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8566 aarch64_enc_cset_eq(res));
8567
8568 ins_pipe(pipe_slow);
8569 %}
8570
8571 // alternative CompareAndSwapX when we are eliding barriers
8572
8573 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8574
8575 predicate(needs_acquiring_load_exclusive(n));
8576 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8577 ins_cost(VOLATILE_REF_COST);
8578
8579 effect(KILL cr);
8580
8581 format %{
8582 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8583 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8584 %}
8585
8586 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8587 aarch64_enc_cset_eq(res));
8588
8589 ins_pipe(pipe_slow);
8590 %}
8591
8592 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8593
8594 predicate(needs_acquiring_load_exclusive(n));
8595 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8596 ins_cost(VOLATILE_REF_COST);
8597
8598 effect(KILL cr);
8599
8600 format %{
8601 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8602 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8603 %}
8604
8605 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8606 aarch64_enc_cset_eq(res));
8607
8608 ins_pipe(pipe_slow);
8609 %}
8610
8611 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8612
8613 predicate(needs_acquiring_load_exclusive(n));
8614 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8615 ins_cost(VOLATILE_REF_COST);
8616
8617 effect(KILL cr);
8618
8619 format %{
8620 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8621 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8622 %}
8623
8624 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8625 aarch64_enc_cset_eq(res));
8626
8627 ins_pipe(pipe_slow);
8628 %}
8629
8630 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8631
8632 predicate(needs_acquiring_load_exclusive(n));
8633 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8634 ins_cost(VOLATILE_REF_COST);
8635
8636 effect(KILL cr);
8637
8638 format %{
8639 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8640 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8641 %}
8642
8643 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8644 aarch64_enc_cset_eq(res));
8645
8646 ins_pipe(pipe_slow);
8647 %}
8648
8649 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8650
8651 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8652 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8653 ins_cost(VOLATILE_REF_COST);
8654
8655 effect(KILL cr);
8656
8657 format %{
8658 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8659 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8660 %}
8661
8662 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8663 aarch64_enc_cset_eq(res));
8664
8665 ins_pipe(pipe_slow);
8666 %}
8667
8668 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8669
8670 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8671 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8672 ins_cost(VOLATILE_REF_COST);
8673
8674 effect(KILL cr);
8675
8676 format %{
8677 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8678 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8679 %}
8680
8681 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8682 aarch64_enc_cset_eq(res));
8683
8684 ins_pipe(pipe_slow);
8685 %}
8686
8687
8688 // ---------------------------------------------------------------------
8689
8690 // BEGIN This section of the file is automatically generated. Do not edit --------------
8691
8692 // Sundry CAS operations. Note that release is always true,
8693 // regardless of the memory ordering of the CAS. This is because we
8694 // need the volatile case to be sequentially consistent but there is
8695 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8696 // can't check the type of memory ordering here, so we always emit a
8697 // STLXR.
8698
8699 // This section is generated from cas.m4
8700
8701
8702 // This pattern is generated automatically from cas.m4.
8703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8704 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8705 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8706 ins_cost(2 * VOLATILE_REF_COST);
8707 effect(TEMP_DEF res, KILL cr);
8708 format %{
8709 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8710 %}
8711 ins_encode %{
8712 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8713 Assembler::byte, /*acquire*/ false, /*release*/ true,
8714 /*weak*/ false, $res$$Register);
8715 __ sxtbw($res$$Register, $res$$Register);
8716 %}
8717 ins_pipe(pipe_slow);
8718 %}
8719
8720 // This pattern is generated automatically from cas.m4.
8721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8722 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8723 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8724 ins_cost(2 * VOLATILE_REF_COST);
8725 effect(TEMP_DEF res, KILL cr);
8726 format %{
8727 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8728 %}
8729 ins_encode %{
8730 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8731 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8732 /*weak*/ false, $res$$Register);
8733 __ sxthw($res$$Register, $res$$Register);
8734 %}
8735 ins_pipe(pipe_slow);
8736 %}
8737
8738 // This pattern is generated automatically from cas.m4.
8739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8740 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8741 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8742 ins_cost(2 * VOLATILE_REF_COST);
8743 effect(TEMP_DEF res, KILL cr);
8744 format %{
8745 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8746 %}
8747 ins_encode %{
8748 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8749 Assembler::word, /*acquire*/ false, /*release*/ true,
8750 /*weak*/ false, $res$$Register);
8751 %}
8752 ins_pipe(pipe_slow);
8753 %}
8754
8755 // This pattern is generated automatically from cas.m4.
8756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8757 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8758 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8759 ins_cost(2 * VOLATILE_REF_COST);
8760 effect(TEMP_DEF res, KILL cr);
8761 format %{
8762 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8763 %}
8764 ins_encode %{
8765 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8766 Assembler::xword, /*acquire*/ false, /*release*/ true,
8767 /*weak*/ false, $res$$Register);
8768 %}
8769 ins_pipe(pipe_slow);
8770 %}
8771
8772 // This pattern is generated automatically from cas.m4.
8773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8774 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8775 predicate(n->as_LoadStore()->barrier_data() == 0);
8776 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8777 ins_cost(2 * VOLATILE_REF_COST);
8778 effect(TEMP_DEF res, KILL cr);
8779 format %{
8780 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8781 %}
8782 ins_encode %{
8783 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8784 Assembler::word, /*acquire*/ false, /*release*/ true,
8785 /*weak*/ false, $res$$Register);
8786 %}
8787 ins_pipe(pipe_slow);
8788 %}
8789
8790 // This pattern is generated automatically from cas.m4.
8791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8792 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8793 predicate(n->as_LoadStore()->barrier_data() == 0);
8794 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8795 ins_cost(2 * VOLATILE_REF_COST);
8796 effect(TEMP_DEF res, KILL cr);
8797 format %{
8798 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8799 %}
8800 ins_encode %{
8801 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8802 Assembler::xword, /*acquire*/ false, /*release*/ true,
8803 /*weak*/ false, $res$$Register);
8804 %}
8805 ins_pipe(pipe_slow);
8806 %}
8807
8808 // This pattern is generated automatically from cas.m4.
8809 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8810 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8811 predicate(needs_acquiring_load_exclusive(n));
8812 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8813 ins_cost(VOLATILE_REF_COST);
8814 effect(TEMP_DEF res, KILL cr);
8815 format %{
8816 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8817 %}
8818 ins_encode %{
8819 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8820 Assembler::byte, /*acquire*/ true, /*release*/ true,
8821 /*weak*/ false, $res$$Register);
8822 __ sxtbw($res$$Register, $res$$Register);
8823 %}
8824 ins_pipe(pipe_slow);
8825 %}
8826
8827 // This pattern is generated automatically from cas.m4.
8828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8829 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8830 predicate(needs_acquiring_load_exclusive(n));
8831 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8832 ins_cost(VOLATILE_REF_COST);
8833 effect(TEMP_DEF res, KILL cr);
8834 format %{
8835 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8836 %}
8837 ins_encode %{
8838 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8839 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8840 /*weak*/ false, $res$$Register);
8841 __ sxthw($res$$Register, $res$$Register);
8842 %}
8843 ins_pipe(pipe_slow);
8844 %}
8845
8846 // This pattern is generated automatically from cas.m4.
8847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8848 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8849 predicate(needs_acquiring_load_exclusive(n));
8850 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8851 ins_cost(VOLATILE_REF_COST);
8852 effect(TEMP_DEF res, KILL cr);
8853 format %{
8854 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8855 %}
8856 ins_encode %{
8857 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8858 Assembler::word, /*acquire*/ true, /*release*/ true,
8859 /*weak*/ false, $res$$Register);
8860 %}
8861 ins_pipe(pipe_slow);
8862 %}
8863
8864 // This pattern is generated automatically from cas.m4.
8865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8866 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8867 predicate(needs_acquiring_load_exclusive(n));
8868 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8869 ins_cost(VOLATILE_REF_COST);
8870 effect(TEMP_DEF res, KILL cr);
8871 format %{
8872 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8873 %}
8874 ins_encode %{
8875 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8876 Assembler::xword, /*acquire*/ true, /*release*/ true,
8877 /*weak*/ false, $res$$Register);
8878 %}
8879 ins_pipe(pipe_slow);
8880 %}
8881
8882 // This pattern is generated automatically from cas.m4.
8883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8884 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8885 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8886 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8887 ins_cost(VOLATILE_REF_COST);
8888 effect(TEMP_DEF res, KILL cr);
8889 format %{
8890 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8891 %}
8892 ins_encode %{
8893 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8894 Assembler::word, /*acquire*/ true, /*release*/ true,
8895 /*weak*/ false, $res$$Register);
8896 %}
8897 ins_pipe(pipe_slow);
8898 %}
8899
8900 // This pattern is generated automatically from cas.m4.
8901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8902 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8903 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8904 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8905 ins_cost(VOLATILE_REF_COST);
8906 effect(TEMP_DEF res, KILL cr);
8907 format %{
8908 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8909 %}
8910 ins_encode %{
8911 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8912 Assembler::xword, /*acquire*/ true, /*release*/ true,
8913 /*weak*/ false, $res$$Register);
8914 %}
8915 ins_pipe(pipe_slow);
8916 %}
8917
8918 // This pattern is generated automatically from cas.m4.
8919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8920 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8921 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8922 ins_cost(2 * VOLATILE_REF_COST);
8923 effect(KILL cr);
8924 format %{
8925 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8926 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8927 %}
8928 ins_encode %{
8929 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8930 Assembler::byte, /*acquire*/ false, /*release*/ true,
8931 /*weak*/ true, noreg);
8932 __ csetw($res$$Register, Assembler::EQ);
8933 %}
8934 ins_pipe(pipe_slow);
8935 %}
8936
8937 // This pattern is generated automatically from cas.m4.
8938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8939 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8940 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8941 ins_cost(2 * VOLATILE_REF_COST);
8942 effect(KILL cr);
8943 format %{
8944 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8945 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8946 %}
8947 ins_encode %{
8948 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8949 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8950 /*weak*/ true, noreg);
8951 __ csetw($res$$Register, Assembler::EQ);
8952 %}
8953 ins_pipe(pipe_slow);
8954 %}
8955
8956 // This pattern is generated automatically from cas.m4.
8957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8958 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8959 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8960 ins_cost(2 * VOLATILE_REF_COST);
8961 effect(KILL cr);
8962 format %{
8963 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8964 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8965 %}
8966 ins_encode %{
8967 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8968 Assembler::word, /*acquire*/ false, /*release*/ true,
8969 /*weak*/ true, noreg);
8970 __ csetw($res$$Register, Assembler::EQ);
8971 %}
8972 ins_pipe(pipe_slow);
8973 %}
8974
8975 // This pattern is generated automatically from cas.m4.
8976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8977 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8978 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8979 ins_cost(2 * VOLATILE_REF_COST);
8980 effect(KILL cr);
8981 format %{
8982 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8983 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8984 %}
8985 ins_encode %{
8986 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8987 Assembler::xword, /*acquire*/ false, /*release*/ true,
8988 /*weak*/ true, noreg);
8989 __ csetw($res$$Register, Assembler::EQ);
8990 %}
8991 ins_pipe(pipe_slow);
8992 %}
8993
8994 // This pattern is generated automatically from cas.m4.
8995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8996 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8997 predicate(n->as_LoadStore()->barrier_data() == 0);
8998 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8999 ins_cost(2 * VOLATILE_REF_COST);
9000 effect(KILL cr);
9001 format %{
9002 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9003 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9004 %}
9005 ins_encode %{
9006 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9007 Assembler::word, /*acquire*/ false, /*release*/ true,
9008 /*weak*/ true, noreg);
9009 __ csetw($res$$Register, Assembler::EQ);
9010 %}
9011 ins_pipe(pipe_slow);
9012 %}
9013
9014 // This pattern is generated automatically from cas.m4.
9015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9016 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9017 predicate(n->as_LoadStore()->barrier_data() == 0);
9018 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9019 ins_cost(2 * VOLATILE_REF_COST);
9020 effect(KILL cr);
9021 format %{
9022 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9023 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9024 %}
9025 ins_encode %{
9026 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9027 Assembler::xword, /*acquire*/ false, /*release*/ true,
9028 /*weak*/ true, noreg);
9029 __ csetw($res$$Register, Assembler::EQ);
9030 %}
9031 ins_pipe(pipe_slow);
9032 %}
9033
9034 // This pattern is generated automatically from cas.m4.
9035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9036 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9037 predicate(needs_acquiring_load_exclusive(n));
9038 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9039 ins_cost(VOLATILE_REF_COST);
9040 effect(KILL cr);
9041 format %{
9042 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9043 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9044 %}
9045 ins_encode %{
9046 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9047 Assembler::byte, /*acquire*/ true, /*release*/ true,
9048 /*weak*/ true, noreg);
9049 __ csetw($res$$Register, Assembler::EQ);
9050 %}
9051 ins_pipe(pipe_slow);
9052 %}
9053
9054 // This pattern is generated automatically from cas.m4.
9055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9056 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9057 predicate(needs_acquiring_load_exclusive(n));
9058 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9059 ins_cost(VOLATILE_REF_COST);
9060 effect(KILL cr);
9061 format %{
9062 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9063 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9064 %}
9065 ins_encode %{
9066 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9067 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9068 /*weak*/ true, noreg);
9069 __ csetw($res$$Register, Assembler::EQ);
9070 %}
9071 ins_pipe(pipe_slow);
9072 %}
9073
9074 // This pattern is generated automatically from cas.m4.
9075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9076 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9077 predicate(needs_acquiring_load_exclusive(n));
9078 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9079 ins_cost(VOLATILE_REF_COST);
9080 effect(KILL cr);
9081 format %{
9082 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9083 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9084 %}
9085 ins_encode %{
9086 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9087 Assembler::word, /*acquire*/ true, /*release*/ true,
9088 /*weak*/ true, noreg);
9089 __ csetw($res$$Register, Assembler::EQ);
9090 %}
9091 ins_pipe(pipe_slow);
9092 %}
9093
9094 // This pattern is generated automatically from cas.m4.
9095 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9096 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9097 predicate(needs_acquiring_load_exclusive(n));
9098 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9099 ins_cost(VOLATILE_REF_COST);
9100 effect(KILL cr);
9101 format %{
9102 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9103 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9104 %}
9105 ins_encode %{
9106 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9107 Assembler::xword, /*acquire*/ true, /*release*/ true,
9108 /*weak*/ true, noreg);
9109 __ csetw($res$$Register, Assembler::EQ);
9110 %}
9111 ins_pipe(pipe_slow);
9112 %}
9113
9114 // This pattern is generated automatically from cas.m4.
9115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9116 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9117 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9118 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9119 ins_cost(VOLATILE_REF_COST);
9120 effect(KILL cr);
9121 format %{
9122 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9123 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9124 %}
9125 ins_encode %{
9126 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9127 Assembler::word, /*acquire*/ true, /*release*/ true,
9128 /*weak*/ true, noreg);
9129 __ csetw($res$$Register, Assembler::EQ);
9130 %}
9131 ins_pipe(pipe_slow);
9132 %}
9133
9134 // This pattern is generated automatically from cas.m4.
9135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9136 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9137 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9138 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9139 ins_cost(VOLATILE_REF_COST);
9140 effect(KILL cr);
9141 format %{
9142 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9143 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9144 %}
9145 ins_encode %{
9146 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9147 Assembler::xword, /*acquire*/ true, /*release*/ true,
9148 /*weak*/ true, noreg);
9149 __ csetw($res$$Register, Assembler::EQ);
9150 %}
9151 ins_pipe(pipe_slow);
9152 %}
9153
9154 // END This section of the file is automatically generated. Do not edit --------------
9155 // ---------------------------------------------------------------------
9156
9157 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9158 match(Set prev (GetAndSetI mem newv));
9159 ins_cost(2 * VOLATILE_REF_COST);
9160 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9161 ins_encode %{
9162 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9163 %}
9164 ins_pipe(pipe_serial);
9165 %}
9166
9167 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9168 match(Set prev (GetAndSetL mem newv));
9169 ins_cost(2 * VOLATILE_REF_COST);
9170 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9171 ins_encode %{
9172 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9173 %}
9174 ins_pipe(pipe_serial);
9175 %}
9176
9177 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9178 predicate(n->as_LoadStore()->barrier_data() == 0);
9179 match(Set prev (GetAndSetN mem newv));
9180 ins_cost(2 * VOLATILE_REF_COST);
9181 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9182 ins_encode %{
9183 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9184 %}
9185 ins_pipe(pipe_serial);
9186 %}
9187
9188 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9189 predicate(n->as_LoadStore()->barrier_data() == 0);
9190 match(Set prev (GetAndSetP mem newv));
9191 ins_cost(2 * VOLATILE_REF_COST);
9192 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9193 ins_encode %{
9194 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9195 %}
9196 ins_pipe(pipe_serial);
9197 %}
9198
9199 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9200 predicate(needs_acquiring_load_exclusive(n));
9201 match(Set prev (GetAndSetI mem newv));
9202 ins_cost(VOLATILE_REF_COST);
9203 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9204 ins_encode %{
9205 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9206 %}
9207 ins_pipe(pipe_serial);
9208 %}
9209
9210 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9211 predicate(needs_acquiring_load_exclusive(n));
9212 match(Set prev (GetAndSetL mem newv));
9213 ins_cost(VOLATILE_REF_COST);
9214 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9215 ins_encode %{
9216 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9217 %}
9218 ins_pipe(pipe_serial);
9219 %}
9220
9221 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9222 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9223 match(Set prev (GetAndSetN mem newv));
9224 ins_cost(VOLATILE_REF_COST);
9225 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9226 ins_encode %{
9227 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9228 %}
9229 ins_pipe(pipe_serial);
9230 %}
9231
9232 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9233 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9234 match(Set prev (GetAndSetP mem newv));
9235 ins_cost(VOLATILE_REF_COST);
9236 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9237 ins_encode %{
9238 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9239 %}
9240 ins_pipe(pipe_serial);
9241 %}
9242
9243
9244 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9245 match(Set newval (GetAndAddL mem incr));
9246 ins_cost(2 * VOLATILE_REF_COST + 1);
9247 format %{ "get_and_addL $newval, [$mem], $incr" %}
9248 ins_encode %{
9249 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9250 %}
9251 ins_pipe(pipe_serial);
9252 %}
9253
9254 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9255 predicate(n->as_LoadStore()->result_not_used());
9256 match(Set dummy (GetAndAddL mem incr));
9257 ins_cost(2 * VOLATILE_REF_COST);
9258 format %{ "get_and_addL [$mem], $incr" %}
9259 ins_encode %{
9260 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9261 %}
9262 ins_pipe(pipe_serial);
9263 %}
9264
9265 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9266 match(Set newval (GetAndAddL mem incr));
9267 ins_cost(2 * VOLATILE_REF_COST + 1);
9268 format %{ "get_and_addL $newval, [$mem], $incr" %}
9269 ins_encode %{
9270 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9271 %}
9272 ins_pipe(pipe_serial);
9273 %}
9274
9275 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9276 predicate(n->as_LoadStore()->result_not_used());
9277 match(Set dummy (GetAndAddL mem incr));
9278 ins_cost(2 * VOLATILE_REF_COST);
9279 format %{ "get_and_addL [$mem], $incr" %}
9280 ins_encode %{
9281 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9282 %}
9283 ins_pipe(pipe_serial);
9284 %}
9285
9286 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9287 match(Set newval (GetAndAddI mem incr));
9288 ins_cost(2 * VOLATILE_REF_COST + 1);
9289 format %{ "get_and_addI $newval, [$mem], $incr" %}
9290 ins_encode %{
9291 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9292 %}
9293 ins_pipe(pipe_serial);
9294 %}
9295
9296 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9297 predicate(n->as_LoadStore()->result_not_used());
9298 match(Set dummy (GetAndAddI mem incr));
9299 ins_cost(2 * VOLATILE_REF_COST);
9300 format %{ "get_and_addI [$mem], $incr" %}
9301 ins_encode %{
9302 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9303 %}
9304 ins_pipe(pipe_serial);
9305 %}
9306
9307 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9308 match(Set newval (GetAndAddI mem incr));
9309 ins_cost(2 * VOLATILE_REF_COST + 1);
9310 format %{ "get_and_addI $newval, [$mem], $incr" %}
9311 ins_encode %{
9312 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9313 %}
9314 ins_pipe(pipe_serial);
9315 %}
9316
9317 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9318 predicate(n->as_LoadStore()->result_not_used());
9319 match(Set dummy (GetAndAddI mem incr));
9320 ins_cost(2 * VOLATILE_REF_COST);
9321 format %{ "get_and_addI [$mem], $incr" %}
9322 ins_encode %{
9323 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9324 %}
9325 ins_pipe(pipe_serial);
9326 %}
9327
9328 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9329 predicate(needs_acquiring_load_exclusive(n));
9330 match(Set newval (GetAndAddL mem incr));
9331 ins_cost(VOLATILE_REF_COST + 1);
9332 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9333 ins_encode %{
9334 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9335 %}
9336 ins_pipe(pipe_serial);
9337 %}
9338
9339 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9340 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9341 match(Set dummy (GetAndAddL mem incr));
9342 ins_cost(VOLATILE_REF_COST);
9343 format %{ "get_and_addL_acq [$mem], $incr" %}
9344 ins_encode %{
9345 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9346 %}
9347 ins_pipe(pipe_serial);
9348 %}
9349
9350 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9351 predicate(needs_acquiring_load_exclusive(n));
9352 match(Set newval (GetAndAddL mem incr));
9353 ins_cost(VOLATILE_REF_COST + 1);
9354 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9355 ins_encode %{
9356 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9357 %}
9358 ins_pipe(pipe_serial);
9359 %}
9360
9361 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9362 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9363 match(Set dummy (GetAndAddL mem incr));
9364 ins_cost(VOLATILE_REF_COST);
9365 format %{ "get_and_addL_acq [$mem], $incr" %}
9366 ins_encode %{
9367 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9368 %}
9369 ins_pipe(pipe_serial);
9370 %}
9371
9372 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9373 predicate(needs_acquiring_load_exclusive(n));
9374 match(Set newval (GetAndAddI mem incr));
9375 ins_cost(VOLATILE_REF_COST + 1);
9376 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9377 ins_encode %{
9378 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9379 %}
9380 ins_pipe(pipe_serial);
9381 %}
9382
9383 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9384 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9385 match(Set dummy (GetAndAddI mem incr));
9386 ins_cost(VOLATILE_REF_COST);
9387 format %{ "get_and_addI_acq [$mem], $incr" %}
9388 ins_encode %{
9389 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9390 %}
9391 ins_pipe(pipe_serial);
9392 %}
9393
9394 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9395 predicate(needs_acquiring_load_exclusive(n));
9396 match(Set newval (GetAndAddI mem incr));
9397 ins_cost(VOLATILE_REF_COST + 1);
9398 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9399 ins_encode %{
9400 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9401 %}
9402 ins_pipe(pipe_serial);
9403 %}
9404
9405 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9406 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9407 match(Set dummy (GetAndAddI mem incr));
9408 ins_cost(VOLATILE_REF_COST);
9409 format %{ "get_and_addI_acq [$mem], $incr" %}
9410 ins_encode %{
9411 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9412 %}
9413 ins_pipe(pipe_serial);
9414 %}
9415
9416 // Manifest a CmpU result in an integer register.
9417 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9418 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9419 %{
9420 match(Set dst (CmpU3 src1 src2));
9421 effect(KILL flags);
9422
9423 ins_cost(INSN_COST * 3);
9424 format %{
9425 "cmpw $src1, $src2\n\t"
9426 "csetw $dst, ne\n\t"
9427 "cnegw $dst, lo\t# CmpU3(reg)"
9428 %}
9429 ins_encode %{
9430 __ cmpw($src1$$Register, $src2$$Register);
9431 __ csetw($dst$$Register, Assembler::NE);
9432 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9433 %}
9434
9435 ins_pipe(pipe_class_default);
9436 %}
9437
9438 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9439 %{
9440 match(Set dst (CmpU3 src1 src2));
9441 effect(KILL flags);
9442
9443 ins_cost(INSN_COST * 3);
9444 format %{
9445 "subsw zr, $src1, $src2\n\t"
9446 "csetw $dst, ne\n\t"
9447 "cnegw $dst, lo\t# CmpU3(imm)"
9448 %}
9449 ins_encode %{
9450 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9451 __ csetw($dst$$Register, Assembler::NE);
9452 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9453 %}
9454
9455 ins_pipe(pipe_class_default);
9456 %}
9457
9458 // Manifest a CmpUL result in an integer register.
9459 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9460 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9461 %{
9462 match(Set dst (CmpUL3 src1 src2));
9463 effect(KILL flags);
9464
9465 ins_cost(INSN_COST * 3);
9466 format %{
9467 "cmp $src1, $src2\n\t"
9468 "csetw $dst, ne\n\t"
9469 "cnegw $dst, lo\t# CmpUL3(reg)"
9470 %}
9471 ins_encode %{
9472 __ cmp($src1$$Register, $src2$$Register);
9473 __ csetw($dst$$Register, Assembler::NE);
9474 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9475 %}
9476
9477 ins_pipe(pipe_class_default);
9478 %}
9479
9480 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9481 %{
9482 match(Set dst (CmpUL3 src1 src2));
9483 effect(KILL flags);
9484
9485 ins_cost(INSN_COST * 3);
9486 format %{
9487 "subs zr, $src1, $src2\n\t"
9488 "csetw $dst, ne\n\t"
9489 "cnegw $dst, lo\t# CmpUL3(imm)"
9490 %}
9491 ins_encode %{
9492 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9493 __ csetw($dst$$Register, Assembler::NE);
9494 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9495 %}
9496
9497 ins_pipe(pipe_class_default);
9498 %}
9499
9500 // Manifest a CmpL result in an integer register.
9501 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9502 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9503 %{
9504 match(Set dst (CmpL3 src1 src2));
9505 effect(KILL flags);
9506
9507 ins_cost(INSN_COST * 3);
9508 format %{
9509 "cmp $src1, $src2\n\t"
9510 "csetw $dst, ne\n\t"
9511 "cnegw $dst, lt\t# CmpL3(reg)"
9512 %}
9513 ins_encode %{
9514 __ cmp($src1$$Register, $src2$$Register);
9515 __ csetw($dst$$Register, Assembler::NE);
9516 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9517 %}
9518
9519 ins_pipe(pipe_class_default);
9520 %}
9521
9522 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9523 %{
9524 match(Set dst (CmpL3 src1 src2));
9525 effect(KILL flags);
9526
9527 ins_cost(INSN_COST * 3);
9528 format %{
9529 "subs zr, $src1, $src2\n\t"
9530 "csetw $dst, ne\n\t"
9531 "cnegw $dst, lt\t# CmpL3(imm)"
9532 %}
9533 ins_encode %{
9534 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9535 __ csetw($dst$$Register, Assembler::NE);
9536 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9537 %}
9538
9539 ins_pipe(pipe_class_default);
9540 %}
9541
9542 // ============================================================================
9543 // Conditional Move Instructions
9544
9545 // n.b. we have identical rules for both a signed compare op (cmpOp)
9546 // and an unsigned compare op (cmpOpU). it would be nice if we could
9547 // define an op class which merged both inputs and use it to type the
9548 // argument to a single rule. unfortunatelyt his fails because the
9549 // opclass does not live up to the COND_INTER interface of its
9550 // component operands. When the generic code tries to negate the
9551 // operand it ends up running the generci Machoper::negate method
9552 // which throws a ShouldNotHappen. So, we have to provide two flavours
9553 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9554
9555 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9556 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9557
9558 ins_cost(INSN_COST * 2);
9559 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9560
9561 ins_encode %{
9562 __ cselw(as_Register($dst$$reg),
9563 as_Register($src2$$reg),
9564 as_Register($src1$$reg),
9565 (Assembler::Condition)$cmp$$cmpcode);
9566 %}
9567
9568 ins_pipe(icond_reg_reg);
9569 %}
9570
9571 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9572 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9573
9574 ins_cost(INSN_COST * 2);
9575 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9576
9577 ins_encode %{
9578 __ cselw(as_Register($dst$$reg),
9579 as_Register($src2$$reg),
9580 as_Register($src1$$reg),
9581 (Assembler::Condition)$cmp$$cmpcode);
9582 %}
9583
9584 ins_pipe(icond_reg_reg);
9585 %}
9586
9587 // special cases where one arg is zero
9588
9589 // n.b. this is selected in preference to the rule above because it
9590 // avoids loading constant 0 into a source register
9591
9592 // TODO
9593 // we ought only to be able to cull one of these variants as the ideal
9594 // transforms ought always to order the zero consistently (to left/right?)
9595
9596 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9597 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9598
9599 ins_cost(INSN_COST * 2);
9600 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9601
9602 ins_encode %{
9603 __ cselw(as_Register($dst$$reg),
9604 as_Register($src$$reg),
9605 zr,
9606 (Assembler::Condition)$cmp$$cmpcode);
9607 %}
9608
9609 ins_pipe(icond_reg);
9610 %}
9611
9612 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9613 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9614
9615 ins_cost(INSN_COST * 2);
9616 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9617
9618 ins_encode %{
9619 __ cselw(as_Register($dst$$reg),
9620 as_Register($src$$reg),
9621 zr,
9622 (Assembler::Condition)$cmp$$cmpcode);
9623 %}
9624
9625 ins_pipe(icond_reg);
9626 %}
9627
9628 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9629 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9630
9631 ins_cost(INSN_COST * 2);
9632 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9633
9634 ins_encode %{
9635 __ cselw(as_Register($dst$$reg),
9636 zr,
9637 as_Register($src$$reg),
9638 (Assembler::Condition)$cmp$$cmpcode);
9639 %}
9640
9641 ins_pipe(icond_reg);
9642 %}
9643
9644 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9645 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9646
9647 ins_cost(INSN_COST * 2);
9648 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9649
9650 ins_encode %{
9651 __ cselw(as_Register($dst$$reg),
9652 zr,
9653 as_Register($src$$reg),
9654 (Assembler::Condition)$cmp$$cmpcode);
9655 %}
9656
9657 ins_pipe(icond_reg);
9658 %}
9659
9660 // special case for creating a boolean 0 or 1
9661
9662 // n.b. this is selected in preference to the rule above because it
9663 // avoids loading constants 0 and 1 into a source register
9664
9665 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9666 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9667
9668 ins_cost(INSN_COST * 2);
9669 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9670
9671 ins_encode %{
9672 // equivalently
9673 // cset(as_Register($dst$$reg),
9674 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9675 __ csincw(as_Register($dst$$reg),
9676 zr,
9677 zr,
9678 (Assembler::Condition)$cmp$$cmpcode);
9679 %}
9680
9681 ins_pipe(icond_none);
9682 %}
9683
9684 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9685 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9686
9687 ins_cost(INSN_COST * 2);
9688 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9689
9690 ins_encode %{
9691 // equivalently
9692 // cset(as_Register($dst$$reg),
9693 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9694 __ csincw(as_Register($dst$$reg),
9695 zr,
9696 zr,
9697 (Assembler::Condition)$cmp$$cmpcode);
9698 %}
9699
9700 ins_pipe(icond_none);
9701 %}
9702
9703 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9704 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9705
9706 ins_cost(INSN_COST * 2);
9707 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9708
9709 ins_encode %{
9710 __ csel(as_Register($dst$$reg),
9711 as_Register($src2$$reg),
9712 as_Register($src1$$reg),
9713 (Assembler::Condition)$cmp$$cmpcode);
9714 %}
9715
9716 ins_pipe(icond_reg_reg);
9717 %}
9718
9719 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9720 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9721
9722 ins_cost(INSN_COST * 2);
9723 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9724
9725 ins_encode %{
9726 __ csel(as_Register($dst$$reg),
9727 as_Register($src2$$reg),
9728 as_Register($src1$$reg),
9729 (Assembler::Condition)$cmp$$cmpcode);
9730 %}
9731
9732 ins_pipe(icond_reg_reg);
9733 %}
9734
9735 // special cases where one arg is zero
9736
9737 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9738 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9739
9740 ins_cost(INSN_COST * 2);
9741 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9742
9743 ins_encode %{
9744 __ csel(as_Register($dst$$reg),
9745 zr,
9746 as_Register($src$$reg),
9747 (Assembler::Condition)$cmp$$cmpcode);
9748 %}
9749
9750 ins_pipe(icond_reg);
9751 %}
9752
9753 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9754 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9755
9756 ins_cost(INSN_COST * 2);
9757 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9758
9759 ins_encode %{
9760 __ csel(as_Register($dst$$reg),
9761 zr,
9762 as_Register($src$$reg),
9763 (Assembler::Condition)$cmp$$cmpcode);
9764 %}
9765
9766 ins_pipe(icond_reg);
9767 %}
9768
9769 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9770 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9771
9772 ins_cost(INSN_COST * 2);
9773 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9774
9775 ins_encode %{
9776 __ csel(as_Register($dst$$reg),
9777 as_Register($src$$reg),
9778 zr,
9779 (Assembler::Condition)$cmp$$cmpcode);
9780 %}
9781
9782 ins_pipe(icond_reg);
9783 %}
9784
9785 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9786 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9787
9788 ins_cost(INSN_COST * 2);
9789 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9790
9791 ins_encode %{
9792 __ csel(as_Register($dst$$reg),
9793 as_Register($src$$reg),
9794 zr,
9795 (Assembler::Condition)$cmp$$cmpcode);
9796 %}
9797
9798 ins_pipe(icond_reg);
9799 %}
9800
9801 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9802 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9803
9804 ins_cost(INSN_COST * 2);
9805 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9806
9807 ins_encode %{
9808 __ csel(as_Register($dst$$reg),
9809 as_Register($src2$$reg),
9810 as_Register($src1$$reg),
9811 (Assembler::Condition)$cmp$$cmpcode);
9812 %}
9813
9814 ins_pipe(icond_reg_reg);
9815 %}
9816
9817 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9818 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9819
9820 ins_cost(INSN_COST * 2);
9821 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9822
9823 ins_encode %{
9824 __ csel(as_Register($dst$$reg),
9825 as_Register($src2$$reg),
9826 as_Register($src1$$reg),
9827 (Assembler::Condition)$cmp$$cmpcode);
9828 %}
9829
9830 ins_pipe(icond_reg_reg);
9831 %}
9832
9833 // special cases where one arg is zero
9834
9835 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9836 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9837
9838 ins_cost(INSN_COST * 2);
9839 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9840
9841 ins_encode %{
9842 __ csel(as_Register($dst$$reg),
9843 zr,
9844 as_Register($src$$reg),
9845 (Assembler::Condition)$cmp$$cmpcode);
9846 %}
9847
9848 ins_pipe(icond_reg);
9849 %}
9850
9851 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9852 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9853
9854 ins_cost(INSN_COST * 2);
9855 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9856
9857 ins_encode %{
9858 __ csel(as_Register($dst$$reg),
9859 zr,
9860 as_Register($src$$reg),
9861 (Assembler::Condition)$cmp$$cmpcode);
9862 %}
9863
9864 ins_pipe(icond_reg);
9865 %}
9866
9867 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9868 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9869
9870 ins_cost(INSN_COST * 2);
9871 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9872
9873 ins_encode %{
9874 __ csel(as_Register($dst$$reg),
9875 as_Register($src$$reg),
9876 zr,
9877 (Assembler::Condition)$cmp$$cmpcode);
9878 %}
9879
9880 ins_pipe(icond_reg);
9881 %}
9882
9883 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9884 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9885
9886 ins_cost(INSN_COST * 2);
9887 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9888
9889 ins_encode %{
9890 __ csel(as_Register($dst$$reg),
9891 as_Register($src$$reg),
9892 zr,
9893 (Assembler::Condition)$cmp$$cmpcode);
9894 %}
9895
9896 ins_pipe(icond_reg);
9897 %}
9898
9899 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9900 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9901
9902 ins_cost(INSN_COST * 2);
9903 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9904
9905 ins_encode %{
9906 __ cselw(as_Register($dst$$reg),
9907 as_Register($src2$$reg),
9908 as_Register($src1$$reg),
9909 (Assembler::Condition)$cmp$$cmpcode);
9910 %}
9911
9912 ins_pipe(icond_reg_reg);
9913 %}
9914
9915 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9916 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9917
9918 ins_cost(INSN_COST * 2);
9919 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9920
9921 ins_encode %{
9922 __ cselw(as_Register($dst$$reg),
9923 as_Register($src2$$reg),
9924 as_Register($src1$$reg),
9925 (Assembler::Condition)$cmp$$cmpcode);
9926 %}
9927
9928 ins_pipe(icond_reg_reg);
9929 %}
9930
9931 // special cases where one arg is zero
9932
9933 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9934 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9935
9936 ins_cost(INSN_COST * 2);
9937 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9938
9939 ins_encode %{
9940 __ cselw(as_Register($dst$$reg),
9941 zr,
9942 as_Register($src$$reg),
9943 (Assembler::Condition)$cmp$$cmpcode);
9944 %}
9945
9946 ins_pipe(icond_reg);
9947 %}
9948
9949 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9950 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9951
9952 ins_cost(INSN_COST * 2);
9953 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9954
9955 ins_encode %{
9956 __ cselw(as_Register($dst$$reg),
9957 zr,
9958 as_Register($src$$reg),
9959 (Assembler::Condition)$cmp$$cmpcode);
9960 %}
9961
9962 ins_pipe(icond_reg);
9963 %}
9964
9965 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9966 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9967
9968 ins_cost(INSN_COST * 2);
9969 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9970
9971 ins_encode %{
9972 __ cselw(as_Register($dst$$reg),
9973 as_Register($src$$reg),
9974 zr,
9975 (Assembler::Condition)$cmp$$cmpcode);
9976 %}
9977
9978 ins_pipe(icond_reg);
9979 %}
9980
9981 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9982 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9983
9984 ins_cost(INSN_COST * 2);
9985 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9986
9987 ins_encode %{
9988 __ cselw(as_Register($dst$$reg),
9989 as_Register($src$$reg),
9990 zr,
9991 (Assembler::Condition)$cmp$$cmpcode);
9992 %}
9993
9994 ins_pipe(icond_reg);
9995 %}
9996
9997 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9998 %{
9999 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10000
10001 ins_cost(INSN_COST * 3);
10002
10003 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10004 ins_encode %{
10005 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10006 __ fcsels(as_FloatRegister($dst$$reg),
10007 as_FloatRegister($src2$$reg),
10008 as_FloatRegister($src1$$reg),
10009 cond);
10010 %}
10011
10012 ins_pipe(fp_cond_reg_reg_s);
10013 %}
10014
10015 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
10016 %{
10017 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10018
10019 ins_cost(INSN_COST * 3);
10020
10021 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10022 ins_encode %{
10023 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10024 __ fcsels(as_FloatRegister($dst$$reg),
10025 as_FloatRegister($src2$$reg),
10026 as_FloatRegister($src1$$reg),
10027 cond);
10028 %}
10029
10030 ins_pipe(fp_cond_reg_reg_s);
10031 %}
10032
10033 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10034 %{
10035 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10036
10037 ins_cost(INSN_COST * 3);
10038
10039 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10040 ins_encode %{
10041 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10042 __ fcseld(as_FloatRegister($dst$$reg),
10043 as_FloatRegister($src2$$reg),
10044 as_FloatRegister($src1$$reg),
10045 cond);
10046 %}
10047
10048 ins_pipe(fp_cond_reg_reg_d);
10049 %}
10050
10051 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10052 %{
10053 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10054
10055 ins_cost(INSN_COST * 3);
10056
10057 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10058 ins_encode %{
10059 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10060 __ fcseld(as_FloatRegister($dst$$reg),
10061 as_FloatRegister($src2$$reg),
10062 as_FloatRegister($src1$$reg),
10063 cond);
10064 %}
10065
10066 ins_pipe(fp_cond_reg_reg_d);
10067 %}
10068
10069 // ============================================================================
10070 // Arithmetic Instructions
10071 //
10072
10073 // Integer Addition
10074
10075 // TODO
10076 // these currently employ operations which do not set CR and hence are
10077 // not flagged as killing CR but we would like to isolate the cases
10078 // where we want to set flags from those where we don't. need to work
10079 // out how to do that.
10080
10081 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10082 match(Set dst (AddI src1 src2));
10083
10084 ins_cost(INSN_COST);
10085 format %{ "addw $dst, $src1, $src2" %}
10086
10087 ins_encode %{
10088 __ addw(as_Register($dst$$reg),
10089 as_Register($src1$$reg),
10090 as_Register($src2$$reg));
10091 %}
10092
10093 ins_pipe(ialu_reg_reg);
10094 %}
10095
10096 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10097 match(Set dst (AddI src1 src2));
10098
10099 ins_cost(INSN_COST);
10100 format %{ "addw $dst, $src1, $src2" %}
10101
10102 // use opcode to indicate that this is an add not a sub
10103 opcode(0x0);
10104
10105 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10106
10107 ins_pipe(ialu_reg_imm);
10108 %}
10109
10110 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10111 match(Set dst (AddI (ConvL2I src1) src2));
10112
10113 ins_cost(INSN_COST);
10114 format %{ "addw $dst, $src1, $src2" %}
10115
10116 // use opcode to indicate that this is an add not a sub
10117 opcode(0x0);
10118
10119 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10120
10121 ins_pipe(ialu_reg_imm);
10122 %}
10123
10124 // Pointer Addition
10125 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
10126 match(Set dst (AddP src1 src2));
10127
10128 ins_cost(INSN_COST);
10129 format %{ "add $dst, $src1, $src2\t# ptr" %}
10130
10131 ins_encode %{
10132 __ add(as_Register($dst$$reg),
10133 as_Register($src1$$reg),
10134 as_Register($src2$$reg));
10135 %}
10136
10137 ins_pipe(ialu_reg_reg);
10138 %}
10139
10140 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
10141 match(Set dst (AddP src1 (ConvI2L src2)));
10142
10143 ins_cost(1.9 * INSN_COST);
10144 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10145
10146 ins_encode %{
10147 __ add(as_Register($dst$$reg),
10148 as_Register($src1$$reg),
10149 as_Register($src2$$reg), ext::sxtw);
10150 %}
10151
10152 ins_pipe(ialu_reg_reg);
10153 %}
10154
10155 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10156 match(Set dst (AddP src1 (LShiftL src2 scale)));
10157
10158 ins_cost(1.9 * INSN_COST);
10159 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10160
10161 ins_encode %{
10162 __ lea(as_Register($dst$$reg),
10163 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10164 Address::lsl($scale$$constant)));
10165 %}
10166
10167 ins_pipe(ialu_reg_reg_shift);
10168 %}
10169
10170 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10171 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10172
10173 ins_cost(1.9 * INSN_COST);
10174 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10175
10176 ins_encode %{
10177 __ lea(as_Register($dst$$reg),
10178 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10179 Address::sxtw($scale$$constant)));
10180 %}
10181
10182 ins_pipe(ialu_reg_reg_shift);
10183 %}
10184
10185 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10186 match(Set dst (LShiftL (ConvI2L src) scale));
10187
10188 ins_cost(INSN_COST);
10189 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10190
10191 ins_encode %{
10192 __ sbfiz(as_Register($dst$$reg),
10193 as_Register($src$$reg),
10194 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10195 %}
10196
10197 ins_pipe(ialu_reg_shift);
10198 %}
10199
10200 // Pointer Immediate Addition
10201 // n.b. this needs to be more expensive than using an indirect memory
10202 // operand
10203 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10204 match(Set dst (AddP src1 src2));
10205
10206 ins_cost(INSN_COST);
10207 format %{ "add $dst, $src1, $src2\t# ptr" %}
10208
10209 // use opcode to indicate that this is an add not a sub
10210 opcode(0x0);
10211
10212 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10213
10214 ins_pipe(ialu_reg_imm);
10215 %}
10216
10217 // Long Addition
10218 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10219
10220 match(Set dst (AddL src1 src2));
10221
10222 ins_cost(INSN_COST);
10223 format %{ "add $dst, $src1, $src2" %}
10224
10225 ins_encode %{
10226 __ add(as_Register($dst$$reg),
10227 as_Register($src1$$reg),
10228 as_Register($src2$$reg));
10229 %}
10230
10231 ins_pipe(ialu_reg_reg);
10232 %}
10233
10234 // No constant pool entries requiredLong Immediate Addition.
10235 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10236 match(Set dst (AddL src1 src2));
10237
10238 ins_cost(INSN_COST);
10239 format %{ "add $dst, $src1, $src2" %}
10240
10241 // use opcode to indicate that this is an add not a sub
10242 opcode(0x0);
10243
10244 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10245
10246 ins_pipe(ialu_reg_imm);
10247 %}
10248
10249 // Integer Subtraction
10250 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10251 match(Set dst (SubI src1 src2));
10252
10253 ins_cost(INSN_COST);
10254 format %{ "subw $dst, $src1, $src2" %}
10255
10256 ins_encode %{
10257 __ subw(as_Register($dst$$reg),
10258 as_Register($src1$$reg),
10259 as_Register($src2$$reg));
10260 %}
10261
10262 ins_pipe(ialu_reg_reg);
10263 %}
10264
10265 // Immediate Subtraction
10266 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10267 match(Set dst (SubI src1 src2));
10268
10269 ins_cost(INSN_COST);
10270 format %{ "subw $dst, $src1, $src2" %}
10271
10272 // use opcode to indicate that this is a sub not an add
10273 opcode(0x1);
10274
10275 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10276
10277 ins_pipe(ialu_reg_imm);
10278 %}
10279
10280 // Long Subtraction
10281 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10282
10283 match(Set dst (SubL src1 src2));
10284
10285 ins_cost(INSN_COST);
10286 format %{ "sub $dst, $src1, $src2" %}
10287
10288 ins_encode %{
10289 __ sub(as_Register($dst$$reg),
10290 as_Register($src1$$reg),
10291 as_Register($src2$$reg));
10292 %}
10293
10294 ins_pipe(ialu_reg_reg);
10295 %}
10296
10297 // No constant pool entries requiredLong Immediate Subtraction.
10298 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10299 match(Set dst (SubL src1 src2));
10300
10301 ins_cost(INSN_COST);
10302 format %{ "sub$dst, $src1, $src2" %}
10303
10304 // use opcode to indicate that this is a sub not an add
10305 opcode(0x1);
10306
10307 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10308
10309 ins_pipe(ialu_reg_imm);
10310 %}
10311
10312 // Integer Negation (special case for sub)
10313
10314 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10315 match(Set dst (SubI zero src));
10316
10317 ins_cost(INSN_COST);
10318 format %{ "negw $dst, $src\t# int" %}
10319
10320 ins_encode %{
10321 __ negw(as_Register($dst$$reg),
10322 as_Register($src$$reg));
10323 %}
10324
10325 ins_pipe(ialu_reg);
10326 %}
10327
10328 // Long Negation
10329
10330 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10331 match(Set dst (SubL zero src));
10332
10333 ins_cost(INSN_COST);
10334 format %{ "neg $dst, $src\t# long" %}
10335
10336 ins_encode %{
10337 __ neg(as_Register($dst$$reg),
10338 as_Register($src$$reg));
10339 %}
10340
10341 ins_pipe(ialu_reg);
10342 %}
10343
10344 // Integer Multiply
10345
10346 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10347 match(Set dst (MulI src1 src2));
10348
10349 ins_cost(INSN_COST * 3);
10350 format %{ "mulw $dst, $src1, $src2" %}
10351
10352 ins_encode %{
10353 __ mulw(as_Register($dst$$reg),
10354 as_Register($src1$$reg),
10355 as_Register($src2$$reg));
10356 %}
10357
10358 ins_pipe(imul_reg_reg);
10359 %}
10360
10361 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10362 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10363
10364 ins_cost(INSN_COST * 3);
10365 format %{ "smull $dst, $src1, $src2" %}
10366
10367 ins_encode %{
10368 __ smull(as_Register($dst$$reg),
10369 as_Register($src1$$reg),
10370 as_Register($src2$$reg));
10371 %}
10372
10373 ins_pipe(imul_reg_reg);
10374 %}
10375
10376 // Long Multiply
10377
10378 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10379 match(Set dst (MulL src1 src2));
10380
10381 ins_cost(INSN_COST * 5);
10382 format %{ "mul $dst, $src1, $src2" %}
10383
10384 ins_encode %{
10385 __ mul(as_Register($dst$$reg),
10386 as_Register($src1$$reg),
10387 as_Register($src2$$reg));
10388 %}
10389
10390 ins_pipe(lmul_reg_reg);
10391 %}
10392
10393 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10394 %{
10395 match(Set dst (MulHiL src1 src2));
10396
10397 ins_cost(INSN_COST * 7);
10398 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10399
10400 ins_encode %{
10401 __ smulh(as_Register($dst$$reg),
10402 as_Register($src1$$reg),
10403 as_Register($src2$$reg));
10404 %}
10405
10406 ins_pipe(lmul_reg_reg);
10407 %}
10408
10409 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10410 %{
10411 match(Set dst (UMulHiL src1 src2));
10412
10413 ins_cost(INSN_COST * 7);
10414 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10415
10416 ins_encode %{
10417 __ umulh(as_Register($dst$$reg),
10418 as_Register($src1$$reg),
10419 as_Register($src2$$reg));
10420 %}
10421
10422 ins_pipe(lmul_reg_reg);
10423 %}
10424
10425 // Combined Integer Multiply & Add/Sub
10426
10427 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10428 match(Set dst (AddI src3 (MulI src1 src2)));
10429
10430 ins_cost(INSN_COST * 3);
10431 format %{ "madd $dst, $src1, $src2, $src3" %}
10432
10433 ins_encode %{
10434 __ maddw(as_Register($dst$$reg),
10435 as_Register($src1$$reg),
10436 as_Register($src2$$reg),
10437 as_Register($src3$$reg));
10438 %}
10439
10440 ins_pipe(imac_reg_reg);
10441 %}
10442
10443 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10444 match(Set dst (SubI src3 (MulI src1 src2)));
10445
10446 ins_cost(INSN_COST * 3);
10447 format %{ "msub $dst, $src1, $src2, $src3" %}
10448
10449 ins_encode %{
10450 __ msubw(as_Register($dst$$reg),
10451 as_Register($src1$$reg),
10452 as_Register($src2$$reg),
10453 as_Register($src3$$reg));
10454 %}
10455
10456 ins_pipe(imac_reg_reg);
10457 %}
10458
10459 // Combined Integer Multiply & Neg
10460
10461 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10462 match(Set dst (MulI (SubI zero src1) src2));
10463
10464 ins_cost(INSN_COST * 3);
10465 format %{ "mneg $dst, $src1, $src2" %}
10466
10467 ins_encode %{
10468 __ mnegw(as_Register($dst$$reg),
10469 as_Register($src1$$reg),
10470 as_Register($src2$$reg));
10471 %}
10472
10473 ins_pipe(imac_reg_reg);
10474 %}
10475
10476 // Combined Long Multiply & Add/Sub
10477
10478 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10479 match(Set dst (AddL src3 (MulL src1 src2)));
10480
10481 ins_cost(INSN_COST * 5);
10482 format %{ "madd $dst, $src1, $src2, $src3" %}
10483
10484 ins_encode %{
10485 __ madd(as_Register($dst$$reg),
10486 as_Register($src1$$reg),
10487 as_Register($src2$$reg),
10488 as_Register($src3$$reg));
10489 %}
10490
10491 ins_pipe(lmac_reg_reg);
10492 %}
10493
10494 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10495 match(Set dst (SubL src3 (MulL src1 src2)));
10496
10497 ins_cost(INSN_COST * 5);
10498 format %{ "msub $dst, $src1, $src2, $src3" %}
10499
10500 ins_encode %{
10501 __ msub(as_Register($dst$$reg),
10502 as_Register($src1$$reg),
10503 as_Register($src2$$reg),
10504 as_Register($src3$$reg));
10505 %}
10506
10507 ins_pipe(lmac_reg_reg);
10508 %}
10509
10510 // Combined Long Multiply & Neg
10511
10512 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10513 match(Set dst (MulL (SubL zero src1) src2));
10514
10515 ins_cost(INSN_COST * 5);
10516 format %{ "mneg $dst, $src1, $src2" %}
10517
10518 ins_encode %{
10519 __ mneg(as_Register($dst$$reg),
10520 as_Register($src1$$reg),
10521 as_Register($src2$$reg));
10522 %}
10523
10524 ins_pipe(lmac_reg_reg);
10525 %}
10526
10527 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10528
10529 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10530 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10531
10532 ins_cost(INSN_COST * 3);
10533 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10534
10535 ins_encode %{
10536 __ smaddl(as_Register($dst$$reg),
10537 as_Register($src1$$reg),
10538 as_Register($src2$$reg),
10539 as_Register($src3$$reg));
10540 %}
10541
10542 ins_pipe(imac_reg_reg);
10543 %}
10544
10545 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10546 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10547
10548 ins_cost(INSN_COST * 3);
10549 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10550
10551 ins_encode %{
10552 __ smsubl(as_Register($dst$$reg),
10553 as_Register($src1$$reg),
10554 as_Register($src2$$reg),
10555 as_Register($src3$$reg));
10556 %}
10557
10558 ins_pipe(imac_reg_reg);
10559 %}
10560
10561 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10562 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10563
10564 ins_cost(INSN_COST * 3);
10565 format %{ "smnegl $dst, $src1, $src2" %}
10566
10567 ins_encode %{
10568 __ smnegl(as_Register($dst$$reg),
10569 as_Register($src1$$reg),
10570 as_Register($src2$$reg));
10571 %}
10572
10573 ins_pipe(imac_reg_reg);
10574 %}
10575
10576 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10577
10578 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10579 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10580
10581 ins_cost(INSN_COST * 5);
10582 format %{ "mulw rscratch1, $src1, $src2\n\t"
10583 "maddw $dst, $src3, $src4, rscratch1" %}
10584
10585 ins_encode %{
10586 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10587 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10588
10589 ins_pipe(imac_reg_reg);
10590 %}
10591
10592 // Integer Divide
10593
10594 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10595 match(Set dst (DivI src1 src2));
10596
10597 ins_cost(INSN_COST * 19);
10598 format %{ "sdivw $dst, $src1, $src2" %}
10599
10600 ins_encode(aarch64_enc_divw(dst, src1, src2));
10601 ins_pipe(idiv_reg_reg);
10602 %}
10603
10604 // Long Divide
10605
10606 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10607 match(Set dst (DivL src1 src2));
10608
10609 ins_cost(INSN_COST * 35);
10610 format %{ "sdiv $dst, $src1, $src2" %}
10611
10612 ins_encode(aarch64_enc_div(dst, src1, src2));
10613 ins_pipe(ldiv_reg_reg);
10614 %}
10615
10616 // Integer Remainder
10617
10618 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10619 match(Set dst (ModI src1 src2));
10620
10621 ins_cost(INSN_COST * 22);
10622 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10623 "msubw $dst, rscratch1, $src2, $src1" %}
10624
10625 ins_encode(aarch64_enc_modw(dst, src1, src2));
10626 ins_pipe(idiv_reg_reg);
10627 %}
10628
10629 // Long Remainder
10630
10631 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10632 match(Set dst (ModL src1 src2));
10633
10634 ins_cost(INSN_COST * 38);
10635 format %{ "sdiv rscratch1, $src1, $src2\n"
10636 "msub $dst, rscratch1, $src2, $src1" %}
10637
10638 ins_encode(aarch64_enc_mod(dst, src1, src2));
10639 ins_pipe(ldiv_reg_reg);
10640 %}
10641
10642 // Unsigned Integer Divide
10643
10644 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10645 match(Set dst (UDivI src1 src2));
10646
10647 ins_cost(INSN_COST * 19);
10648 format %{ "udivw $dst, $src1, $src2" %}
10649
10650 ins_encode %{
10651 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10652 %}
10653
10654 ins_pipe(idiv_reg_reg);
10655 %}
10656
10657 // Unsigned Long Divide
10658
10659 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10660 match(Set dst (UDivL src1 src2));
10661
10662 ins_cost(INSN_COST * 35);
10663 format %{ "udiv $dst, $src1, $src2" %}
10664
10665 ins_encode %{
10666 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10667 %}
10668
10669 ins_pipe(ldiv_reg_reg);
10670 %}
10671
10672 // Unsigned Integer Remainder
10673
10674 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10675 match(Set dst (UModI src1 src2));
10676
10677 ins_cost(INSN_COST * 22);
10678 format %{ "udivw rscratch1, $src1, $src2\n\t"
10679 "msubw $dst, rscratch1, $src2, $src1" %}
10680
10681 ins_encode %{
10682 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10683 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10684 %}
10685
10686 ins_pipe(idiv_reg_reg);
10687 %}
10688
10689 // Unsigned Long Remainder
10690
10691 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10692 match(Set dst (UModL src1 src2));
10693
10694 ins_cost(INSN_COST * 38);
10695 format %{ "udiv rscratch1, $src1, $src2\n"
10696 "msub $dst, rscratch1, $src2, $src1" %}
10697
10698 ins_encode %{
10699 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10700 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10701 %}
10702
10703 ins_pipe(ldiv_reg_reg);
10704 %}
10705
10706 // Integer Shifts
10707
10708 // Shift Left Register
10709 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10710 match(Set dst (LShiftI src1 src2));
10711
10712 ins_cost(INSN_COST * 2);
10713 format %{ "lslvw $dst, $src1, $src2" %}
10714
10715 ins_encode %{
10716 __ lslvw(as_Register($dst$$reg),
10717 as_Register($src1$$reg),
10718 as_Register($src2$$reg));
10719 %}
10720
10721 ins_pipe(ialu_reg_reg_vshift);
10722 %}
10723
10724 // Shift Left Immediate
10725 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10726 match(Set dst (LShiftI src1 src2));
10727
10728 ins_cost(INSN_COST);
10729 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10730
10731 ins_encode %{
10732 __ lslw(as_Register($dst$$reg),
10733 as_Register($src1$$reg),
10734 $src2$$constant & 0x1f);
10735 %}
10736
10737 ins_pipe(ialu_reg_shift);
10738 %}
10739
10740 // Shift Right Logical Register
10741 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10742 match(Set dst (URShiftI src1 src2));
10743
10744 ins_cost(INSN_COST * 2);
10745 format %{ "lsrvw $dst, $src1, $src2" %}
10746
10747 ins_encode %{
10748 __ lsrvw(as_Register($dst$$reg),
10749 as_Register($src1$$reg),
10750 as_Register($src2$$reg));
10751 %}
10752
10753 ins_pipe(ialu_reg_reg_vshift);
10754 %}
10755
10756 // Shift Right Logical Immediate
10757 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10758 match(Set dst (URShiftI src1 src2));
10759
10760 ins_cost(INSN_COST);
10761 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10762
10763 ins_encode %{
10764 __ lsrw(as_Register($dst$$reg),
10765 as_Register($src1$$reg),
10766 $src2$$constant & 0x1f);
10767 %}
10768
10769 ins_pipe(ialu_reg_shift);
10770 %}
10771
10772 // Shift Right Arithmetic Register
10773 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10774 match(Set dst (RShiftI src1 src2));
10775
10776 ins_cost(INSN_COST * 2);
10777 format %{ "asrvw $dst, $src1, $src2" %}
10778
10779 ins_encode %{
10780 __ asrvw(as_Register($dst$$reg),
10781 as_Register($src1$$reg),
10782 as_Register($src2$$reg));
10783 %}
10784
10785 ins_pipe(ialu_reg_reg_vshift);
10786 %}
10787
10788 // Shift Right Arithmetic Immediate
10789 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10790 match(Set dst (RShiftI src1 src2));
10791
10792 ins_cost(INSN_COST);
10793 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10794
10795 ins_encode %{
10796 __ asrw(as_Register($dst$$reg),
10797 as_Register($src1$$reg),
10798 $src2$$constant & 0x1f);
10799 %}
10800
10801 ins_pipe(ialu_reg_shift);
10802 %}
10803
10804 // Combined Int Mask and Right Shift (using UBFM)
10805 // TODO
10806
10807 // Long Shifts
10808
10809 // Shift Left Register
10810 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10811 match(Set dst (LShiftL src1 src2));
10812
10813 ins_cost(INSN_COST * 2);
10814 format %{ "lslv $dst, $src1, $src2" %}
10815
10816 ins_encode %{
10817 __ lslv(as_Register($dst$$reg),
10818 as_Register($src1$$reg),
10819 as_Register($src2$$reg));
10820 %}
10821
10822 ins_pipe(ialu_reg_reg_vshift);
10823 %}
10824
10825 // Shift Left Immediate
10826 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10827 match(Set dst (LShiftL src1 src2));
10828
10829 ins_cost(INSN_COST);
10830 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10831
10832 ins_encode %{
10833 __ lsl(as_Register($dst$$reg),
10834 as_Register($src1$$reg),
10835 $src2$$constant & 0x3f);
10836 %}
10837
10838 ins_pipe(ialu_reg_shift);
10839 %}
10840
10841 // Shift Right Logical Register
10842 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10843 match(Set dst (URShiftL src1 src2));
10844
10845 ins_cost(INSN_COST * 2);
10846 format %{ "lsrv $dst, $src1, $src2" %}
10847
10848 ins_encode %{
10849 __ lsrv(as_Register($dst$$reg),
10850 as_Register($src1$$reg),
10851 as_Register($src2$$reg));
10852 %}
10853
10854 ins_pipe(ialu_reg_reg_vshift);
10855 %}
10856
10857 // Shift Right Logical Immediate
10858 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10859 match(Set dst (URShiftL src1 src2));
10860
10861 ins_cost(INSN_COST);
10862 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10863
10864 ins_encode %{
10865 __ lsr(as_Register($dst$$reg),
10866 as_Register($src1$$reg),
10867 $src2$$constant & 0x3f);
10868 %}
10869
10870 ins_pipe(ialu_reg_shift);
10871 %}
10872
10873 // A special-case pattern for card table stores.
10874 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10875 match(Set dst (URShiftL (CastP2X src1) src2));
10876
10877 ins_cost(INSN_COST);
10878 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10879
10880 ins_encode %{
10881 __ lsr(as_Register($dst$$reg),
10882 as_Register($src1$$reg),
10883 $src2$$constant & 0x3f);
10884 %}
10885
10886 ins_pipe(ialu_reg_shift);
10887 %}
10888
10889 // Shift Right Arithmetic Register
10890 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10891 match(Set dst (RShiftL src1 src2));
10892
10893 ins_cost(INSN_COST * 2);
10894 format %{ "asrv $dst, $src1, $src2" %}
10895
10896 ins_encode %{
10897 __ asrv(as_Register($dst$$reg),
10898 as_Register($src1$$reg),
10899 as_Register($src2$$reg));
10900 %}
10901
10902 ins_pipe(ialu_reg_reg_vshift);
10903 %}
10904
10905 // Shift Right Arithmetic Immediate
10906 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10907 match(Set dst (RShiftL src1 src2));
10908
10909 ins_cost(INSN_COST);
10910 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10911
10912 ins_encode %{
10913 __ asr(as_Register($dst$$reg),
10914 as_Register($src1$$reg),
10915 $src2$$constant & 0x3f);
10916 %}
10917
10918 ins_pipe(ialu_reg_shift);
10919 %}
10920
10921 // BEGIN This section of the file is automatically generated. Do not edit --------------
10922 // This section is generated from aarch64_ad.m4
10923
10924 // This pattern is automatically generated from aarch64_ad.m4
10925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10926 instruct regL_not_reg(iRegLNoSp dst,
10927 iRegL src1, immL_M1 m1,
10928 rFlagsReg cr) %{
10929 match(Set dst (XorL src1 m1));
10930 ins_cost(INSN_COST);
10931 format %{ "eon $dst, $src1, zr" %}
10932
10933 ins_encode %{
10934 __ eon(as_Register($dst$$reg),
10935 as_Register($src1$$reg),
10936 zr,
10937 Assembler::LSL, 0);
10938 %}
10939
10940 ins_pipe(ialu_reg);
10941 %}
10942
10943 // This pattern is automatically generated from aarch64_ad.m4
10944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10945 instruct regI_not_reg(iRegINoSp dst,
10946 iRegIorL2I src1, immI_M1 m1,
10947 rFlagsReg cr) %{
10948 match(Set dst (XorI src1 m1));
10949 ins_cost(INSN_COST);
10950 format %{ "eonw $dst, $src1, zr" %}
10951
10952 ins_encode %{
10953 __ eonw(as_Register($dst$$reg),
10954 as_Register($src1$$reg),
10955 zr,
10956 Assembler::LSL, 0);
10957 %}
10958
10959 ins_pipe(ialu_reg);
10960 %}
10961
10962 // This pattern is automatically generated from aarch64_ad.m4
10963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10964 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10965 immI0 zero, iRegIorL2I src1, immI src2) %{
10966 match(Set dst (SubI zero (URShiftI src1 src2)));
10967
10968 ins_cost(1.9 * INSN_COST);
10969 format %{ "negw $dst, $src1, LSR $src2" %}
10970
10971 ins_encode %{
10972 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10973 Assembler::LSR, $src2$$constant & 0x1f);
10974 %}
10975
10976 ins_pipe(ialu_reg_shift);
10977 %}
10978
10979 // This pattern is automatically generated from aarch64_ad.m4
10980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10981 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10982 immI0 zero, iRegIorL2I src1, immI src2) %{
10983 match(Set dst (SubI zero (RShiftI src1 src2)));
10984
10985 ins_cost(1.9 * INSN_COST);
10986 format %{ "negw $dst, $src1, ASR $src2" %}
10987
10988 ins_encode %{
10989 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10990 Assembler::ASR, $src2$$constant & 0x1f);
10991 %}
10992
10993 ins_pipe(ialu_reg_shift);
10994 %}
10995
10996 // This pattern is automatically generated from aarch64_ad.m4
10997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10998 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10999 immI0 zero, iRegIorL2I src1, immI src2) %{
11000 match(Set dst (SubI zero (LShiftI src1 src2)));
11001
11002 ins_cost(1.9 * INSN_COST);
11003 format %{ "negw $dst, $src1, LSL $src2" %}
11004
11005 ins_encode %{
11006 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11007 Assembler::LSL, $src2$$constant & 0x1f);
11008 %}
11009
11010 ins_pipe(ialu_reg_shift);
11011 %}
11012
11013 // This pattern is automatically generated from aarch64_ad.m4
11014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11015 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11016 immL0 zero, iRegL src1, immI src2) %{
11017 match(Set dst (SubL zero (URShiftL src1 src2)));
11018
11019 ins_cost(1.9 * INSN_COST);
11020 format %{ "neg $dst, $src1, LSR $src2" %}
11021
11022 ins_encode %{
11023 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11024 Assembler::LSR, $src2$$constant & 0x3f);
11025 %}
11026
11027 ins_pipe(ialu_reg_shift);
11028 %}
11029
11030 // This pattern is automatically generated from aarch64_ad.m4
11031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11032 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11033 immL0 zero, iRegL src1, immI src2) %{
11034 match(Set dst (SubL zero (RShiftL src1 src2)));
11035
11036 ins_cost(1.9 * INSN_COST);
11037 format %{ "neg $dst, $src1, ASR $src2" %}
11038
11039 ins_encode %{
11040 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11041 Assembler::ASR, $src2$$constant & 0x3f);
11042 %}
11043
11044 ins_pipe(ialu_reg_shift);
11045 %}
11046
11047 // This pattern is automatically generated from aarch64_ad.m4
11048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11049 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11050 immL0 zero, iRegL src1, immI src2) %{
11051 match(Set dst (SubL zero (LShiftL src1 src2)));
11052
11053 ins_cost(1.9 * INSN_COST);
11054 format %{ "neg $dst, $src1, LSL $src2" %}
11055
11056 ins_encode %{
11057 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11058 Assembler::LSL, $src2$$constant & 0x3f);
11059 %}
11060
11061 ins_pipe(ialu_reg_shift);
11062 %}
11063
11064 // This pattern is automatically generated from aarch64_ad.m4
11065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11066 instruct AndI_reg_not_reg(iRegINoSp dst,
11067 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11068 match(Set dst (AndI src1 (XorI src2 m1)));
11069 ins_cost(INSN_COST);
11070 format %{ "bicw $dst, $src1, $src2" %}
11071
11072 ins_encode %{
11073 __ bicw(as_Register($dst$$reg),
11074 as_Register($src1$$reg),
11075 as_Register($src2$$reg),
11076 Assembler::LSL, 0);
11077 %}
11078
11079 ins_pipe(ialu_reg_reg);
11080 %}
11081
11082 // This pattern is automatically generated from aarch64_ad.m4
11083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11084 instruct AndL_reg_not_reg(iRegLNoSp dst,
11085 iRegL src1, iRegL src2, immL_M1 m1) %{
11086 match(Set dst (AndL src1 (XorL src2 m1)));
11087 ins_cost(INSN_COST);
11088 format %{ "bic $dst, $src1, $src2" %}
11089
11090 ins_encode %{
11091 __ bic(as_Register($dst$$reg),
11092 as_Register($src1$$reg),
11093 as_Register($src2$$reg),
11094 Assembler::LSL, 0);
11095 %}
11096
11097 ins_pipe(ialu_reg_reg);
11098 %}
11099
11100 // This pattern is automatically generated from aarch64_ad.m4
11101 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11102 instruct OrI_reg_not_reg(iRegINoSp dst,
11103 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11104 match(Set dst (OrI src1 (XorI src2 m1)));
11105 ins_cost(INSN_COST);
11106 format %{ "ornw $dst, $src1, $src2" %}
11107
11108 ins_encode %{
11109 __ ornw(as_Register($dst$$reg),
11110 as_Register($src1$$reg),
11111 as_Register($src2$$reg),
11112 Assembler::LSL, 0);
11113 %}
11114
11115 ins_pipe(ialu_reg_reg);
11116 %}
11117
11118 // This pattern is automatically generated from aarch64_ad.m4
11119 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11120 instruct OrL_reg_not_reg(iRegLNoSp dst,
11121 iRegL src1, iRegL src2, immL_M1 m1) %{
11122 match(Set dst (OrL src1 (XorL src2 m1)));
11123 ins_cost(INSN_COST);
11124 format %{ "orn $dst, $src1, $src2" %}
11125
11126 ins_encode %{
11127 __ orn(as_Register($dst$$reg),
11128 as_Register($src1$$reg),
11129 as_Register($src2$$reg),
11130 Assembler::LSL, 0);
11131 %}
11132
11133 ins_pipe(ialu_reg_reg);
11134 %}
11135
11136 // This pattern is automatically generated from aarch64_ad.m4
11137 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11138 instruct XorI_reg_not_reg(iRegINoSp dst,
11139 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11140 match(Set dst (XorI m1 (XorI src2 src1)));
11141 ins_cost(INSN_COST);
11142 format %{ "eonw $dst, $src1, $src2" %}
11143
11144 ins_encode %{
11145 __ eonw(as_Register($dst$$reg),
11146 as_Register($src1$$reg),
11147 as_Register($src2$$reg),
11148 Assembler::LSL, 0);
11149 %}
11150
11151 ins_pipe(ialu_reg_reg);
11152 %}
11153
11154 // This pattern is automatically generated from aarch64_ad.m4
11155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11156 instruct XorL_reg_not_reg(iRegLNoSp dst,
11157 iRegL src1, iRegL src2, immL_M1 m1) %{
11158 match(Set dst (XorL m1 (XorL src2 src1)));
11159 ins_cost(INSN_COST);
11160 format %{ "eon $dst, $src1, $src2" %}
11161
11162 ins_encode %{
11163 __ eon(as_Register($dst$$reg),
11164 as_Register($src1$$reg),
11165 as_Register($src2$$reg),
11166 Assembler::LSL, 0);
11167 %}
11168
11169 ins_pipe(ialu_reg_reg);
11170 %}
11171
11172 // This pattern is automatically generated from aarch64_ad.m4
11173 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11174 // val & (-1 ^ (val >>> shift)) ==> bicw
11175 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11176 iRegIorL2I src1, iRegIorL2I src2,
11177 immI src3, immI_M1 src4) %{
11178 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11179 ins_cost(1.9 * INSN_COST);
11180 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11181
11182 ins_encode %{
11183 __ bicw(as_Register($dst$$reg),
11184 as_Register($src1$$reg),
11185 as_Register($src2$$reg),
11186 Assembler::LSR,
11187 $src3$$constant & 0x1f);
11188 %}
11189
11190 ins_pipe(ialu_reg_reg_shift);
11191 %}
11192
11193 // This pattern is automatically generated from aarch64_ad.m4
11194 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11195 // val & (-1 ^ (val >>> shift)) ==> bic
11196 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11197 iRegL src1, iRegL src2,
11198 immI src3, immL_M1 src4) %{
11199 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11200 ins_cost(1.9 * INSN_COST);
11201 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11202
11203 ins_encode %{
11204 __ bic(as_Register($dst$$reg),
11205 as_Register($src1$$reg),
11206 as_Register($src2$$reg),
11207 Assembler::LSR,
11208 $src3$$constant & 0x3f);
11209 %}
11210
11211 ins_pipe(ialu_reg_reg_shift);
11212 %}
11213
11214 // This pattern is automatically generated from aarch64_ad.m4
11215 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11216 // val & (-1 ^ (val >> shift)) ==> bicw
11217 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11218 iRegIorL2I src1, iRegIorL2I src2,
11219 immI src3, immI_M1 src4) %{
11220 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11221 ins_cost(1.9 * INSN_COST);
11222 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11223
11224 ins_encode %{
11225 __ bicw(as_Register($dst$$reg),
11226 as_Register($src1$$reg),
11227 as_Register($src2$$reg),
11228 Assembler::ASR,
11229 $src3$$constant & 0x1f);
11230 %}
11231
11232 ins_pipe(ialu_reg_reg_shift);
11233 %}
11234
11235 // This pattern is automatically generated from aarch64_ad.m4
11236 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11237 // val & (-1 ^ (val >> shift)) ==> bic
11238 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11239 iRegL src1, iRegL src2,
11240 immI src3, immL_M1 src4) %{
11241 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11242 ins_cost(1.9 * INSN_COST);
11243 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11244
11245 ins_encode %{
11246 __ bic(as_Register($dst$$reg),
11247 as_Register($src1$$reg),
11248 as_Register($src2$$reg),
11249 Assembler::ASR,
11250 $src3$$constant & 0x3f);
11251 %}
11252
11253 ins_pipe(ialu_reg_reg_shift);
11254 %}
11255
11256 // This pattern is automatically generated from aarch64_ad.m4
11257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11258 // val & (-1 ^ (val ror shift)) ==> bicw
11259 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11260 iRegIorL2I src1, iRegIorL2I src2,
11261 immI src3, immI_M1 src4) %{
11262 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11263 ins_cost(1.9 * INSN_COST);
11264 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11265
11266 ins_encode %{
11267 __ bicw(as_Register($dst$$reg),
11268 as_Register($src1$$reg),
11269 as_Register($src2$$reg),
11270 Assembler::ROR,
11271 $src3$$constant & 0x1f);
11272 %}
11273
11274 ins_pipe(ialu_reg_reg_shift);
11275 %}
11276
11277 // This pattern is automatically generated from aarch64_ad.m4
11278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11279 // val & (-1 ^ (val ror shift)) ==> bic
11280 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11281 iRegL src1, iRegL src2,
11282 immI src3, immL_M1 src4) %{
11283 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11284 ins_cost(1.9 * INSN_COST);
11285 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11286
11287 ins_encode %{
11288 __ bic(as_Register($dst$$reg),
11289 as_Register($src1$$reg),
11290 as_Register($src2$$reg),
11291 Assembler::ROR,
11292 $src3$$constant & 0x3f);
11293 %}
11294
11295 ins_pipe(ialu_reg_reg_shift);
11296 %}
11297
11298 // This pattern is automatically generated from aarch64_ad.m4
11299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11300 // val & (-1 ^ (val << shift)) ==> bicw
11301 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11302 iRegIorL2I src1, iRegIorL2I src2,
11303 immI src3, immI_M1 src4) %{
11304 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11305 ins_cost(1.9 * INSN_COST);
11306 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11307
11308 ins_encode %{
11309 __ bicw(as_Register($dst$$reg),
11310 as_Register($src1$$reg),
11311 as_Register($src2$$reg),
11312 Assembler::LSL,
11313 $src3$$constant & 0x1f);
11314 %}
11315
11316 ins_pipe(ialu_reg_reg_shift);
11317 %}
11318
11319 // This pattern is automatically generated from aarch64_ad.m4
11320 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11321 // val & (-1 ^ (val << shift)) ==> bic
11322 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11323 iRegL src1, iRegL src2,
11324 immI src3, immL_M1 src4) %{
11325 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11326 ins_cost(1.9 * INSN_COST);
11327 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11328
11329 ins_encode %{
11330 __ bic(as_Register($dst$$reg),
11331 as_Register($src1$$reg),
11332 as_Register($src2$$reg),
11333 Assembler::LSL,
11334 $src3$$constant & 0x3f);
11335 %}
11336
11337 ins_pipe(ialu_reg_reg_shift);
11338 %}
11339
11340 // This pattern is automatically generated from aarch64_ad.m4
11341 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11342 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11343 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11344 iRegIorL2I src1, iRegIorL2I src2,
11345 immI src3, immI_M1 src4) %{
11346 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11347 ins_cost(1.9 * INSN_COST);
11348 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11349
11350 ins_encode %{
11351 __ eonw(as_Register($dst$$reg),
11352 as_Register($src1$$reg),
11353 as_Register($src2$$reg),
11354 Assembler::LSR,
11355 $src3$$constant & 0x1f);
11356 %}
11357
11358 ins_pipe(ialu_reg_reg_shift);
11359 %}
11360
11361 // This pattern is automatically generated from aarch64_ad.m4
11362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11363 // val ^ (-1 ^ (val >>> shift)) ==> eon
11364 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11365 iRegL src1, iRegL src2,
11366 immI src3, immL_M1 src4) %{
11367 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11368 ins_cost(1.9 * INSN_COST);
11369 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11370
11371 ins_encode %{
11372 __ eon(as_Register($dst$$reg),
11373 as_Register($src1$$reg),
11374 as_Register($src2$$reg),
11375 Assembler::LSR,
11376 $src3$$constant & 0x3f);
11377 %}
11378
11379 ins_pipe(ialu_reg_reg_shift);
11380 %}
11381
11382 // This pattern is automatically generated from aarch64_ad.m4
11383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11384 // val ^ (-1 ^ (val >> shift)) ==> eonw
11385 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11386 iRegIorL2I src1, iRegIorL2I src2,
11387 immI src3, immI_M1 src4) %{
11388 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11389 ins_cost(1.9 * INSN_COST);
11390 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11391
11392 ins_encode %{
11393 __ eonw(as_Register($dst$$reg),
11394 as_Register($src1$$reg),
11395 as_Register($src2$$reg),
11396 Assembler::ASR,
11397 $src3$$constant & 0x1f);
11398 %}
11399
11400 ins_pipe(ialu_reg_reg_shift);
11401 %}
11402
11403 // This pattern is automatically generated from aarch64_ad.m4
11404 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11405 // val ^ (-1 ^ (val >> shift)) ==> eon
11406 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11407 iRegL src1, iRegL src2,
11408 immI src3, immL_M1 src4) %{
11409 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11410 ins_cost(1.9 * INSN_COST);
11411 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11412
11413 ins_encode %{
11414 __ eon(as_Register($dst$$reg),
11415 as_Register($src1$$reg),
11416 as_Register($src2$$reg),
11417 Assembler::ASR,
11418 $src3$$constant & 0x3f);
11419 %}
11420
11421 ins_pipe(ialu_reg_reg_shift);
11422 %}
11423
11424 // This pattern is automatically generated from aarch64_ad.m4
11425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11426 // val ^ (-1 ^ (val ror shift)) ==> eonw
11427 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11428 iRegIorL2I src1, iRegIorL2I src2,
11429 immI src3, immI_M1 src4) %{
11430 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11431 ins_cost(1.9 * INSN_COST);
11432 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11433
11434 ins_encode %{
11435 __ eonw(as_Register($dst$$reg),
11436 as_Register($src1$$reg),
11437 as_Register($src2$$reg),
11438 Assembler::ROR,
11439 $src3$$constant & 0x1f);
11440 %}
11441
11442 ins_pipe(ialu_reg_reg_shift);
11443 %}
11444
11445 // This pattern is automatically generated from aarch64_ad.m4
11446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11447 // val ^ (-1 ^ (val ror shift)) ==> eon
11448 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11449 iRegL src1, iRegL src2,
11450 immI src3, immL_M1 src4) %{
11451 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11452 ins_cost(1.9 * INSN_COST);
11453 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11454
11455 ins_encode %{
11456 __ eon(as_Register($dst$$reg),
11457 as_Register($src1$$reg),
11458 as_Register($src2$$reg),
11459 Assembler::ROR,
11460 $src3$$constant & 0x3f);
11461 %}
11462
11463 ins_pipe(ialu_reg_reg_shift);
11464 %}
11465
11466 // This pattern is automatically generated from aarch64_ad.m4
11467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11468 // val ^ (-1 ^ (val << shift)) ==> eonw
11469 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11470 iRegIorL2I src1, iRegIorL2I src2,
11471 immI src3, immI_M1 src4) %{
11472 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11473 ins_cost(1.9 * INSN_COST);
11474 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11475
11476 ins_encode %{
11477 __ eonw(as_Register($dst$$reg),
11478 as_Register($src1$$reg),
11479 as_Register($src2$$reg),
11480 Assembler::LSL,
11481 $src3$$constant & 0x1f);
11482 %}
11483
11484 ins_pipe(ialu_reg_reg_shift);
11485 %}
11486
11487 // This pattern is automatically generated from aarch64_ad.m4
11488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11489 // val ^ (-1 ^ (val << shift)) ==> eon
11490 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11491 iRegL src1, iRegL src2,
11492 immI src3, immL_M1 src4) %{
11493 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11494 ins_cost(1.9 * INSN_COST);
11495 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11496
11497 ins_encode %{
11498 __ eon(as_Register($dst$$reg),
11499 as_Register($src1$$reg),
11500 as_Register($src2$$reg),
11501 Assembler::LSL,
11502 $src3$$constant & 0x3f);
11503 %}
11504
11505 ins_pipe(ialu_reg_reg_shift);
11506 %}
11507
11508 // This pattern is automatically generated from aarch64_ad.m4
11509 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11510 // val | (-1 ^ (val >>> shift)) ==> ornw
11511 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11512 iRegIorL2I src1, iRegIorL2I src2,
11513 immI src3, immI_M1 src4) %{
11514 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11515 ins_cost(1.9 * INSN_COST);
11516 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11517
11518 ins_encode %{
11519 __ ornw(as_Register($dst$$reg),
11520 as_Register($src1$$reg),
11521 as_Register($src2$$reg),
11522 Assembler::LSR,
11523 $src3$$constant & 0x1f);
11524 %}
11525
11526 ins_pipe(ialu_reg_reg_shift);
11527 %}
11528
11529 // This pattern is automatically generated from aarch64_ad.m4
11530 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11531 // val | (-1 ^ (val >>> shift)) ==> orn
11532 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11533 iRegL src1, iRegL src2,
11534 immI src3, immL_M1 src4) %{
11535 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11536 ins_cost(1.9 * INSN_COST);
11537 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11538
11539 ins_encode %{
11540 __ orn(as_Register($dst$$reg),
11541 as_Register($src1$$reg),
11542 as_Register($src2$$reg),
11543 Assembler::LSR,
11544 $src3$$constant & 0x3f);
11545 %}
11546
11547 ins_pipe(ialu_reg_reg_shift);
11548 %}
11549
11550 // This pattern is automatically generated from aarch64_ad.m4
11551 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11552 // val | (-1 ^ (val >> shift)) ==> ornw
11553 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11554 iRegIorL2I src1, iRegIorL2I src2,
11555 immI src3, immI_M1 src4) %{
11556 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11557 ins_cost(1.9 * INSN_COST);
11558 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11559
11560 ins_encode %{
11561 __ ornw(as_Register($dst$$reg),
11562 as_Register($src1$$reg),
11563 as_Register($src2$$reg),
11564 Assembler::ASR,
11565 $src3$$constant & 0x1f);
11566 %}
11567
11568 ins_pipe(ialu_reg_reg_shift);
11569 %}
11570
11571 // This pattern is automatically generated from aarch64_ad.m4
11572 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11573 // val | (-1 ^ (val >> shift)) ==> orn
11574 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11575 iRegL src1, iRegL src2,
11576 immI src3, immL_M1 src4) %{
11577 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11578 ins_cost(1.9 * INSN_COST);
11579 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11580
11581 ins_encode %{
11582 __ orn(as_Register($dst$$reg),
11583 as_Register($src1$$reg),
11584 as_Register($src2$$reg),
11585 Assembler::ASR,
11586 $src3$$constant & 0x3f);
11587 %}
11588
11589 ins_pipe(ialu_reg_reg_shift);
11590 %}
11591
11592 // This pattern is automatically generated from aarch64_ad.m4
11593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11594 // val | (-1 ^ (val ror shift)) ==> ornw
11595 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11596 iRegIorL2I src1, iRegIorL2I src2,
11597 immI src3, immI_M1 src4) %{
11598 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11599 ins_cost(1.9 * INSN_COST);
11600 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11601
11602 ins_encode %{
11603 __ ornw(as_Register($dst$$reg),
11604 as_Register($src1$$reg),
11605 as_Register($src2$$reg),
11606 Assembler::ROR,
11607 $src3$$constant & 0x1f);
11608 %}
11609
11610 ins_pipe(ialu_reg_reg_shift);
11611 %}
11612
11613 // This pattern is automatically generated from aarch64_ad.m4
11614 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11615 // val | (-1 ^ (val ror shift)) ==> orn
11616 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11617 iRegL src1, iRegL src2,
11618 immI src3, immL_M1 src4) %{
11619 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11620 ins_cost(1.9 * INSN_COST);
11621 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11622
11623 ins_encode %{
11624 __ orn(as_Register($dst$$reg),
11625 as_Register($src1$$reg),
11626 as_Register($src2$$reg),
11627 Assembler::ROR,
11628 $src3$$constant & 0x3f);
11629 %}
11630
11631 ins_pipe(ialu_reg_reg_shift);
11632 %}
11633
11634 // This pattern is automatically generated from aarch64_ad.m4
11635 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11636 // val | (-1 ^ (val << shift)) ==> ornw
11637 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11638 iRegIorL2I src1, iRegIorL2I src2,
11639 immI src3, immI_M1 src4) %{
11640 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11641 ins_cost(1.9 * INSN_COST);
11642 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11643
11644 ins_encode %{
11645 __ ornw(as_Register($dst$$reg),
11646 as_Register($src1$$reg),
11647 as_Register($src2$$reg),
11648 Assembler::LSL,
11649 $src3$$constant & 0x1f);
11650 %}
11651
11652 ins_pipe(ialu_reg_reg_shift);
11653 %}
11654
11655 // This pattern is automatically generated from aarch64_ad.m4
11656 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11657 // val | (-1 ^ (val << shift)) ==> orn
11658 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11659 iRegL src1, iRegL src2,
11660 immI src3, immL_M1 src4) %{
11661 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11662 ins_cost(1.9 * INSN_COST);
11663 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11664
11665 ins_encode %{
11666 __ orn(as_Register($dst$$reg),
11667 as_Register($src1$$reg),
11668 as_Register($src2$$reg),
11669 Assembler::LSL,
11670 $src3$$constant & 0x3f);
11671 %}
11672
11673 ins_pipe(ialu_reg_reg_shift);
11674 %}
11675
11676 // This pattern is automatically generated from aarch64_ad.m4
11677 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11678 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11679 iRegIorL2I src1, iRegIorL2I src2,
11680 immI src3) %{
11681 match(Set dst (AndI src1 (URShiftI src2 src3)));
11682
11683 ins_cost(1.9 * INSN_COST);
11684 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11685
11686 ins_encode %{
11687 __ andw(as_Register($dst$$reg),
11688 as_Register($src1$$reg),
11689 as_Register($src2$$reg),
11690 Assembler::LSR,
11691 $src3$$constant & 0x1f);
11692 %}
11693
11694 ins_pipe(ialu_reg_reg_shift);
11695 %}
11696
11697 // This pattern is automatically generated from aarch64_ad.m4
11698 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11699 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11700 iRegL src1, iRegL src2,
11701 immI src3) %{
11702 match(Set dst (AndL src1 (URShiftL src2 src3)));
11703
11704 ins_cost(1.9 * INSN_COST);
11705 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11706
11707 ins_encode %{
11708 __ andr(as_Register($dst$$reg),
11709 as_Register($src1$$reg),
11710 as_Register($src2$$reg),
11711 Assembler::LSR,
11712 $src3$$constant & 0x3f);
11713 %}
11714
11715 ins_pipe(ialu_reg_reg_shift);
11716 %}
11717
11718 // This pattern is automatically generated from aarch64_ad.m4
11719 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11720 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11721 iRegIorL2I src1, iRegIorL2I src2,
11722 immI src3) %{
11723 match(Set dst (AndI src1 (RShiftI src2 src3)));
11724
11725 ins_cost(1.9 * INSN_COST);
11726 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11727
11728 ins_encode %{
11729 __ andw(as_Register($dst$$reg),
11730 as_Register($src1$$reg),
11731 as_Register($src2$$reg),
11732 Assembler::ASR,
11733 $src3$$constant & 0x1f);
11734 %}
11735
11736 ins_pipe(ialu_reg_reg_shift);
11737 %}
11738
11739 // This pattern is automatically generated from aarch64_ad.m4
11740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11741 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11742 iRegL src1, iRegL src2,
11743 immI src3) %{
11744 match(Set dst (AndL src1 (RShiftL src2 src3)));
11745
11746 ins_cost(1.9 * INSN_COST);
11747 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11748
11749 ins_encode %{
11750 __ andr(as_Register($dst$$reg),
11751 as_Register($src1$$reg),
11752 as_Register($src2$$reg),
11753 Assembler::ASR,
11754 $src3$$constant & 0x3f);
11755 %}
11756
11757 ins_pipe(ialu_reg_reg_shift);
11758 %}
11759
11760 // This pattern is automatically generated from aarch64_ad.m4
11761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11762 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11763 iRegIorL2I src1, iRegIorL2I src2,
11764 immI src3) %{
11765 match(Set dst (AndI src1 (LShiftI src2 src3)));
11766
11767 ins_cost(1.9 * INSN_COST);
11768 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11769
11770 ins_encode %{
11771 __ andw(as_Register($dst$$reg),
11772 as_Register($src1$$reg),
11773 as_Register($src2$$reg),
11774 Assembler::LSL,
11775 $src3$$constant & 0x1f);
11776 %}
11777
11778 ins_pipe(ialu_reg_reg_shift);
11779 %}
11780
11781 // This pattern is automatically generated from aarch64_ad.m4
11782 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11783 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11784 iRegL src1, iRegL src2,
11785 immI src3) %{
11786 match(Set dst (AndL src1 (LShiftL src2 src3)));
11787
11788 ins_cost(1.9 * INSN_COST);
11789 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11790
11791 ins_encode %{
11792 __ andr(as_Register($dst$$reg),
11793 as_Register($src1$$reg),
11794 as_Register($src2$$reg),
11795 Assembler::LSL,
11796 $src3$$constant & 0x3f);
11797 %}
11798
11799 ins_pipe(ialu_reg_reg_shift);
11800 %}
11801
11802 // This pattern is automatically generated from aarch64_ad.m4
11803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11804 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11805 iRegIorL2I src1, iRegIorL2I src2,
11806 immI src3) %{
11807 match(Set dst (AndI src1 (RotateRight src2 src3)));
11808
11809 ins_cost(1.9 * INSN_COST);
11810 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11811
11812 ins_encode %{
11813 __ andw(as_Register($dst$$reg),
11814 as_Register($src1$$reg),
11815 as_Register($src2$$reg),
11816 Assembler::ROR,
11817 $src3$$constant & 0x1f);
11818 %}
11819
11820 ins_pipe(ialu_reg_reg_shift);
11821 %}
11822
11823 // This pattern is automatically generated from aarch64_ad.m4
11824 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11825 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11826 iRegL src1, iRegL src2,
11827 immI src3) %{
11828 match(Set dst (AndL src1 (RotateRight src2 src3)));
11829
11830 ins_cost(1.9 * INSN_COST);
11831 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11832
11833 ins_encode %{
11834 __ andr(as_Register($dst$$reg),
11835 as_Register($src1$$reg),
11836 as_Register($src2$$reg),
11837 Assembler::ROR,
11838 $src3$$constant & 0x3f);
11839 %}
11840
11841 ins_pipe(ialu_reg_reg_shift);
11842 %}
11843
11844 // This pattern is automatically generated from aarch64_ad.m4
11845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11846 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11847 iRegIorL2I src1, iRegIorL2I src2,
11848 immI src3) %{
11849 match(Set dst (XorI src1 (URShiftI src2 src3)));
11850
11851 ins_cost(1.9 * INSN_COST);
11852 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11853
11854 ins_encode %{
11855 __ eorw(as_Register($dst$$reg),
11856 as_Register($src1$$reg),
11857 as_Register($src2$$reg),
11858 Assembler::LSR,
11859 $src3$$constant & 0x1f);
11860 %}
11861
11862 ins_pipe(ialu_reg_reg_shift);
11863 %}
11864
11865 // This pattern is automatically generated from aarch64_ad.m4
11866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11867 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11868 iRegL src1, iRegL src2,
11869 immI src3) %{
11870 match(Set dst (XorL src1 (URShiftL src2 src3)));
11871
11872 ins_cost(1.9 * INSN_COST);
11873 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11874
11875 ins_encode %{
11876 __ eor(as_Register($dst$$reg),
11877 as_Register($src1$$reg),
11878 as_Register($src2$$reg),
11879 Assembler::LSR,
11880 $src3$$constant & 0x3f);
11881 %}
11882
11883 ins_pipe(ialu_reg_reg_shift);
11884 %}
11885
11886 // This pattern is automatically generated from aarch64_ad.m4
11887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11888 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11889 iRegIorL2I src1, iRegIorL2I src2,
11890 immI src3) %{
11891 match(Set dst (XorI src1 (RShiftI src2 src3)));
11892
11893 ins_cost(1.9 * INSN_COST);
11894 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11895
11896 ins_encode %{
11897 __ eorw(as_Register($dst$$reg),
11898 as_Register($src1$$reg),
11899 as_Register($src2$$reg),
11900 Assembler::ASR,
11901 $src3$$constant & 0x1f);
11902 %}
11903
11904 ins_pipe(ialu_reg_reg_shift);
11905 %}
11906
11907 // This pattern is automatically generated from aarch64_ad.m4
11908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11909 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11910 iRegL src1, iRegL src2,
11911 immI src3) %{
11912 match(Set dst (XorL src1 (RShiftL src2 src3)));
11913
11914 ins_cost(1.9 * INSN_COST);
11915 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11916
11917 ins_encode %{
11918 __ eor(as_Register($dst$$reg),
11919 as_Register($src1$$reg),
11920 as_Register($src2$$reg),
11921 Assembler::ASR,
11922 $src3$$constant & 0x3f);
11923 %}
11924
11925 ins_pipe(ialu_reg_reg_shift);
11926 %}
11927
11928 // This pattern is automatically generated from aarch64_ad.m4
11929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11930 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11931 iRegIorL2I src1, iRegIorL2I src2,
11932 immI src3) %{
11933 match(Set dst (XorI src1 (LShiftI src2 src3)));
11934
11935 ins_cost(1.9 * INSN_COST);
11936 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11937
11938 ins_encode %{
11939 __ eorw(as_Register($dst$$reg),
11940 as_Register($src1$$reg),
11941 as_Register($src2$$reg),
11942 Assembler::LSL,
11943 $src3$$constant & 0x1f);
11944 %}
11945
11946 ins_pipe(ialu_reg_reg_shift);
11947 %}
11948
11949 // This pattern is automatically generated from aarch64_ad.m4
11950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11951 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11952 iRegL src1, iRegL src2,
11953 immI src3) %{
11954 match(Set dst (XorL src1 (LShiftL src2 src3)));
11955
11956 ins_cost(1.9 * INSN_COST);
11957 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11958
11959 ins_encode %{
11960 __ eor(as_Register($dst$$reg),
11961 as_Register($src1$$reg),
11962 as_Register($src2$$reg),
11963 Assembler::LSL,
11964 $src3$$constant & 0x3f);
11965 %}
11966
11967 ins_pipe(ialu_reg_reg_shift);
11968 %}
11969
11970 // This pattern is automatically generated from aarch64_ad.m4
11971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11972 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11973 iRegIorL2I src1, iRegIorL2I src2,
11974 immI src3) %{
11975 match(Set dst (XorI src1 (RotateRight src2 src3)));
11976
11977 ins_cost(1.9 * INSN_COST);
11978 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11979
11980 ins_encode %{
11981 __ eorw(as_Register($dst$$reg),
11982 as_Register($src1$$reg),
11983 as_Register($src2$$reg),
11984 Assembler::ROR,
11985 $src3$$constant & 0x1f);
11986 %}
11987
11988 ins_pipe(ialu_reg_reg_shift);
11989 %}
11990
11991 // This pattern is automatically generated from aarch64_ad.m4
11992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11993 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11994 iRegL src1, iRegL src2,
11995 immI src3) %{
11996 match(Set dst (XorL src1 (RotateRight src2 src3)));
11997
11998 ins_cost(1.9 * INSN_COST);
11999 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
12000
12001 ins_encode %{
12002 __ eor(as_Register($dst$$reg),
12003 as_Register($src1$$reg),
12004 as_Register($src2$$reg),
12005 Assembler::ROR,
12006 $src3$$constant & 0x3f);
12007 %}
12008
12009 ins_pipe(ialu_reg_reg_shift);
12010 %}
12011
12012 // This pattern is automatically generated from aarch64_ad.m4
12013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12014 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12015 iRegIorL2I src1, iRegIorL2I src2,
12016 immI src3) %{
12017 match(Set dst (OrI src1 (URShiftI src2 src3)));
12018
12019 ins_cost(1.9 * INSN_COST);
12020 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
12021
12022 ins_encode %{
12023 __ orrw(as_Register($dst$$reg),
12024 as_Register($src1$$reg),
12025 as_Register($src2$$reg),
12026 Assembler::LSR,
12027 $src3$$constant & 0x1f);
12028 %}
12029
12030 ins_pipe(ialu_reg_reg_shift);
12031 %}
12032
12033 // This pattern is automatically generated from aarch64_ad.m4
12034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12035 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12036 iRegL src1, iRegL src2,
12037 immI src3) %{
12038 match(Set dst (OrL src1 (URShiftL src2 src3)));
12039
12040 ins_cost(1.9 * INSN_COST);
12041 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12042
12043 ins_encode %{
12044 __ orr(as_Register($dst$$reg),
12045 as_Register($src1$$reg),
12046 as_Register($src2$$reg),
12047 Assembler::LSR,
12048 $src3$$constant & 0x3f);
12049 %}
12050
12051 ins_pipe(ialu_reg_reg_shift);
12052 %}
12053
12054 // This pattern is automatically generated from aarch64_ad.m4
12055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12056 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12057 iRegIorL2I src1, iRegIorL2I src2,
12058 immI src3) %{
12059 match(Set dst (OrI src1 (RShiftI src2 src3)));
12060
12061 ins_cost(1.9 * INSN_COST);
12062 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12063
12064 ins_encode %{
12065 __ orrw(as_Register($dst$$reg),
12066 as_Register($src1$$reg),
12067 as_Register($src2$$reg),
12068 Assembler::ASR,
12069 $src3$$constant & 0x1f);
12070 %}
12071
12072 ins_pipe(ialu_reg_reg_shift);
12073 %}
12074
12075 // This pattern is automatically generated from aarch64_ad.m4
12076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12077 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12078 iRegL src1, iRegL src2,
12079 immI src3) %{
12080 match(Set dst (OrL src1 (RShiftL src2 src3)));
12081
12082 ins_cost(1.9 * INSN_COST);
12083 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12084
12085 ins_encode %{
12086 __ orr(as_Register($dst$$reg),
12087 as_Register($src1$$reg),
12088 as_Register($src2$$reg),
12089 Assembler::ASR,
12090 $src3$$constant & 0x3f);
12091 %}
12092
12093 ins_pipe(ialu_reg_reg_shift);
12094 %}
12095
12096 // This pattern is automatically generated from aarch64_ad.m4
12097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12098 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12099 iRegIorL2I src1, iRegIorL2I src2,
12100 immI src3) %{
12101 match(Set dst (OrI src1 (LShiftI src2 src3)));
12102
12103 ins_cost(1.9 * INSN_COST);
12104 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12105
12106 ins_encode %{
12107 __ orrw(as_Register($dst$$reg),
12108 as_Register($src1$$reg),
12109 as_Register($src2$$reg),
12110 Assembler::LSL,
12111 $src3$$constant & 0x1f);
12112 %}
12113
12114 ins_pipe(ialu_reg_reg_shift);
12115 %}
12116
12117 // This pattern is automatically generated from aarch64_ad.m4
12118 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12119 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12120 iRegL src1, iRegL src2,
12121 immI src3) %{
12122 match(Set dst (OrL src1 (LShiftL src2 src3)));
12123
12124 ins_cost(1.9 * INSN_COST);
12125 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12126
12127 ins_encode %{
12128 __ orr(as_Register($dst$$reg),
12129 as_Register($src1$$reg),
12130 as_Register($src2$$reg),
12131 Assembler::LSL,
12132 $src3$$constant & 0x3f);
12133 %}
12134
12135 ins_pipe(ialu_reg_reg_shift);
12136 %}
12137
12138 // This pattern is automatically generated from aarch64_ad.m4
12139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12140 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12141 iRegIorL2I src1, iRegIorL2I src2,
12142 immI src3) %{
12143 match(Set dst (OrI src1 (RotateRight src2 src3)));
12144
12145 ins_cost(1.9 * INSN_COST);
12146 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12147
12148 ins_encode %{
12149 __ orrw(as_Register($dst$$reg),
12150 as_Register($src1$$reg),
12151 as_Register($src2$$reg),
12152 Assembler::ROR,
12153 $src3$$constant & 0x1f);
12154 %}
12155
12156 ins_pipe(ialu_reg_reg_shift);
12157 %}
12158
12159 // This pattern is automatically generated from aarch64_ad.m4
12160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12161 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12162 iRegL src1, iRegL src2,
12163 immI src3) %{
12164 match(Set dst (OrL src1 (RotateRight src2 src3)));
12165
12166 ins_cost(1.9 * INSN_COST);
12167 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12168
12169 ins_encode %{
12170 __ orr(as_Register($dst$$reg),
12171 as_Register($src1$$reg),
12172 as_Register($src2$$reg),
12173 Assembler::ROR,
12174 $src3$$constant & 0x3f);
12175 %}
12176
12177 ins_pipe(ialu_reg_reg_shift);
12178 %}
12179
12180 // This pattern is automatically generated from aarch64_ad.m4
12181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12182 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12183 iRegIorL2I src1, iRegIorL2I src2,
12184 immI src3) %{
12185 match(Set dst (AddI src1 (URShiftI src2 src3)));
12186
12187 ins_cost(1.9 * INSN_COST);
12188 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12189
12190 ins_encode %{
12191 __ addw(as_Register($dst$$reg),
12192 as_Register($src1$$reg),
12193 as_Register($src2$$reg),
12194 Assembler::LSR,
12195 $src3$$constant & 0x1f);
12196 %}
12197
12198 ins_pipe(ialu_reg_reg_shift);
12199 %}
12200
12201 // This pattern is automatically generated from aarch64_ad.m4
12202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12203 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12204 iRegL src1, iRegL src2,
12205 immI src3) %{
12206 match(Set dst (AddL src1 (URShiftL src2 src3)));
12207
12208 ins_cost(1.9 * INSN_COST);
12209 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12210
12211 ins_encode %{
12212 __ add(as_Register($dst$$reg),
12213 as_Register($src1$$reg),
12214 as_Register($src2$$reg),
12215 Assembler::LSR,
12216 $src3$$constant & 0x3f);
12217 %}
12218
12219 ins_pipe(ialu_reg_reg_shift);
12220 %}
12221
12222 // This pattern is automatically generated from aarch64_ad.m4
12223 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12224 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12225 iRegIorL2I src1, iRegIorL2I src2,
12226 immI src3) %{
12227 match(Set dst (AddI src1 (RShiftI src2 src3)));
12228
12229 ins_cost(1.9 * INSN_COST);
12230 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12231
12232 ins_encode %{
12233 __ addw(as_Register($dst$$reg),
12234 as_Register($src1$$reg),
12235 as_Register($src2$$reg),
12236 Assembler::ASR,
12237 $src3$$constant & 0x1f);
12238 %}
12239
12240 ins_pipe(ialu_reg_reg_shift);
12241 %}
12242
12243 // This pattern is automatically generated from aarch64_ad.m4
12244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12245 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12246 iRegL src1, iRegL src2,
12247 immI src3) %{
12248 match(Set dst (AddL src1 (RShiftL src2 src3)));
12249
12250 ins_cost(1.9 * INSN_COST);
12251 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12252
12253 ins_encode %{
12254 __ add(as_Register($dst$$reg),
12255 as_Register($src1$$reg),
12256 as_Register($src2$$reg),
12257 Assembler::ASR,
12258 $src3$$constant & 0x3f);
12259 %}
12260
12261 ins_pipe(ialu_reg_reg_shift);
12262 %}
12263
12264 // This pattern is automatically generated from aarch64_ad.m4
12265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12266 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12267 iRegIorL2I src1, iRegIorL2I src2,
12268 immI src3) %{
12269 match(Set dst (AddI src1 (LShiftI src2 src3)));
12270
12271 ins_cost(1.9 * INSN_COST);
12272 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12273
12274 ins_encode %{
12275 __ addw(as_Register($dst$$reg),
12276 as_Register($src1$$reg),
12277 as_Register($src2$$reg),
12278 Assembler::LSL,
12279 $src3$$constant & 0x1f);
12280 %}
12281
12282 ins_pipe(ialu_reg_reg_shift);
12283 %}
12284
12285 // This pattern is automatically generated from aarch64_ad.m4
12286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12287 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12288 iRegL src1, iRegL src2,
12289 immI src3) %{
12290 match(Set dst (AddL src1 (LShiftL src2 src3)));
12291
12292 ins_cost(1.9 * INSN_COST);
12293 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12294
12295 ins_encode %{
12296 __ add(as_Register($dst$$reg),
12297 as_Register($src1$$reg),
12298 as_Register($src2$$reg),
12299 Assembler::LSL,
12300 $src3$$constant & 0x3f);
12301 %}
12302
12303 ins_pipe(ialu_reg_reg_shift);
12304 %}
12305
12306 // This pattern is automatically generated from aarch64_ad.m4
12307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12308 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12309 iRegIorL2I src1, iRegIorL2I src2,
12310 immI src3) %{
12311 match(Set dst (SubI src1 (URShiftI src2 src3)));
12312
12313 ins_cost(1.9 * INSN_COST);
12314 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12315
12316 ins_encode %{
12317 __ subw(as_Register($dst$$reg),
12318 as_Register($src1$$reg),
12319 as_Register($src2$$reg),
12320 Assembler::LSR,
12321 $src3$$constant & 0x1f);
12322 %}
12323
12324 ins_pipe(ialu_reg_reg_shift);
12325 %}
12326
12327 // This pattern is automatically generated from aarch64_ad.m4
12328 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12329 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12330 iRegL src1, iRegL src2,
12331 immI src3) %{
12332 match(Set dst (SubL src1 (URShiftL src2 src3)));
12333
12334 ins_cost(1.9 * INSN_COST);
12335 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12336
12337 ins_encode %{
12338 __ sub(as_Register($dst$$reg),
12339 as_Register($src1$$reg),
12340 as_Register($src2$$reg),
12341 Assembler::LSR,
12342 $src3$$constant & 0x3f);
12343 %}
12344
12345 ins_pipe(ialu_reg_reg_shift);
12346 %}
12347
12348 // This pattern is automatically generated from aarch64_ad.m4
12349 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12350 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12351 iRegIorL2I src1, iRegIorL2I src2,
12352 immI src3) %{
12353 match(Set dst (SubI src1 (RShiftI src2 src3)));
12354
12355 ins_cost(1.9 * INSN_COST);
12356 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12357
12358 ins_encode %{
12359 __ subw(as_Register($dst$$reg),
12360 as_Register($src1$$reg),
12361 as_Register($src2$$reg),
12362 Assembler::ASR,
12363 $src3$$constant & 0x1f);
12364 %}
12365
12366 ins_pipe(ialu_reg_reg_shift);
12367 %}
12368
12369 // This pattern is automatically generated from aarch64_ad.m4
12370 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12371 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12372 iRegL src1, iRegL src2,
12373 immI src3) %{
12374 match(Set dst (SubL src1 (RShiftL src2 src3)));
12375
12376 ins_cost(1.9 * INSN_COST);
12377 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12378
12379 ins_encode %{
12380 __ sub(as_Register($dst$$reg),
12381 as_Register($src1$$reg),
12382 as_Register($src2$$reg),
12383 Assembler::ASR,
12384 $src3$$constant & 0x3f);
12385 %}
12386
12387 ins_pipe(ialu_reg_reg_shift);
12388 %}
12389
12390 // This pattern is automatically generated from aarch64_ad.m4
12391 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12392 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12393 iRegIorL2I src1, iRegIorL2I src2,
12394 immI src3) %{
12395 match(Set dst (SubI src1 (LShiftI src2 src3)));
12396
12397 ins_cost(1.9 * INSN_COST);
12398 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12399
12400 ins_encode %{
12401 __ subw(as_Register($dst$$reg),
12402 as_Register($src1$$reg),
12403 as_Register($src2$$reg),
12404 Assembler::LSL,
12405 $src3$$constant & 0x1f);
12406 %}
12407
12408 ins_pipe(ialu_reg_reg_shift);
12409 %}
12410
12411 // This pattern is automatically generated from aarch64_ad.m4
12412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12413 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12414 iRegL src1, iRegL src2,
12415 immI src3) %{
12416 match(Set dst (SubL src1 (LShiftL src2 src3)));
12417
12418 ins_cost(1.9 * INSN_COST);
12419 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12420
12421 ins_encode %{
12422 __ sub(as_Register($dst$$reg),
12423 as_Register($src1$$reg),
12424 as_Register($src2$$reg),
12425 Assembler::LSL,
12426 $src3$$constant & 0x3f);
12427 %}
12428
12429 ins_pipe(ialu_reg_reg_shift);
12430 %}
12431
12432 // This pattern is automatically generated from aarch64_ad.m4
12433 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12434
12435 // Shift Left followed by Shift Right.
12436 // This idiom is used by the compiler for the i2b bytecode etc.
12437 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12438 %{
12439 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12440 ins_cost(INSN_COST * 2);
12441 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12442 ins_encode %{
12443 int lshift = $lshift_count$$constant & 63;
12444 int rshift = $rshift_count$$constant & 63;
12445 int s = 63 - lshift;
12446 int r = (rshift - lshift) & 63;
12447 __ sbfm(as_Register($dst$$reg),
12448 as_Register($src$$reg),
12449 r, s);
12450 %}
12451
12452 ins_pipe(ialu_reg_shift);
12453 %}
12454
12455 // This pattern is automatically generated from aarch64_ad.m4
12456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12457
12458 // Shift Left followed by Shift Right.
12459 // This idiom is used by the compiler for the i2b bytecode etc.
12460 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12461 %{
12462 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12463 ins_cost(INSN_COST * 2);
12464 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12465 ins_encode %{
12466 int lshift = $lshift_count$$constant & 31;
12467 int rshift = $rshift_count$$constant & 31;
12468 int s = 31 - lshift;
12469 int r = (rshift - lshift) & 31;
12470 __ sbfmw(as_Register($dst$$reg),
12471 as_Register($src$$reg),
12472 r, s);
12473 %}
12474
12475 ins_pipe(ialu_reg_shift);
12476 %}
12477
12478 // This pattern is automatically generated from aarch64_ad.m4
12479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12480
12481 // Shift Left followed by Shift Right.
12482 // This idiom is used by the compiler for the i2b bytecode etc.
12483 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12484 %{
12485 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12486 ins_cost(INSN_COST * 2);
12487 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12488 ins_encode %{
12489 int lshift = $lshift_count$$constant & 63;
12490 int rshift = $rshift_count$$constant & 63;
12491 int s = 63 - lshift;
12492 int r = (rshift - lshift) & 63;
12493 __ ubfm(as_Register($dst$$reg),
12494 as_Register($src$$reg),
12495 r, s);
12496 %}
12497
12498 ins_pipe(ialu_reg_shift);
12499 %}
12500
12501 // This pattern is automatically generated from aarch64_ad.m4
12502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12503
12504 // Shift Left followed by Shift Right.
12505 // This idiom is used by the compiler for the i2b bytecode etc.
12506 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12507 %{
12508 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12509 ins_cost(INSN_COST * 2);
12510 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12511 ins_encode %{
12512 int lshift = $lshift_count$$constant & 31;
12513 int rshift = $rshift_count$$constant & 31;
12514 int s = 31 - lshift;
12515 int r = (rshift - lshift) & 31;
12516 __ ubfmw(as_Register($dst$$reg),
12517 as_Register($src$$reg),
12518 r, s);
12519 %}
12520
12521 ins_pipe(ialu_reg_shift);
12522 %}
12523
12524 // Bitfield extract with shift & mask
12525
12526 // This pattern is automatically generated from aarch64_ad.m4
12527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12528 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12529 %{
12530 match(Set dst (AndI (URShiftI src rshift) mask));
12531 // Make sure we are not going to exceed what ubfxw can do.
12532 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12533
12534 ins_cost(INSN_COST);
12535 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12536 ins_encode %{
12537 int rshift = $rshift$$constant & 31;
12538 intptr_t mask = $mask$$constant;
12539 int width = exact_log2(mask+1);
12540 __ ubfxw(as_Register($dst$$reg),
12541 as_Register($src$$reg), rshift, width);
12542 %}
12543 ins_pipe(ialu_reg_shift);
12544 %}
12545
12546 // This pattern is automatically generated from aarch64_ad.m4
12547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12548 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12549 %{
12550 match(Set dst (AndL (URShiftL src rshift) mask));
12551 // Make sure we are not going to exceed what ubfx can do.
12552 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12553
12554 ins_cost(INSN_COST);
12555 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12556 ins_encode %{
12557 int rshift = $rshift$$constant & 63;
12558 intptr_t mask = $mask$$constant;
12559 int width = exact_log2_long(mask+1);
12560 __ ubfx(as_Register($dst$$reg),
12561 as_Register($src$$reg), rshift, width);
12562 %}
12563 ins_pipe(ialu_reg_shift);
12564 %}
12565
12566
12567 // This pattern is automatically generated from aarch64_ad.m4
12568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12569
12570 // We can use ubfx when extending an And with a mask when we know mask
12571 // is positive. We know that because immI_bitmask guarantees it.
12572 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12573 %{
12574 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12575 // Make sure we are not going to exceed what ubfxw can do.
12576 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12577
12578 ins_cost(INSN_COST * 2);
12579 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12580 ins_encode %{
12581 int rshift = $rshift$$constant & 31;
12582 intptr_t mask = $mask$$constant;
12583 int width = exact_log2(mask+1);
12584 __ ubfx(as_Register($dst$$reg),
12585 as_Register($src$$reg), rshift, width);
12586 %}
12587 ins_pipe(ialu_reg_shift);
12588 %}
12589
12590
12591 // This pattern is automatically generated from aarch64_ad.m4
12592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12593
12594 // We can use ubfiz when masking by a positive number and then left shifting the result.
12595 // We know that the mask is positive because immI_bitmask guarantees it.
12596 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12597 %{
12598 match(Set dst (LShiftI (AndI src mask) lshift));
12599 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12600
12601 ins_cost(INSN_COST);
12602 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12603 ins_encode %{
12604 int lshift = $lshift$$constant & 31;
12605 intptr_t mask = $mask$$constant;
12606 int width = exact_log2(mask+1);
12607 __ ubfizw(as_Register($dst$$reg),
12608 as_Register($src$$reg), lshift, width);
12609 %}
12610 ins_pipe(ialu_reg_shift);
12611 %}
12612
12613 // This pattern is automatically generated from aarch64_ad.m4
12614 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12615
12616 // We can use ubfiz when masking by a positive number and then left shifting the result.
12617 // We know that the mask is positive because immL_bitmask guarantees it.
12618 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12619 %{
12620 match(Set dst (LShiftL (AndL src mask) lshift));
12621 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12622
12623 ins_cost(INSN_COST);
12624 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12625 ins_encode %{
12626 int lshift = $lshift$$constant & 63;
12627 intptr_t mask = $mask$$constant;
12628 int width = exact_log2_long(mask+1);
12629 __ ubfiz(as_Register($dst$$reg),
12630 as_Register($src$$reg), lshift, width);
12631 %}
12632 ins_pipe(ialu_reg_shift);
12633 %}
12634
12635 // This pattern is automatically generated from aarch64_ad.m4
12636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12637
12638 // We can use ubfiz when masking by a positive number and then left shifting the result.
12639 // We know that the mask is positive because immI_bitmask guarantees it.
12640 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12641 %{
12642 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12643 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12644
12645 ins_cost(INSN_COST);
12646 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12647 ins_encode %{
12648 int lshift = $lshift$$constant & 31;
12649 intptr_t mask = $mask$$constant;
12650 int width = exact_log2(mask+1);
12651 __ ubfizw(as_Register($dst$$reg),
12652 as_Register($src$$reg), lshift, width);
12653 %}
12654 ins_pipe(ialu_reg_shift);
12655 %}
12656
12657 // This pattern is automatically generated from aarch64_ad.m4
12658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12659
12660 // We can use ubfiz when masking by a positive number and then left shifting the result.
12661 // We know that the mask is positive because immL_bitmask guarantees it.
12662 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12663 %{
12664 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12665 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12666
12667 ins_cost(INSN_COST);
12668 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12669 ins_encode %{
12670 int lshift = $lshift$$constant & 63;
12671 intptr_t mask = $mask$$constant;
12672 int width = exact_log2_long(mask+1);
12673 __ ubfiz(as_Register($dst$$reg),
12674 as_Register($src$$reg), lshift, width);
12675 %}
12676 ins_pipe(ialu_reg_shift);
12677 %}
12678
12679
12680 // This pattern is automatically generated from aarch64_ad.m4
12681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12682
12683 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12684 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12685 %{
12686 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12687 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12688
12689 ins_cost(INSN_COST);
12690 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12691 ins_encode %{
12692 int lshift = $lshift$$constant & 63;
12693 intptr_t mask = $mask$$constant;
12694 int width = exact_log2(mask+1);
12695 __ ubfiz(as_Register($dst$$reg),
12696 as_Register($src$$reg), lshift, width);
12697 %}
12698 ins_pipe(ialu_reg_shift);
12699 %}
12700
12701 // This pattern is automatically generated from aarch64_ad.m4
12702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12703
12704 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12705 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12706 %{
12707 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12708 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12709
12710 ins_cost(INSN_COST);
12711 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12712 ins_encode %{
12713 int lshift = $lshift$$constant & 31;
12714 intptr_t mask = $mask$$constant;
12715 int width = exact_log2(mask+1);
12716 __ ubfiz(as_Register($dst$$reg),
12717 as_Register($src$$reg), lshift, width);
12718 %}
12719 ins_pipe(ialu_reg_shift);
12720 %}
12721
12722 // This pattern is automatically generated from aarch64_ad.m4
12723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12724
12725 // Can skip int2long conversions after AND with small bitmask
12726 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12727 %{
12728 match(Set dst (ConvI2L (AndI src msk)));
12729 ins_cost(INSN_COST);
12730 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12731 ins_encode %{
12732 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12733 %}
12734 ins_pipe(ialu_reg_shift);
12735 %}
12736
12737
12738 // Rotations
12739
12740 // This pattern is automatically generated from aarch64_ad.m4
12741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12742 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12743 %{
12744 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12745 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12746
12747 ins_cost(INSN_COST);
12748 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12749
12750 ins_encode %{
12751 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12752 $rshift$$constant & 63);
12753 %}
12754 ins_pipe(ialu_reg_reg_extr);
12755 %}
12756
12757
12758 // This pattern is automatically generated from aarch64_ad.m4
12759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12760 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12761 %{
12762 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12763 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12764
12765 ins_cost(INSN_COST);
12766 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12767
12768 ins_encode %{
12769 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12770 $rshift$$constant & 31);
12771 %}
12772 ins_pipe(ialu_reg_reg_extr);
12773 %}
12774
12775
12776 // This pattern is automatically generated from aarch64_ad.m4
12777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12778 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12779 %{
12780 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12781 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12782
12783 ins_cost(INSN_COST);
12784 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12785
12786 ins_encode %{
12787 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12788 $rshift$$constant & 63);
12789 %}
12790 ins_pipe(ialu_reg_reg_extr);
12791 %}
12792
12793
12794 // This pattern is automatically generated from aarch64_ad.m4
12795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12796 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12797 %{
12798 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12799 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12800
12801 ins_cost(INSN_COST);
12802 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12803
12804 ins_encode %{
12805 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12806 $rshift$$constant & 31);
12807 %}
12808 ins_pipe(ialu_reg_reg_extr);
12809 %}
12810
12811 // This pattern is automatically generated from aarch64_ad.m4
12812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12813 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12814 %{
12815 match(Set dst (RotateRight src shift));
12816
12817 ins_cost(INSN_COST);
12818 format %{ "ror $dst, $src, $shift" %}
12819
12820 ins_encode %{
12821 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12822 $shift$$constant & 0x1f);
12823 %}
12824 ins_pipe(ialu_reg_reg_vshift);
12825 %}
12826
12827 // This pattern is automatically generated from aarch64_ad.m4
12828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12829 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12830 %{
12831 match(Set dst (RotateRight src shift));
12832
12833 ins_cost(INSN_COST);
12834 format %{ "ror $dst, $src, $shift" %}
12835
12836 ins_encode %{
12837 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12838 $shift$$constant & 0x3f);
12839 %}
12840 ins_pipe(ialu_reg_reg_vshift);
12841 %}
12842
12843 // This pattern is automatically generated from aarch64_ad.m4
12844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12845 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12846 %{
12847 match(Set dst (RotateRight src shift));
12848
12849 ins_cost(INSN_COST);
12850 format %{ "ror $dst, $src, $shift" %}
12851
12852 ins_encode %{
12853 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12854 %}
12855 ins_pipe(ialu_reg_reg_vshift);
12856 %}
12857
12858 // This pattern is automatically generated from aarch64_ad.m4
12859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12860 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12861 %{
12862 match(Set dst (RotateRight src shift));
12863
12864 ins_cost(INSN_COST);
12865 format %{ "ror $dst, $src, $shift" %}
12866
12867 ins_encode %{
12868 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12869 %}
12870 ins_pipe(ialu_reg_reg_vshift);
12871 %}
12872
12873 // This pattern is automatically generated from aarch64_ad.m4
12874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12875 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12876 %{
12877 match(Set dst (RotateLeft src shift));
12878
12879 ins_cost(INSN_COST);
12880 format %{ "rol $dst, $src, $shift" %}
12881
12882 ins_encode %{
12883 __ subw(rscratch1, zr, as_Register($shift$$reg));
12884 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12885 %}
12886 ins_pipe(ialu_reg_reg_vshift);
12887 %}
12888
12889 // This pattern is automatically generated from aarch64_ad.m4
12890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12891 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12892 %{
12893 match(Set dst (RotateLeft src shift));
12894
12895 ins_cost(INSN_COST);
12896 format %{ "rol $dst, $src, $shift" %}
12897
12898 ins_encode %{
12899 __ subw(rscratch1, zr, as_Register($shift$$reg));
12900 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12901 %}
12902 ins_pipe(ialu_reg_reg_vshift);
12903 %}
12904
12905
12906 // Add/subtract (extended)
12907
12908 // This pattern is automatically generated from aarch64_ad.m4
12909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12910 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12911 %{
12912 match(Set dst (AddL src1 (ConvI2L src2)));
12913 ins_cost(INSN_COST);
12914 format %{ "add $dst, $src1, $src2, sxtw" %}
12915
12916 ins_encode %{
12917 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12918 as_Register($src2$$reg), ext::sxtw);
12919 %}
12920 ins_pipe(ialu_reg_reg);
12921 %}
12922
12923 // This pattern is automatically generated from aarch64_ad.m4
12924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12925 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12926 %{
12927 match(Set dst (SubL src1 (ConvI2L src2)));
12928 ins_cost(INSN_COST);
12929 format %{ "sub $dst, $src1, $src2, sxtw" %}
12930
12931 ins_encode %{
12932 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12933 as_Register($src2$$reg), ext::sxtw);
12934 %}
12935 ins_pipe(ialu_reg_reg);
12936 %}
12937
12938 // This pattern is automatically generated from aarch64_ad.m4
12939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12940 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12941 %{
12942 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12943 ins_cost(INSN_COST);
12944 format %{ "add $dst, $src1, $src2, sxth" %}
12945
12946 ins_encode %{
12947 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12948 as_Register($src2$$reg), ext::sxth);
12949 %}
12950 ins_pipe(ialu_reg_reg);
12951 %}
12952
12953 // This pattern is automatically generated from aarch64_ad.m4
12954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12955 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12956 %{
12957 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12958 ins_cost(INSN_COST);
12959 format %{ "add $dst, $src1, $src2, sxtb" %}
12960
12961 ins_encode %{
12962 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12963 as_Register($src2$$reg), ext::sxtb);
12964 %}
12965 ins_pipe(ialu_reg_reg);
12966 %}
12967
12968 // This pattern is automatically generated from aarch64_ad.m4
12969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12970 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12971 %{
12972 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12973 ins_cost(INSN_COST);
12974 format %{ "add $dst, $src1, $src2, uxtb" %}
12975
12976 ins_encode %{
12977 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12978 as_Register($src2$$reg), ext::uxtb);
12979 %}
12980 ins_pipe(ialu_reg_reg);
12981 %}
12982
12983 // This pattern is automatically generated from aarch64_ad.m4
12984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12985 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12986 %{
12987 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12988 ins_cost(INSN_COST);
12989 format %{ "add $dst, $src1, $src2, sxth" %}
12990
12991 ins_encode %{
12992 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12993 as_Register($src2$$reg), ext::sxth);
12994 %}
12995 ins_pipe(ialu_reg_reg);
12996 %}
12997
12998 // This pattern is automatically generated from aarch64_ad.m4
12999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13000 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13001 %{
13002 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13003 ins_cost(INSN_COST);
13004 format %{ "add $dst, $src1, $src2, sxtw" %}
13005
13006 ins_encode %{
13007 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13008 as_Register($src2$$reg), ext::sxtw);
13009 %}
13010 ins_pipe(ialu_reg_reg);
13011 %}
13012
13013 // This pattern is automatically generated from aarch64_ad.m4
13014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13015 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13016 %{
13017 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13018 ins_cost(INSN_COST);
13019 format %{ "add $dst, $src1, $src2, sxtb" %}
13020
13021 ins_encode %{
13022 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13023 as_Register($src2$$reg), ext::sxtb);
13024 %}
13025 ins_pipe(ialu_reg_reg);
13026 %}
13027
13028 // This pattern is automatically generated from aarch64_ad.m4
13029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13030 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13031 %{
13032 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13033 ins_cost(INSN_COST);
13034 format %{ "add $dst, $src1, $src2, uxtb" %}
13035
13036 ins_encode %{
13037 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13038 as_Register($src2$$reg), ext::uxtb);
13039 %}
13040 ins_pipe(ialu_reg_reg);
13041 %}
13042
13043 // This pattern is automatically generated from aarch64_ad.m4
13044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13045 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13046 %{
13047 match(Set dst (AddI src1 (AndI src2 mask)));
13048 ins_cost(INSN_COST);
13049 format %{ "addw $dst, $src1, $src2, uxtb" %}
13050
13051 ins_encode %{
13052 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13053 as_Register($src2$$reg), ext::uxtb);
13054 %}
13055 ins_pipe(ialu_reg_reg);
13056 %}
13057
13058 // This pattern is automatically generated from aarch64_ad.m4
13059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13060 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13061 %{
13062 match(Set dst (AddI src1 (AndI src2 mask)));
13063 ins_cost(INSN_COST);
13064 format %{ "addw $dst, $src1, $src2, uxth" %}
13065
13066 ins_encode %{
13067 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13068 as_Register($src2$$reg), ext::uxth);
13069 %}
13070 ins_pipe(ialu_reg_reg);
13071 %}
13072
13073 // This pattern is automatically generated from aarch64_ad.m4
13074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13075 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13076 %{
13077 match(Set dst (AddL src1 (AndL src2 mask)));
13078 ins_cost(INSN_COST);
13079 format %{ "add $dst, $src1, $src2, uxtb" %}
13080
13081 ins_encode %{
13082 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13083 as_Register($src2$$reg), ext::uxtb);
13084 %}
13085 ins_pipe(ialu_reg_reg);
13086 %}
13087
13088 // This pattern is automatically generated from aarch64_ad.m4
13089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13090 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13091 %{
13092 match(Set dst (AddL src1 (AndL src2 mask)));
13093 ins_cost(INSN_COST);
13094 format %{ "add $dst, $src1, $src2, uxth" %}
13095
13096 ins_encode %{
13097 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13098 as_Register($src2$$reg), ext::uxth);
13099 %}
13100 ins_pipe(ialu_reg_reg);
13101 %}
13102
13103 // This pattern is automatically generated from aarch64_ad.m4
13104 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13105 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13106 %{
13107 match(Set dst (AddL src1 (AndL src2 mask)));
13108 ins_cost(INSN_COST);
13109 format %{ "add $dst, $src1, $src2, uxtw" %}
13110
13111 ins_encode %{
13112 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13113 as_Register($src2$$reg), ext::uxtw);
13114 %}
13115 ins_pipe(ialu_reg_reg);
13116 %}
13117
13118 // This pattern is automatically generated from aarch64_ad.m4
13119 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13120 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13121 %{
13122 match(Set dst (SubI src1 (AndI src2 mask)));
13123 ins_cost(INSN_COST);
13124 format %{ "subw $dst, $src1, $src2, uxtb" %}
13125
13126 ins_encode %{
13127 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13128 as_Register($src2$$reg), ext::uxtb);
13129 %}
13130 ins_pipe(ialu_reg_reg);
13131 %}
13132
13133 // This pattern is automatically generated from aarch64_ad.m4
13134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13135 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13136 %{
13137 match(Set dst (SubI src1 (AndI src2 mask)));
13138 ins_cost(INSN_COST);
13139 format %{ "subw $dst, $src1, $src2, uxth" %}
13140
13141 ins_encode %{
13142 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13143 as_Register($src2$$reg), ext::uxth);
13144 %}
13145 ins_pipe(ialu_reg_reg);
13146 %}
13147
13148 // This pattern is automatically generated from aarch64_ad.m4
13149 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13150 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13151 %{
13152 match(Set dst (SubL src1 (AndL src2 mask)));
13153 ins_cost(INSN_COST);
13154 format %{ "sub $dst, $src1, $src2, uxtb" %}
13155
13156 ins_encode %{
13157 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13158 as_Register($src2$$reg), ext::uxtb);
13159 %}
13160 ins_pipe(ialu_reg_reg);
13161 %}
13162
13163 // This pattern is automatically generated from aarch64_ad.m4
13164 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13165 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13166 %{
13167 match(Set dst (SubL src1 (AndL src2 mask)));
13168 ins_cost(INSN_COST);
13169 format %{ "sub $dst, $src1, $src2, uxth" %}
13170
13171 ins_encode %{
13172 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13173 as_Register($src2$$reg), ext::uxth);
13174 %}
13175 ins_pipe(ialu_reg_reg);
13176 %}
13177
13178 // This pattern is automatically generated from aarch64_ad.m4
13179 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13180 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13181 %{
13182 match(Set dst (SubL src1 (AndL src2 mask)));
13183 ins_cost(INSN_COST);
13184 format %{ "sub $dst, $src1, $src2, uxtw" %}
13185
13186 ins_encode %{
13187 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13188 as_Register($src2$$reg), ext::uxtw);
13189 %}
13190 ins_pipe(ialu_reg_reg);
13191 %}
13192
13193
13194 // This pattern is automatically generated from aarch64_ad.m4
13195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13196 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13197 %{
13198 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13199 ins_cost(1.9 * INSN_COST);
13200 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13201
13202 ins_encode %{
13203 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13204 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13205 %}
13206 ins_pipe(ialu_reg_reg_shift);
13207 %}
13208
13209 // This pattern is automatically generated from aarch64_ad.m4
13210 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13211 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13212 %{
13213 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13214 ins_cost(1.9 * INSN_COST);
13215 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13216
13217 ins_encode %{
13218 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13219 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13220 %}
13221 ins_pipe(ialu_reg_reg_shift);
13222 %}
13223
13224 // This pattern is automatically generated from aarch64_ad.m4
13225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13226 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13227 %{
13228 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13229 ins_cost(1.9 * INSN_COST);
13230 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13231
13232 ins_encode %{
13233 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13234 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13235 %}
13236 ins_pipe(ialu_reg_reg_shift);
13237 %}
13238
13239 // This pattern is automatically generated from aarch64_ad.m4
13240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13241 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13242 %{
13243 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13244 ins_cost(1.9 * INSN_COST);
13245 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13246
13247 ins_encode %{
13248 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13249 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13250 %}
13251 ins_pipe(ialu_reg_reg_shift);
13252 %}
13253
13254 // This pattern is automatically generated from aarch64_ad.m4
13255 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13256 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13257 %{
13258 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13259 ins_cost(1.9 * INSN_COST);
13260 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13261
13262 ins_encode %{
13263 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13264 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13265 %}
13266 ins_pipe(ialu_reg_reg_shift);
13267 %}
13268
13269 // This pattern is automatically generated from aarch64_ad.m4
13270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13271 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13272 %{
13273 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13274 ins_cost(1.9 * INSN_COST);
13275 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13276
13277 ins_encode %{
13278 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13279 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13280 %}
13281 ins_pipe(ialu_reg_reg_shift);
13282 %}
13283
13284 // This pattern is automatically generated from aarch64_ad.m4
13285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13286 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13287 %{
13288 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13289 ins_cost(1.9 * INSN_COST);
13290 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13291
13292 ins_encode %{
13293 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13294 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13295 %}
13296 ins_pipe(ialu_reg_reg_shift);
13297 %}
13298
13299 // This pattern is automatically generated from aarch64_ad.m4
13300 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13301 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13302 %{
13303 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13304 ins_cost(1.9 * INSN_COST);
13305 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13306
13307 ins_encode %{
13308 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13309 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13310 %}
13311 ins_pipe(ialu_reg_reg_shift);
13312 %}
13313
13314 // This pattern is automatically generated from aarch64_ad.m4
13315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13316 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13317 %{
13318 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13319 ins_cost(1.9 * INSN_COST);
13320 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13321
13322 ins_encode %{
13323 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13324 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13325 %}
13326 ins_pipe(ialu_reg_reg_shift);
13327 %}
13328
13329 // This pattern is automatically generated from aarch64_ad.m4
13330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13331 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13332 %{
13333 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13334 ins_cost(1.9 * INSN_COST);
13335 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13336
13337 ins_encode %{
13338 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13339 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13340 %}
13341 ins_pipe(ialu_reg_reg_shift);
13342 %}
13343
13344 // This pattern is automatically generated from aarch64_ad.m4
13345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13346 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13347 %{
13348 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13349 ins_cost(1.9 * INSN_COST);
13350 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13351
13352 ins_encode %{
13353 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13354 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13355 %}
13356 ins_pipe(ialu_reg_reg_shift);
13357 %}
13358
13359 // This pattern is automatically generated from aarch64_ad.m4
13360 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13361 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13362 %{
13363 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13364 ins_cost(1.9 * INSN_COST);
13365 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13366
13367 ins_encode %{
13368 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13369 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13370 %}
13371 ins_pipe(ialu_reg_reg_shift);
13372 %}
13373
13374 // This pattern is automatically generated from aarch64_ad.m4
13375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13376 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13377 %{
13378 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13379 ins_cost(1.9 * INSN_COST);
13380 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13381
13382 ins_encode %{
13383 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13384 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13385 %}
13386 ins_pipe(ialu_reg_reg_shift);
13387 %}
13388
13389 // This pattern is automatically generated from aarch64_ad.m4
13390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13391 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13392 %{
13393 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13394 ins_cost(1.9 * INSN_COST);
13395 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13396
13397 ins_encode %{
13398 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13399 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13400 %}
13401 ins_pipe(ialu_reg_reg_shift);
13402 %}
13403
13404 // This pattern is automatically generated from aarch64_ad.m4
13405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13406 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13407 %{
13408 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13409 ins_cost(1.9 * INSN_COST);
13410 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13411
13412 ins_encode %{
13413 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13414 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13415 %}
13416 ins_pipe(ialu_reg_reg_shift);
13417 %}
13418
13419 // This pattern is automatically generated from aarch64_ad.m4
13420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13421 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13422 %{
13423 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13424 ins_cost(1.9 * INSN_COST);
13425 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13426
13427 ins_encode %{
13428 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13429 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13430 %}
13431 ins_pipe(ialu_reg_reg_shift);
13432 %}
13433
13434 // This pattern is automatically generated from aarch64_ad.m4
13435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13436 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13437 %{
13438 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13439 ins_cost(1.9 * INSN_COST);
13440 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13441
13442 ins_encode %{
13443 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13444 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13445 %}
13446 ins_pipe(ialu_reg_reg_shift);
13447 %}
13448
13449 // This pattern is automatically generated from aarch64_ad.m4
13450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13451 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13452 %{
13453 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13454 ins_cost(1.9 * INSN_COST);
13455 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13456
13457 ins_encode %{
13458 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13459 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13460 %}
13461 ins_pipe(ialu_reg_reg_shift);
13462 %}
13463
13464 // This pattern is automatically generated from aarch64_ad.m4
13465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13466 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13467 %{
13468 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13469 ins_cost(1.9 * INSN_COST);
13470 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13471
13472 ins_encode %{
13473 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13474 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13475 %}
13476 ins_pipe(ialu_reg_reg_shift);
13477 %}
13478
13479 // This pattern is automatically generated from aarch64_ad.m4
13480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13481 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13482 %{
13483 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13484 ins_cost(1.9 * INSN_COST);
13485 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13486
13487 ins_encode %{
13488 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13489 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13490 %}
13491 ins_pipe(ialu_reg_reg_shift);
13492 %}
13493
13494 // This pattern is automatically generated from aarch64_ad.m4
13495 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13496 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13497 %{
13498 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13499 ins_cost(1.9 * INSN_COST);
13500 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13501
13502 ins_encode %{
13503 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13504 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13505 %}
13506 ins_pipe(ialu_reg_reg_shift);
13507 %}
13508
13509 // This pattern is automatically generated from aarch64_ad.m4
13510 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13511 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13512 %{
13513 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13514 ins_cost(1.9 * INSN_COST);
13515 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13516
13517 ins_encode %{
13518 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13519 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13520 %}
13521 ins_pipe(ialu_reg_reg_shift);
13522 %}
13523
13524 // This pattern is automatically generated from aarch64_ad.m4
13525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13526 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13527 %{
13528 effect(DEF dst, USE src1, USE src2, USE cr);
13529 ins_cost(INSN_COST * 2);
13530 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13531
13532 ins_encode %{
13533 __ cselw($dst$$Register,
13534 $src1$$Register,
13535 $src2$$Register,
13536 Assembler::LT);
13537 %}
13538 ins_pipe(icond_reg_reg);
13539 %}
13540
13541 // This pattern is automatically generated from aarch64_ad.m4
13542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13543 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13544 %{
13545 effect(DEF dst, USE src1, USE src2, USE cr);
13546 ins_cost(INSN_COST * 2);
13547 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13548
13549 ins_encode %{
13550 __ cselw($dst$$Register,
13551 $src1$$Register,
13552 $src2$$Register,
13553 Assembler::GT);
13554 %}
13555 ins_pipe(icond_reg_reg);
13556 %}
13557
13558 // This pattern is automatically generated from aarch64_ad.m4
13559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13560 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13561 %{
13562 effect(DEF dst, USE src1, USE cr);
13563 ins_cost(INSN_COST * 2);
13564 format %{ "cselw $dst, $src1, zr lt\t" %}
13565
13566 ins_encode %{
13567 __ cselw($dst$$Register,
13568 $src1$$Register,
13569 zr,
13570 Assembler::LT);
13571 %}
13572 ins_pipe(icond_reg);
13573 %}
13574
13575 // This pattern is automatically generated from aarch64_ad.m4
13576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13577 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13578 %{
13579 effect(DEF dst, USE src1, USE cr);
13580 ins_cost(INSN_COST * 2);
13581 format %{ "cselw $dst, $src1, zr gt\t" %}
13582
13583 ins_encode %{
13584 __ cselw($dst$$Register,
13585 $src1$$Register,
13586 zr,
13587 Assembler::GT);
13588 %}
13589 ins_pipe(icond_reg);
13590 %}
13591
13592 // This pattern is automatically generated from aarch64_ad.m4
13593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13594 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13595 %{
13596 effect(DEF dst, USE src1, USE cr);
13597 ins_cost(INSN_COST * 2);
13598 format %{ "csincw $dst, $src1, zr le\t" %}
13599
13600 ins_encode %{
13601 __ csincw($dst$$Register,
13602 $src1$$Register,
13603 zr,
13604 Assembler::LE);
13605 %}
13606 ins_pipe(icond_reg);
13607 %}
13608
13609 // This pattern is automatically generated from aarch64_ad.m4
13610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13611 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13612 %{
13613 effect(DEF dst, USE src1, USE cr);
13614 ins_cost(INSN_COST * 2);
13615 format %{ "csincw $dst, $src1, zr gt\t" %}
13616
13617 ins_encode %{
13618 __ csincw($dst$$Register,
13619 $src1$$Register,
13620 zr,
13621 Assembler::GT);
13622 %}
13623 ins_pipe(icond_reg);
13624 %}
13625
13626 // This pattern is automatically generated from aarch64_ad.m4
13627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13628 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13629 %{
13630 effect(DEF dst, USE src1, USE cr);
13631 ins_cost(INSN_COST * 2);
13632 format %{ "csinvw $dst, $src1, zr lt\t" %}
13633
13634 ins_encode %{
13635 __ csinvw($dst$$Register,
13636 $src1$$Register,
13637 zr,
13638 Assembler::LT);
13639 %}
13640 ins_pipe(icond_reg);
13641 %}
13642
13643 // This pattern is automatically generated from aarch64_ad.m4
13644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13645 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13646 %{
13647 effect(DEF dst, USE src1, USE cr);
13648 ins_cost(INSN_COST * 2);
13649 format %{ "csinvw $dst, $src1, zr ge\t" %}
13650
13651 ins_encode %{
13652 __ csinvw($dst$$Register,
13653 $src1$$Register,
13654 zr,
13655 Assembler::GE);
13656 %}
13657 ins_pipe(icond_reg);
13658 %}
13659
13660 // This pattern is automatically generated from aarch64_ad.m4
13661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13662 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13663 %{
13664 match(Set dst (MinI src imm));
13665 ins_cost(INSN_COST * 3);
13666 expand %{
13667 rFlagsReg cr;
13668 compI_reg_imm0(cr, src);
13669 cmovI_reg_imm0_lt(dst, src, cr);
13670 %}
13671 %}
13672
13673 // This pattern is automatically generated from aarch64_ad.m4
13674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13675 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13676 %{
13677 match(Set dst (MinI imm src));
13678 ins_cost(INSN_COST * 3);
13679 expand %{
13680 rFlagsReg cr;
13681 compI_reg_imm0(cr, src);
13682 cmovI_reg_imm0_lt(dst, src, cr);
13683 %}
13684 %}
13685
13686 // This pattern is automatically generated from aarch64_ad.m4
13687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13688 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13689 %{
13690 match(Set dst (MinI src imm));
13691 ins_cost(INSN_COST * 3);
13692 expand %{
13693 rFlagsReg cr;
13694 compI_reg_imm0(cr, src);
13695 cmovI_reg_imm1_le(dst, src, cr);
13696 %}
13697 %}
13698
13699 // This pattern is automatically generated from aarch64_ad.m4
13700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13701 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13702 %{
13703 match(Set dst (MinI imm src));
13704 ins_cost(INSN_COST * 3);
13705 expand %{
13706 rFlagsReg cr;
13707 compI_reg_imm0(cr, src);
13708 cmovI_reg_imm1_le(dst, src, cr);
13709 %}
13710 %}
13711
13712 // This pattern is automatically generated from aarch64_ad.m4
13713 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13714 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13715 %{
13716 match(Set dst (MinI src imm));
13717 ins_cost(INSN_COST * 3);
13718 expand %{
13719 rFlagsReg cr;
13720 compI_reg_imm0(cr, src);
13721 cmovI_reg_immM1_lt(dst, src, cr);
13722 %}
13723 %}
13724
13725 // This pattern is automatically generated from aarch64_ad.m4
13726 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13727 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13728 %{
13729 match(Set dst (MinI imm src));
13730 ins_cost(INSN_COST * 3);
13731 expand %{
13732 rFlagsReg cr;
13733 compI_reg_imm0(cr, src);
13734 cmovI_reg_immM1_lt(dst, src, cr);
13735 %}
13736 %}
13737
13738 // This pattern is automatically generated from aarch64_ad.m4
13739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13740 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13741 %{
13742 match(Set dst (MaxI src imm));
13743 ins_cost(INSN_COST * 3);
13744 expand %{
13745 rFlagsReg cr;
13746 compI_reg_imm0(cr, src);
13747 cmovI_reg_imm0_gt(dst, src, cr);
13748 %}
13749 %}
13750
13751 // This pattern is automatically generated from aarch64_ad.m4
13752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13753 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13754 %{
13755 match(Set dst (MaxI imm src));
13756 ins_cost(INSN_COST * 3);
13757 expand %{
13758 rFlagsReg cr;
13759 compI_reg_imm0(cr, src);
13760 cmovI_reg_imm0_gt(dst, src, cr);
13761 %}
13762 %}
13763
13764 // This pattern is automatically generated from aarch64_ad.m4
13765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13766 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13767 %{
13768 match(Set dst (MaxI src imm));
13769 ins_cost(INSN_COST * 3);
13770 expand %{
13771 rFlagsReg cr;
13772 compI_reg_imm0(cr, src);
13773 cmovI_reg_imm1_gt(dst, src, cr);
13774 %}
13775 %}
13776
13777 // This pattern is automatically generated from aarch64_ad.m4
13778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13779 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13780 %{
13781 match(Set dst (MaxI imm src));
13782 ins_cost(INSN_COST * 3);
13783 expand %{
13784 rFlagsReg cr;
13785 compI_reg_imm0(cr, src);
13786 cmovI_reg_imm1_gt(dst, src, cr);
13787 %}
13788 %}
13789
13790 // This pattern is automatically generated from aarch64_ad.m4
13791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13792 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13793 %{
13794 match(Set dst (MaxI src imm));
13795 ins_cost(INSN_COST * 3);
13796 expand %{
13797 rFlagsReg cr;
13798 compI_reg_imm0(cr, src);
13799 cmovI_reg_immM1_ge(dst, src, cr);
13800 %}
13801 %}
13802
13803 // This pattern is automatically generated from aarch64_ad.m4
13804 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13805 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13806 %{
13807 match(Set dst (MaxI imm src));
13808 ins_cost(INSN_COST * 3);
13809 expand %{
13810 rFlagsReg cr;
13811 compI_reg_imm0(cr, src);
13812 cmovI_reg_immM1_ge(dst, src, cr);
13813 %}
13814 %}
13815
13816 // This pattern is automatically generated from aarch64_ad.m4
13817 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13818 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13819 %{
13820 match(Set dst (ReverseI src));
13821 ins_cost(INSN_COST);
13822 format %{ "rbitw $dst, $src" %}
13823 ins_encode %{
13824 __ rbitw($dst$$Register, $src$$Register);
13825 %}
13826 ins_pipe(ialu_reg);
13827 %}
13828
13829 // This pattern is automatically generated from aarch64_ad.m4
13830 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13831 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13832 %{
13833 match(Set dst (ReverseL src));
13834 ins_cost(INSN_COST);
13835 format %{ "rbit $dst, $src" %}
13836 ins_encode %{
13837 __ rbit($dst$$Register, $src$$Register);
13838 %}
13839 ins_pipe(ialu_reg);
13840 %}
13841
13842
13843 // END This section of the file is automatically generated. Do not edit --------------
13844
13845
13846 // ============================================================================
13847 // Floating Point Arithmetic Instructions
13848
13849 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13850 match(Set dst (AddHF src1 src2));
13851 format %{ "faddh $dst, $src1, $src2" %}
13852 ins_encode %{
13853 __ faddh($dst$$FloatRegister,
13854 $src1$$FloatRegister,
13855 $src2$$FloatRegister);
13856 %}
13857 ins_pipe(fp_dop_reg_reg_s);
13858 %}
13859
13860 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13861 match(Set dst (AddF src1 src2));
13862
13863 ins_cost(INSN_COST * 5);
13864 format %{ "fadds $dst, $src1, $src2" %}
13865
13866 ins_encode %{
13867 __ fadds(as_FloatRegister($dst$$reg),
13868 as_FloatRegister($src1$$reg),
13869 as_FloatRegister($src2$$reg));
13870 %}
13871
13872 ins_pipe(fp_dop_reg_reg_s);
13873 %}
13874
13875 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13876 match(Set dst (AddD src1 src2));
13877
13878 ins_cost(INSN_COST * 5);
13879 format %{ "faddd $dst, $src1, $src2" %}
13880
13881 ins_encode %{
13882 __ faddd(as_FloatRegister($dst$$reg),
13883 as_FloatRegister($src1$$reg),
13884 as_FloatRegister($src2$$reg));
13885 %}
13886
13887 ins_pipe(fp_dop_reg_reg_d);
13888 %}
13889
13890 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13891 match(Set dst (SubHF src1 src2));
13892 format %{ "fsubh $dst, $src1, $src2" %}
13893 ins_encode %{
13894 __ fsubh($dst$$FloatRegister,
13895 $src1$$FloatRegister,
13896 $src2$$FloatRegister);
13897 %}
13898 ins_pipe(fp_dop_reg_reg_s);
13899 %}
13900
13901 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13902 match(Set dst (SubF src1 src2));
13903
13904 ins_cost(INSN_COST * 5);
13905 format %{ "fsubs $dst, $src1, $src2" %}
13906
13907 ins_encode %{
13908 __ fsubs(as_FloatRegister($dst$$reg),
13909 as_FloatRegister($src1$$reg),
13910 as_FloatRegister($src2$$reg));
13911 %}
13912
13913 ins_pipe(fp_dop_reg_reg_s);
13914 %}
13915
13916 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13917 match(Set dst (SubD src1 src2));
13918
13919 ins_cost(INSN_COST * 5);
13920 format %{ "fsubd $dst, $src1, $src2" %}
13921
13922 ins_encode %{
13923 __ fsubd(as_FloatRegister($dst$$reg),
13924 as_FloatRegister($src1$$reg),
13925 as_FloatRegister($src2$$reg));
13926 %}
13927
13928 ins_pipe(fp_dop_reg_reg_d);
13929 %}
13930
13931 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13932 match(Set dst (MulHF src1 src2));
13933 format %{ "fmulh $dst, $src1, $src2" %}
13934 ins_encode %{
13935 __ fmulh($dst$$FloatRegister,
13936 $src1$$FloatRegister,
13937 $src2$$FloatRegister);
13938 %}
13939 ins_pipe(fp_dop_reg_reg_s);
13940 %}
13941
13942 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13943 match(Set dst (MulF src1 src2));
13944
13945 ins_cost(INSN_COST * 6);
13946 format %{ "fmuls $dst, $src1, $src2" %}
13947
13948 ins_encode %{
13949 __ fmuls(as_FloatRegister($dst$$reg),
13950 as_FloatRegister($src1$$reg),
13951 as_FloatRegister($src2$$reg));
13952 %}
13953
13954 ins_pipe(fp_dop_reg_reg_s);
13955 %}
13956
13957 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13958 match(Set dst (MulD src1 src2));
13959
13960 ins_cost(INSN_COST * 6);
13961 format %{ "fmuld $dst, $src1, $src2" %}
13962
13963 ins_encode %{
13964 __ fmuld(as_FloatRegister($dst$$reg),
13965 as_FloatRegister($src1$$reg),
13966 as_FloatRegister($src2$$reg));
13967 %}
13968
13969 ins_pipe(fp_dop_reg_reg_d);
13970 %}
13971
13972 // src1 * src2 + src3 (half-precision float)
13973 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13974 match(Set dst (FmaHF src3 (Binary src1 src2)));
13975 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13976 ins_encode %{
13977 assert(UseFMA, "Needs FMA instructions support.");
13978 __ fmaddh($dst$$FloatRegister,
13979 $src1$$FloatRegister,
13980 $src2$$FloatRegister,
13981 $src3$$FloatRegister);
13982 %}
13983 ins_pipe(pipe_class_default);
13984 %}
13985
13986 // src1 * src2 + src3
13987 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13988 match(Set dst (FmaF src3 (Binary src1 src2)));
13989
13990 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13991
13992 ins_encode %{
13993 assert(UseFMA, "Needs FMA instructions support.");
13994 __ fmadds(as_FloatRegister($dst$$reg),
13995 as_FloatRegister($src1$$reg),
13996 as_FloatRegister($src2$$reg),
13997 as_FloatRegister($src3$$reg));
13998 %}
13999
14000 ins_pipe(pipe_class_default);
14001 %}
14002
14003 // src1 * src2 + src3
14004 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14005 match(Set dst (FmaD src3 (Binary src1 src2)));
14006
14007 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
14008
14009 ins_encode %{
14010 assert(UseFMA, "Needs FMA instructions support.");
14011 __ fmaddd(as_FloatRegister($dst$$reg),
14012 as_FloatRegister($src1$$reg),
14013 as_FloatRegister($src2$$reg),
14014 as_FloatRegister($src3$$reg));
14015 %}
14016
14017 ins_pipe(pipe_class_default);
14018 %}
14019
14020 // src1 * (-src2) + src3
14021 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
14022 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14023 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14024
14025 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
14026
14027 ins_encode %{
14028 assert(UseFMA, "Needs FMA instructions support.");
14029 __ fmsubs(as_FloatRegister($dst$$reg),
14030 as_FloatRegister($src1$$reg),
14031 as_FloatRegister($src2$$reg),
14032 as_FloatRegister($src3$$reg));
14033 %}
14034
14035 ins_pipe(pipe_class_default);
14036 %}
14037
14038 // src1 * (-src2) + src3
14039 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
14040 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14041 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14042
14043 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
14044
14045 ins_encode %{
14046 assert(UseFMA, "Needs FMA instructions support.");
14047 __ fmsubd(as_FloatRegister($dst$$reg),
14048 as_FloatRegister($src1$$reg),
14049 as_FloatRegister($src2$$reg),
14050 as_FloatRegister($src3$$reg));
14051 %}
14052
14053 ins_pipe(pipe_class_default);
14054 %}
14055
14056 // src1 * (-src2) - src3
14057 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14058 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14059 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14060
14061 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
14062
14063 ins_encode %{
14064 assert(UseFMA, "Needs FMA instructions support.");
14065 __ fnmadds(as_FloatRegister($dst$$reg),
14066 as_FloatRegister($src1$$reg),
14067 as_FloatRegister($src2$$reg),
14068 as_FloatRegister($src3$$reg));
14069 %}
14070
14071 ins_pipe(pipe_class_default);
14072 %}
14073
14074 // src1 * (-src2) - src3
14075 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14076 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14077 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14078
14079 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14080
14081 ins_encode %{
14082 assert(UseFMA, "Needs FMA instructions support.");
14083 __ fnmaddd(as_FloatRegister($dst$$reg),
14084 as_FloatRegister($src1$$reg),
14085 as_FloatRegister($src2$$reg),
14086 as_FloatRegister($src3$$reg));
14087 %}
14088
14089 ins_pipe(pipe_class_default);
14090 %}
14091
14092 // src1 * src2 - src3
14093 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14094 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14095
14096 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14097
14098 ins_encode %{
14099 assert(UseFMA, "Needs FMA instructions support.");
14100 __ fnmsubs(as_FloatRegister($dst$$reg),
14101 as_FloatRegister($src1$$reg),
14102 as_FloatRegister($src2$$reg),
14103 as_FloatRegister($src3$$reg));
14104 %}
14105
14106 ins_pipe(pipe_class_default);
14107 %}
14108
14109 // src1 * src2 - src3
14110 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14111 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14112
14113 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14114
14115 ins_encode %{
14116 assert(UseFMA, "Needs FMA instructions support.");
14117 // n.b. insn name should be fnmsubd
14118 __ fnmsub(as_FloatRegister($dst$$reg),
14119 as_FloatRegister($src1$$reg),
14120 as_FloatRegister($src2$$reg),
14121 as_FloatRegister($src3$$reg));
14122 %}
14123
14124 ins_pipe(pipe_class_default);
14125 %}
14126
14127 // Math.max(HH)H (half-precision float)
14128 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14129 match(Set dst (MaxHF src1 src2));
14130 format %{ "fmaxh $dst, $src1, $src2" %}
14131 ins_encode %{
14132 __ fmaxh($dst$$FloatRegister,
14133 $src1$$FloatRegister,
14134 $src2$$FloatRegister);
14135 %}
14136 ins_pipe(fp_dop_reg_reg_s);
14137 %}
14138
14139 // Math.min(HH)H (half-precision float)
14140 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14141 match(Set dst (MinHF src1 src2));
14142 format %{ "fminh $dst, $src1, $src2" %}
14143 ins_encode %{
14144 __ fminh($dst$$FloatRegister,
14145 $src1$$FloatRegister,
14146 $src2$$FloatRegister);
14147 %}
14148 ins_pipe(fp_dop_reg_reg_s);
14149 %}
14150
14151 // Math.max(FF)F
14152 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14153 match(Set dst (MaxF src1 src2));
14154
14155 format %{ "fmaxs $dst, $src1, $src2" %}
14156 ins_encode %{
14157 __ fmaxs(as_FloatRegister($dst$$reg),
14158 as_FloatRegister($src1$$reg),
14159 as_FloatRegister($src2$$reg));
14160 %}
14161
14162 ins_pipe(fp_dop_reg_reg_s);
14163 %}
14164
14165 // Math.min(FF)F
14166 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14167 match(Set dst (MinF src1 src2));
14168
14169 format %{ "fmins $dst, $src1, $src2" %}
14170 ins_encode %{
14171 __ fmins(as_FloatRegister($dst$$reg),
14172 as_FloatRegister($src1$$reg),
14173 as_FloatRegister($src2$$reg));
14174 %}
14175
14176 ins_pipe(fp_dop_reg_reg_s);
14177 %}
14178
14179 // Math.max(DD)D
14180 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14181 match(Set dst (MaxD src1 src2));
14182
14183 format %{ "fmaxd $dst, $src1, $src2" %}
14184 ins_encode %{
14185 __ fmaxd(as_FloatRegister($dst$$reg),
14186 as_FloatRegister($src1$$reg),
14187 as_FloatRegister($src2$$reg));
14188 %}
14189
14190 ins_pipe(fp_dop_reg_reg_d);
14191 %}
14192
14193 // Math.min(DD)D
14194 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14195 match(Set dst (MinD src1 src2));
14196
14197 format %{ "fmind $dst, $src1, $src2" %}
14198 ins_encode %{
14199 __ fmind(as_FloatRegister($dst$$reg),
14200 as_FloatRegister($src1$$reg),
14201 as_FloatRegister($src2$$reg));
14202 %}
14203
14204 ins_pipe(fp_dop_reg_reg_d);
14205 %}
14206
14207 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14208 match(Set dst (DivHF src1 src2));
14209 format %{ "fdivh $dst, $src1, $src2" %}
14210 ins_encode %{
14211 __ fdivh($dst$$FloatRegister,
14212 $src1$$FloatRegister,
14213 $src2$$FloatRegister);
14214 %}
14215 ins_pipe(fp_div_s);
14216 %}
14217
14218 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14219 match(Set dst (DivF src1 src2));
14220
14221 ins_cost(INSN_COST * 18);
14222 format %{ "fdivs $dst, $src1, $src2" %}
14223
14224 ins_encode %{
14225 __ fdivs(as_FloatRegister($dst$$reg),
14226 as_FloatRegister($src1$$reg),
14227 as_FloatRegister($src2$$reg));
14228 %}
14229
14230 ins_pipe(fp_div_s);
14231 %}
14232
14233 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14234 match(Set dst (DivD src1 src2));
14235
14236 ins_cost(INSN_COST * 32);
14237 format %{ "fdivd $dst, $src1, $src2" %}
14238
14239 ins_encode %{
14240 __ fdivd(as_FloatRegister($dst$$reg),
14241 as_FloatRegister($src1$$reg),
14242 as_FloatRegister($src2$$reg));
14243 %}
14244
14245 ins_pipe(fp_div_d);
14246 %}
14247
14248 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14249 match(Set dst (NegF src));
14250
14251 ins_cost(INSN_COST * 3);
14252 format %{ "fneg $dst, $src" %}
14253
14254 ins_encode %{
14255 __ fnegs(as_FloatRegister($dst$$reg),
14256 as_FloatRegister($src$$reg));
14257 %}
14258
14259 ins_pipe(fp_uop_s);
14260 %}
14261
14262 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14263 match(Set dst (NegD src));
14264
14265 ins_cost(INSN_COST * 3);
14266 format %{ "fnegd $dst, $src" %}
14267
14268 ins_encode %{
14269 __ fnegd(as_FloatRegister($dst$$reg),
14270 as_FloatRegister($src$$reg));
14271 %}
14272
14273 ins_pipe(fp_uop_d);
14274 %}
14275
14276 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14277 %{
14278 match(Set dst (AbsI src));
14279
14280 effect(KILL cr);
14281 ins_cost(INSN_COST * 2);
14282 format %{ "cmpw $src, zr\n\t"
14283 "cnegw $dst, $src, Assembler::LT\t# int abs"
14284 %}
14285
14286 ins_encode %{
14287 __ cmpw(as_Register($src$$reg), zr);
14288 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14289 %}
14290 ins_pipe(pipe_class_default);
14291 %}
14292
14293 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14294 %{
14295 match(Set dst (AbsL src));
14296
14297 effect(KILL cr);
14298 ins_cost(INSN_COST * 2);
14299 format %{ "cmp $src, zr\n\t"
14300 "cneg $dst, $src, Assembler::LT\t# long abs"
14301 %}
14302
14303 ins_encode %{
14304 __ cmp(as_Register($src$$reg), zr);
14305 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14306 %}
14307 ins_pipe(pipe_class_default);
14308 %}
14309
14310 instruct absF_reg(vRegF dst, vRegF src) %{
14311 match(Set dst (AbsF src));
14312
14313 ins_cost(INSN_COST * 3);
14314 format %{ "fabss $dst, $src" %}
14315 ins_encode %{
14316 __ fabss(as_FloatRegister($dst$$reg),
14317 as_FloatRegister($src$$reg));
14318 %}
14319
14320 ins_pipe(fp_uop_s);
14321 %}
14322
14323 instruct absD_reg(vRegD dst, vRegD src) %{
14324 match(Set dst (AbsD src));
14325
14326 ins_cost(INSN_COST * 3);
14327 format %{ "fabsd $dst, $src" %}
14328 ins_encode %{
14329 __ fabsd(as_FloatRegister($dst$$reg),
14330 as_FloatRegister($src$$reg));
14331 %}
14332
14333 ins_pipe(fp_uop_d);
14334 %}
14335
14336 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14337 match(Set dst (AbsF (SubF src1 src2)));
14338
14339 ins_cost(INSN_COST * 3);
14340 format %{ "fabds $dst, $src1, $src2" %}
14341 ins_encode %{
14342 __ fabds(as_FloatRegister($dst$$reg),
14343 as_FloatRegister($src1$$reg),
14344 as_FloatRegister($src2$$reg));
14345 %}
14346
14347 ins_pipe(fp_uop_s);
14348 %}
14349
14350 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14351 match(Set dst (AbsD (SubD src1 src2)));
14352
14353 ins_cost(INSN_COST * 3);
14354 format %{ "fabdd $dst, $src1, $src2" %}
14355 ins_encode %{
14356 __ fabdd(as_FloatRegister($dst$$reg),
14357 as_FloatRegister($src1$$reg),
14358 as_FloatRegister($src2$$reg));
14359 %}
14360
14361 ins_pipe(fp_uop_d);
14362 %}
14363
14364 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14365 match(Set dst (SqrtD src));
14366
14367 ins_cost(INSN_COST * 50);
14368 format %{ "fsqrtd $dst, $src" %}
14369 ins_encode %{
14370 __ fsqrtd(as_FloatRegister($dst$$reg),
14371 as_FloatRegister($src$$reg));
14372 %}
14373
14374 ins_pipe(fp_div_s);
14375 %}
14376
14377 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14378 match(Set dst (SqrtF src));
14379
14380 ins_cost(INSN_COST * 50);
14381 format %{ "fsqrts $dst, $src" %}
14382 ins_encode %{
14383 __ fsqrts(as_FloatRegister($dst$$reg),
14384 as_FloatRegister($src$$reg));
14385 %}
14386
14387 ins_pipe(fp_div_d);
14388 %}
14389
14390 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14391 match(Set dst (SqrtHF src));
14392 format %{ "fsqrth $dst, $src" %}
14393 ins_encode %{
14394 __ fsqrth($dst$$FloatRegister,
14395 $src$$FloatRegister);
14396 %}
14397 ins_pipe(fp_div_s);
14398 %}
14399
14400 // Math.rint, floor, ceil
14401 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14402 match(Set dst (RoundDoubleMode src rmode));
14403 format %{ "frint $dst, $src, $rmode" %}
14404 ins_encode %{
14405 switch ($rmode$$constant) {
14406 case RoundDoubleModeNode::rmode_rint:
14407 __ frintnd(as_FloatRegister($dst$$reg),
14408 as_FloatRegister($src$$reg));
14409 break;
14410 case RoundDoubleModeNode::rmode_floor:
14411 __ frintmd(as_FloatRegister($dst$$reg),
14412 as_FloatRegister($src$$reg));
14413 break;
14414 case RoundDoubleModeNode::rmode_ceil:
14415 __ frintpd(as_FloatRegister($dst$$reg),
14416 as_FloatRegister($src$$reg));
14417 break;
14418 }
14419 %}
14420 ins_pipe(fp_uop_d);
14421 %}
14422
14423 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14424 match(Set dst (CopySignD src1 (Binary src2 zero)));
14425 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14426 format %{ "CopySignD $dst $src1 $src2" %}
14427 ins_encode %{
14428 FloatRegister dst = as_FloatRegister($dst$$reg),
14429 src1 = as_FloatRegister($src1$$reg),
14430 src2 = as_FloatRegister($src2$$reg),
14431 zero = as_FloatRegister($zero$$reg);
14432 __ fnegd(dst, zero);
14433 __ bsl(dst, __ T8B, src2, src1);
14434 %}
14435 ins_pipe(fp_uop_d);
14436 %}
14437
14438 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14439 match(Set dst (CopySignF src1 src2));
14440 effect(TEMP_DEF dst, USE src1, USE src2);
14441 format %{ "CopySignF $dst $src1 $src2" %}
14442 ins_encode %{
14443 FloatRegister dst = as_FloatRegister($dst$$reg),
14444 src1 = as_FloatRegister($src1$$reg),
14445 src2 = as_FloatRegister($src2$$reg);
14446 __ movi(dst, __ T2S, 0x80, 24);
14447 __ bsl(dst, __ T8B, src2, src1);
14448 %}
14449 ins_pipe(fp_uop_d);
14450 %}
14451
14452 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14453 match(Set dst (SignumD src (Binary zero one)));
14454 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14455 format %{ "signumD $dst, $src" %}
14456 ins_encode %{
14457 FloatRegister src = as_FloatRegister($src$$reg),
14458 dst = as_FloatRegister($dst$$reg),
14459 zero = as_FloatRegister($zero$$reg),
14460 one = as_FloatRegister($one$$reg);
14461 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14462 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14463 // Bit selection instruction gets bit from "one" for each enabled bit in
14464 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14465 // NaN the whole "src" will be copied because "dst" is zero. For all other
14466 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14467 // from "src", and all other bits are copied from 1.0.
14468 __ bsl(dst, __ T8B, one, src);
14469 %}
14470 ins_pipe(fp_uop_d);
14471 %}
14472
14473 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14474 match(Set dst (SignumF src (Binary zero one)));
14475 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14476 format %{ "signumF $dst, $src" %}
14477 ins_encode %{
14478 FloatRegister src = as_FloatRegister($src$$reg),
14479 dst = as_FloatRegister($dst$$reg),
14480 zero = as_FloatRegister($zero$$reg),
14481 one = as_FloatRegister($one$$reg);
14482 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14483 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14484 // Bit selection instruction gets bit from "one" for each enabled bit in
14485 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14486 // NaN the whole "src" will be copied because "dst" is zero. For all other
14487 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14488 // from "src", and all other bits are copied from 1.0.
14489 __ bsl(dst, __ T8B, one, src);
14490 %}
14491 ins_pipe(fp_uop_d);
14492 %}
14493
14494 instruct onspinwait() %{
14495 match(OnSpinWait);
14496 ins_cost(INSN_COST);
14497
14498 format %{ "onspinwait" %}
14499
14500 ins_encode %{
14501 __ spin_wait();
14502 %}
14503 ins_pipe(pipe_class_empty);
14504 %}
14505
14506 // ============================================================================
14507 // Logical Instructions
14508
14509 // Integer Logical Instructions
14510
14511 // And Instructions
14512
14513
14514 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14515 match(Set dst (AndI src1 src2));
14516
14517 format %{ "andw $dst, $src1, $src2\t# int" %}
14518
14519 ins_cost(INSN_COST);
14520 ins_encode %{
14521 __ andw(as_Register($dst$$reg),
14522 as_Register($src1$$reg),
14523 as_Register($src2$$reg));
14524 %}
14525
14526 ins_pipe(ialu_reg_reg);
14527 %}
14528
14529 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14530 match(Set dst (AndI src1 src2));
14531
14532 format %{ "andsw $dst, $src1, $src2\t# int" %}
14533
14534 ins_cost(INSN_COST);
14535 ins_encode %{
14536 __ andw(as_Register($dst$$reg),
14537 as_Register($src1$$reg),
14538 (uint64_t)($src2$$constant));
14539 %}
14540
14541 ins_pipe(ialu_reg_imm);
14542 %}
14543
14544 // Or Instructions
14545
14546 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14547 match(Set dst (OrI src1 src2));
14548
14549 format %{ "orrw $dst, $src1, $src2\t# int" %}
14550
14551 ins_cost(INSN_COST);
14552 ins_encode %{
14553 __ orrw(as_Register($dst$$reg),
14554 as_Register($src1$$reg),
14555 as_Register($src2$$reg));
14556 %}
14557
14558 ins_pipe(ialu_reg_reg);
14559 %}
14560
14561 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14562 match(Set dst (OrI src1 src2));
14563
14564 format %{ "orrw $dst, $src1, $src2\t# int" %}
14565
14566 ins_cost(INSN_COST);
14567 ins_encode %{
14568 __ orrw(as_Register($dst$$reg),
14569 as_Register($src1$$reg),
14570 (uint64_t)($src2$$constant));
14571 %}
14572
14573 ins_pipe(ialu_reg_imm);
14574 %}
14575
14576 // Xor Instructions
14577
14578 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14579 match(Set dst (XorI src1 src2));
14580
14581 format %{ "eorw $dst, $src1, $src2\t# int" %}
14582
14583 ins_cost(INSN_COST);
14584 ins_encode %{
14585 __ eorw(as_Register($dst$$reg),
14586 as_Register($src1$$reg),
14587 as_Register($src2$$reg));
14588 %}
14589
14590 ins_pipe(ialu_reg_reg);
14591 %}
14592
14593 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14594 match(Set dst (XorI src1 src2));
14595
14596 format %{ "eorw $dst, $src1, $src2\t# int" %}
14597
14598 ins_cost(INSN_COST);
14599 ins_encode %{
14600 __ eorw(as_Register($dst$$reg),
14601 as_Register($src1$$reg),
14602 (uint64_t)($src2$$constant));
14603 %}
14604
14605 ins_pipe(ialu_reg_imm);
14606 %}
14607
14608 // Long Logical Instructions
14609 // TODO
14610
14611 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14612 match(Set dst (AndL src1 src2));
14613
14614 format %{ "and $dst, $src1, $src2\t# int" %}
14615
14616 ins_cost(INSN_COST);
14617 ins_encode %{
14618 __ andr(as_Register($dst$$reg),
14619 as_Register($src1$$reg),
14620 as_Register($src2$$reg));
14621 %}
14622
14623 ins_pipe(ialu_reg_reg);
14624 %}
14625
14626 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14627 match(Set dst (AndL src1 src2));
14628
14629 format %{ "and $dst, $src1, $src2\t# int" %}
14630
14631 ins_cost(INSN_COST);
14632 ins_encode %{
14633 __ andr(as_Register($dst$$reg),
14634 as_Register($src1$$reg),
14635 (uint64_t)($src2$$constant));
14636 %}
14637
14638 ins_pipe(ialu_reg_imm);
14639 %}
14640
14641 // Or Instructions
14642
14643 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14644 match(Set dst (OrL src1 src2));
14645
14646 format %{ "orr $dst, $src1, $src2\t# int" %}
14647
14648 ins_cost(INSN_COST);
14649 ins_encode %{
14650 __ orr(as_Register($dst$$reg),
14651 as_Register($src1$$reg),
14652 as_Register($src2$$reg));
14653 %}
14654
14655 ins_pipe(ialu_reg_reg);
14656 %}
14657
14658 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14659 match(Set dst (OrL src1 src2));
14660
14661 format %{ "orr $dst, $src1, $src2\t# int" %}
14662
14663 ins_cost(INSN_COST);
14664 ins_encode %{
14665 __ orr(as_Register($dst$$reg),
14666 as_Register($src1$$reg),
14667 (uint64_t)($src2$$constant));
14668 %}
14669
14670 ins_pipe(ialu_reg_imm);
14671 %}
14672
14673 // Xor Instructions
14674
14675 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14676 match(Set dst (XorL src1 src2));
14677
14678 format %{ "eor $dst, $src1, $src2\t# int" %}
14679
14680 ins_cost(INSN_COST);
14681 ins_encode %{
14682 __ eor(as_Register($dst$$reg),
14683 as_Register($src1$$reg),
14684 as_Register($src2$$reg));
14685 %}
14686
14687 ins_pipe(ialu_reg_reg);
14688 %}
14689
14690 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14691 match(Set dst (XorL src1 src2));
14692
14693 ins_cost(INSN_COST);
14694 format %{ "eor $dst, $src1, $src2\t# int" %}
14695
14696 ins_encode %{
14697 __ eor(as_Register($dst$$reg),
14698 as_Register($src1$$reg),
14699 (uint64_t)($src2$$constant));
14700 %}
14701
14702 ins_pipe(ialu_reg_imm);
14703 %}
14704
14705 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14706 %{
14707 match(Set dst (ConvI2L src));
14708
14709 ins_cost(INSN_COST);
14710 format %{ "sxtw $dst, $src\t# i2l" %}
14711 ins_encode %{
14712 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14713 %}
14714 ins_pipe(ialu_reg_shift);
14715 %}
14716
14717 // this pattern occurs in bigmath arithmetic
14718 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14719 %{
14720 match(Set dst (AndL (ConvI2L src) mask));
14721
14722 ins_cost(INSN_COST);
14723 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14724 ins_encode %{
14725 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14726 %}
14727
14728 ins_pipe(ialu_reg_shift);
14729 %}
14730
14731 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14732 match(Set dst (ConvL2I src));
14733
14734 ins_cost(INSN_COST);
14735 format %{ "movw $dst, $src \t// l2i" %}
14736
14737 ins_encode %{
14738 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14739 %}
14740
14741 ins_pipe(ialu_reg);
14742 %}
14743
14744 instruct convD2F_reg(vRegF dst, vRegD src) %{
14745 match(Set dst (ConvD2F src));
14746
14747 ins_cost(INSN_COST * 5);
14748 format %{ "fcvtd $dst, $src \t// d2f" %}
14749
14750 ins_encode %{
14751 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14752 %}
14753
14754 ins_pipe(fp_d2f);
14755 %}
14756
14757 instruct convF2D_reg(vRegD dst, vRegF src) %{
14758 match(Set dst (ConvF2D src));
14759
14760 ins_cost(INSN_COST * 5);
14761 format %{ "fcvts $dst, $src \t// f2d" %}
14762
14763 ins_encode %{
14764 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14765 %}
14766
14767 ins_pipe(fp_f2d);
14768 %}
14769
14770 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14771 match(Set dst (ConvF2I src));
14772
14773 ins_cost(INSN_COST * 5);
14774 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14775
14776 ins_encode %{
14777 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14778 %}
14779
14780 ins_pipe(fp_f2i);
14781 %}
14782
14783 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14784 match(Set dst (ConvF2L src));
14785
14786 ins_cost(INSN_COST * 5);
14787 format %{ "fcvtzs $dst, $src \t// f2l" %}
14788
14789 ins_encode %{
14790 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14791 %}
14792
14793 ins_pipe(fp_f2l);
14794 %}
14795
14796 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14797 match(Set dst (ConvF2HF src));
14798 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14799 "smov $dst, $tmp\t# move result from $tmp to $dst"
14800 %}
14801 effect(TEMP tmp);
14802 ins_encode %{
14803 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14804 %}
14805 ins_pipe(pipe_slow);
14806 %}
14807
14808 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14809 match(Set dst (ConvHF2F src));
14810 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14811 "fcvt $dst, $tmp\t# convert half to single precision"
14812 %}
14813 effect(TEMP tmp);
14814 ins_encode %{
14815 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14816 %}
14817 ins_pipe(pipe_slow);
14818 %}
14819
14820 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14821 match(Set dst (ConvI2F src));
14822
14823 ins_cost(INSN_COST * 5);
14824 format %{ "scvtfws $dst, $src \t// i2f" %}
14825
14826 ins_encode %{
14827 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14828 %}
14829
14830 ins_pipe(fp_i2f);
14831 %}
14832
14833 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14834 match(Set dst (ConvL2F src));
14835
14836 ins_cost(INSN_COST * 5);
14837 format %{ "scvtfs $dst, $src \t// l2f" %}
14838
14839 ins_encode %{
14840 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14841 %}
14842
14843 ins_pipe(fp_l2f);
14844 %}
14845
14846 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14847 match(Set dst (ConvD2I src));
14848
14849 ins_cost(INSN_COST * 5);
14850 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14851
14852 ins_encode %{
14853 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14854 %}
14855
14856 ins_pipe(fp_d2i);
14857 %}
14858
14859 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14860 match(Set dst (ConvD2L src));
14861
14862 ins_cost(INSN_COST * 5);
14863 format %{ "fcvtzd $dst, $src \t// d2l" %}
14864
14865 ins_encode %{
14866 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14867 %}
14868
14869 ins_pipe(fp_d2l);
14870 %}
14871
14872 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14873 match(Set dst (ConvI2D src));
14874
14875 ins_cost(INSN_COST * 5);
14876 format %{ "scvtfwd $dst, $src \t// i2d" %}
14877
14878 ins_encode %{
14879 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14880 %}
14881
14882 ins_pipe(fp_i2d);
14883 %}
14884
14885 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14886 match(Set dst (ConvL2D src));
14887
14888 ins_cost(INSN_COST * 5);
14889 format %{ "scvtfd $dst, $src \t// l2d" %}
14890
14891 ins_encode %{
14892 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14893 %}
14894
14895 ins_pipe(fp_l2d);
14896 %}
14897
14898 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14899 %{
14900 match(Set dst (RoundD src));
14901 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14902 format %{ "java_round_double $dst,$src"%}
14903 ins_encode %{
14904 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14905 as_FloatRegister($ftmp$$reg));
14906 %}
14907 ins_pipe(pipe_slow);
14908 %}
14909
14910 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14911 %{
14912 match(Set dst (RoundF src));
14913 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14914 format %{ "java_round_float $dst,$src"%}
14915 ins_encode %{
14916 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14917 as_FloatRegister($ftmp$$reg));
14918 %}
14919 ins_pipe(pipe_slow);
14920 %}
14921
14922 // stack <-> reg and reg <-> reg shuffles with no conversion
14923
14924 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14925
14926 match(Set dst (MoveF2I src));
14927
14928 effect(DEF dst, USE src);
14929
14930 ins_cost(4 * INSN_COST);
14931
14932 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14933
14934 ins_encode %{
14935 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14936 %}
14937
14938 ins_pipe(iload_reg_reg);
14939
14940 %}
14941
14942 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14943
14944 match(Set dst (MoveI2F src));
14945
14946 effect(DEF dst, USE src);
14947
14948 ins_cost(4 * INSN_COST);
14949
14950 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14951
14952 ins_encode %{
14953 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14954 %}
14955
14956 ins_pipe(pipe_class_memory);
14957
14958 %}
14959
14960 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14961
14962 match(Set dst (MoveD2L src));
14963
14964 effect(DEF dst, USE src);
14965
14966 ins_cost(4 * INSN_COST);
14967
14968 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14969
14970 ins_encode %{
14971 __ ldr($dst$$Register, Address(sp, $src$$disp));
14972 %}
14973
14974 ins_pipe(iload_reg_reg);
14975
14976 %}
14977
14978 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14979
14980 match(Set dst (MoveL2D src));
14981
14982 effect(DEF dst, USE src);
14983
14984 ins_cost(4 * INSN_COST);
14985
14986 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14987
14988 ins_encode %{
14989 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14990 %}
14991
14992 ins_pipe(pipe_class_memory);
14993
14994 %}
14995
14996 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14997
14998 match(Set dst (MoveF2I src));
14999
15000 effect(DEF dst, USE src);
15001
15002 ins_cost(INSN_COST);
15003
15004 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15005
15006 ins_encode %{
15007 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15008 %}
15009
15010 ins_pipe(pipe_class_memory);
15011
15012 %}
15013
15014 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15015
15016 match(Set dst (MoveI2F src));
15017
15018 effect(DEF dst, USE src);
15019
15020 ins_cost(INSN_COST);
15021
15022 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15023
15024 ins_encode %{
15025 __ strw($src$$Register, Address(sp, $dst$$disp));
15026 %}
15027
15028 ins_pipe(istore_reg_reg);
15029
15030 %}
15031
15032 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15033
15034 match(Set dst (MoveD2L src));
15035
15036 effect(DEF dst, USE src);
15037
15038 ins_cost(INSN_COST);
15039
15040 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15041
15042 ins_encode %{
15043 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15044 %}
15045
15046 ins_pipe(pipe_class_memory);
15047
15048 %}
15049
15050 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15051
15052 match(Set dst (MoveL2D src));
15053
15054 effect(DEF dst, USE src);
15055
15056 ins_cost(INSN_COST);
15057
15058 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15059
15060 ins_encode %{
15061 __ str($src$$Register, Address(sp, $dst$$disp));
15062 %}
15063
15064 ins_pipe(istore_reg_reg);
15065
15066 %}
15067
15068 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15069
15070 match(Set dst (MoveF2I src));
15071
15072 effect(DEF dst, USE src);
15073
15074 ins_cost(INSN_COST);
15075
15076 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15077
15078 ins_encode %{
15079 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15080 %}
15081
15082 ins_pipe(fp_f2i);
15083
15084 %}
15085
15086 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15087
15088 match(Set dst (MoveI2F src));
15089
15090 effect(DEF dst, USE src);
15091
15092 ins_cost(INSN_COST);
15093
15094 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15095
15096 ins_encode %{
15097 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15098 %}
15099
15100 ins_pipe(fp_i2f);
15101
15102 %}
15103
15104 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15105
15106 match(Set dst (MoveD2L src));
15107
15108 effect(DEF dst, USE src);
15109
15110 ins_cost(INSN_COST);
15111
15112 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15113
15114 ins_encode %{
15115 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15116 %}
15117
15118 ins_pipe(fp_d2l);
15119
15120 %}
15121
15122 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15123
15124 match(Set dst (MoveL2D src));
15125
15126 effect(DEF dst, USE src);
15127
15128 ins_cost(INSN_COST);
15129
15130 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15131
15132 ins_encode %{
15133 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15134 %}
15135
15136 ins_pipe(fp_l2d);
15137
15138 %}
15139
15140 // ============================================================================
15141 // clearing of an array
15142
15143 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15144 %{
15145 match(Set dummy (ClearArray cnt base));
15146 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15147
15148 ins_cost(4 * INSN_COST);
15149 format %{ "ClearArray $cnt, $base" %}
15150
15151 ins_encode %{
15152 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15153 if (tpc == nullptr) {
15154 ciEnv::current()->record_failure("CodeCache is full");
15155 return;
15156 }
15157 %}
15158
15159 ins_pipe(pipe_class_memory);
15160 %}
15161
15162 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15163 %{
15164 predicate((uint64_t)n->in(2)->get_long()
15165 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15166 match(Set dummy (ClearArray cnt base));
15167 effect(TEMP temp, USE_KILL base, KILL cr);
15168
15169 ins_cost(4 * INSN_COST);
15170 format %{ "ClearArray $cnt, $base" %}
15171
15172 ins_encode %{
15173 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15174 if (tpc == nullptr) {
15175 ciEnv::current()->record_failure("CodeCache is full");
15176 return;
15177 }
15178 %}
15179
15180 ins_pipe(pipe_class_memory);
15181 %}
15182
15183 // ============================================================================
15184 // Overflow Math Instructions
15185
15186 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15187 %{
15188 match(Set cr (OverflowAddI op1 op2));
15189
15190 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15191 ins_cost(INSN_COST);
15192 ins_encode %{
15193 __ cmnw($op1$$Register, $op2$$Register);
15194 %}
15195
15196 ins_pipe(icmp_reg_reg);
15197 %}
15198
15199 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15200 %{
15201 match(Set cr (OverflowAddI op1 op2));
15202
15203 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15204 ins_cost(INSN_COST);
15205 ins_encode %{
15206 __ cmnw($op1$$Register, $op2$$constant);
15207 %}
15208
15209 ins_pipe(icmp_reg_imm);
15210 %}
15211
15212 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15213 %{
15214 match(Set cr (OverflowAddL op1 op2));
15215
15216 format %{ "cmn $op1, $op2\t# overflow check long" %}
15217 ins_cost(INSN_COST);
15218 ins_encode %{
15219 __ cmn($op1$$Register, $op2$$Register);
15220 %}
15221
15222 ins_pipe(icmp_reg_reg);
15223 %}
15224
15225 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15226 %{
15227 match(Set cr (OverflowAddL op1 op2));
15228
15229 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15230 ins_cost(INSN_COST);
15231 ins_encode %{
15232 __ adds(zr, $op1$$Register, $op2$$constant);
15233 %}
15234
15235 ins_pipe(icmp_reg_imm);
15236 %}
15237
15238 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15239 %{
15240 match(Set cr (OverflowSubI op1 op2));
15241
15242 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15243 ins_cost(INSN_COST);
15244 ins_encode %{
15245 __ cmpw($op1$$Register, $op2$$Register);
15246 %}
15247
15248 ins_pipe(icmp_reg_reg);
15249 %}
15250
15251 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15252 %{
15253 match(Set cr (OverflowSubI op1 op2));
15254
15255 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15256 ins_cost(INSN_COST);
15257 ins_encode %{
15258 __ cmpw($op1$$Register, $op2$$constant);
15259 %}
15260
15261 ins_pipe(icmp_reg_imm);
15262 %}
15263
15264 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15265 %{
15266 match(Set cr (OverflowSubL op1 op2));
15267
15268 format %{ "cmp $op1, $op2\t# overflow check long" %}
15269 ins_cost(INSN_COST);
15270 ins_encode %{
15271 __ cmp($op1$$Register, $op2$$Register);
15272 %}
15273
15274 ins_pipe(icmp_reg_reg);
15275 %}
15276
15277 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15278 %{
15279 match(Set cr (OverflowSubL op1 op2));
15280
15281 format %{ "cmp $op1, $op2\t# overflow check long" %}
15282 ins_cost(INSN_COST);
15283 ins_encode %{
15284 __ subs(zr, $op1$$Register, $op2$$constant);
15285 %}
15286
15287 ins_pipe(icmp_reg_imm);
15288 %}
15289
15290 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15291 %{
15292 match(Set cr (OverflowSubI zero op1));
15293
15294 format %{ "cmpw zr, $op1\t# overflow check int" %}
15295 ins_cost(INSN_COST);
15296 ins_encode %{
15297 __ cmpw(zr, $op1$$Register);
15298 %}
15299
15300 ins_pipe(icmp_reg_imm);
15301 %}
15302
15303 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15304 %{
15305 match(Set cr (OverflowSubL zero op1));
15306
15307 format %{ "cmp zr, $op1\t# overflow check long" %}
15308 ins_cost(INSN_COST);
15309 ins_encode %{
15310 __ cmp(zr, $op1$$Register);
15311 %}
15312
15313 ins_pipe(icmp_reg_imm);
15314 %}
15315
15316 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15317 %{
15318 match(Set cr (OverflowMulI op1 op2));
15319
15320 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15321 "cmp rscratch1, rscratch1, sxtw\n\t"
15322 "movw rscratch1, #0x80000000\n\t"
15323 "cselw rscratch1, rscratch1, zr, NE\n\t"
15324 "cmpw rscratch1, #1" %}
15325 ins_cost(5 * INSN_COST);
15326 ins_encode %{
15327 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15328 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15329 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15330 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15331 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15332 %}
15333
15334 ins_pipe(pipe_slow);
15335 %}
15336
15337 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15338 %{
15339 match(If cmp (OverflowMulI op1 op2));
15340 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15341 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15342 effect(USE labl, KILL cr);
15343
15344 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15345 "cmp rscratch1, rscratch1, sxtw\n\t"
15346 "b$cmp $labl" %}
15347 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15348 ins_encode %{
15349 Label* L = $labl$$label;
15350 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15351 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15352 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15353 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15354 %}
15355
15356 ins_pipe(pipe_serial);
15357 %}
15358
15359 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15360 %{
15361 match(Set cr (OverflowMulL op1 op2));
15362
15363 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15364 "smulh rscratch2, $op1, $op2\n\t"
15365 "cmp rscratch2, rscratch1, ASR #63\n\t"
15366 "movw rscratch1, #0x80000000\n\t"
15367 "cselw rscratch1, rscratch1, zr, NE\n\t"
15368 "cmpw rscratch1, #1" %}
15369 ins_cost(6 * INSN_COST);
15370 ins_encode %{
15371 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15372 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15373 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15374 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15375 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15376 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15377 %}
15378
15379 ins_pipe(pipe_slow);
15380 %}
15381
15382 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15383 %{
15384 match(If cmp (OverflowMulL op1 op2));
15385 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15386 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15387 effect(USE labl, KILL cr);
15388
15389 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15390 "smulh rscratch2, $op1, $op2\n\t"
15391 "cmp rscratch2, rscratch1, ASR #63\n\t"
15392 "b$cmp $labl" %}
15393 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15394 ins_encode %{
15395 Label* L = $labl$$label;
15396 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15397 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15398 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15399 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15400 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15401 %}
15402
15403 ins_pipe(pipe_serial);
15404 %}
15405
15406 // ============================================================================
15407 // Compare Instructions
15408
15409 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15410 %{
15411 match(Set cr (CmpI op1 op2));
15412
15413 effect(DEF cr, USE op1, USE op2);
15414
15415 ins_cost(INSN_COST);
15416 format %{ "cmpw $op1, $op2" %}
15417
15418 ins_encode(aarch64_enc_cmpw(op1, op2));
15419
15420 ins_pipe(icmp_reg_reg);
15421 %}
15422
15423 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15424 %{
15425 match(Set cr (CmpI op1 zero));
15426
15427 effect(DEF cr, USE op1);
15428
15429 ins_cost(INSN_COST);
15430 format %{ "cmpw $op1, 0" %}
15431
15432 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15433
15434 ins_pipe(icmp_reg_imm);
15435 %}
15436
15437 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15438 %{
15439 match(Set cr (CmpI op1 op2));
15440
15441 effect(DEF cr, USE op1);
15442
15443 ins_cost(INSN_COST);
15444 format %{ "cmpw $op1, $op2" %}
15445
15446 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15447
15448 ins_pipe(icmp_reg_imm);
15449 %}
15450
15451 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15452 %{
15453 match(Set cr (CmpI op1 op2));
15454
15455 effect(DEF cr, USE op1);
15456
15457 ins_cost(INSN_COST * 2);
15458 format %{ "cmpw $op1, $op2" %}
15459
15460 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15461
15462 ins_pipe(icmp_reg_imm);
15463 %}
15464
15465 // Unsigned compare Instructions; really, same as signed compare
15466 // except it should only be used to feed an If or a CMovI which takes a
15467 // cmpOpU.
15468
15469 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15470 %{
15471 match(Set cr (CmpU op1 op2));
15472
15473 effect(DEF cr, USE op1, USE op2);
15474
15475 ins_cost(INSN_COST);
15476 format %{ "cmpw $op1, $op2\t# unsigned" %}
15477
15478 ins_encode(aarch64_enc_cmpw(op1, op2));
15479
15480 ins_pipe(icmp_reg_reg);
15481 %}
15482
15483 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15484 %{
15485 match(Set cr (CmpU op1 zero));
15486
15487 effect(DEF cr, USE op1);
15488
15489 ins_cost(INSN_COST);
15490 format %{ "cmpw $op1, #0\t# unsigned" %}
15491
15492 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15493
15494 ins_pipe(icmp_reg_imm);
15495 %}
15496
15497 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15498 %{
15499 match(Set cr (CmpU op1 op2));
15500
15501 effect(DEF cr, USE op1);
15502
15503 ins_cost(INSN_COST);
15504 format %{ "cmpw $op1, $op2\t# unsigned" %}
15505
15506 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15507
15508 ins_pipe(icmp_reg_imm);
15509 %}
15510
15511 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15512 %{
15513 match(Set cr (CmpU op1 op2));
15514
15515 effect(DEF cr, USE op1);
15516
15517 ins_cost(INSN_COST * 2);
15518 format %{ "cmpw $op1, $op2\t# unsigned" %}
15519
15520 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15521
15522 ins_pipe(icmp_reg_imm);
15523 %}
15524
15525 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15526 %{
15527 match(Set cr (CmpL op1 op2));
15528
15529 effect(DEF cr, USE op1, USE op2);
15530
15531 ins_cost(INSN_COST);
15532 format %{ "cmp $op1, $op2" %}
15533
15534 ins_encode(aarch64_enc_cmp(op1, op2));
15535
15536 ins_pipe(icmp_reg_reg);
15537 %}
15538
15539 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15540 %{
15541 match(Set cr (CmpL op1 zero));
15542
15543 effect(DEF cr, USE op1);
15544
15545 ins_cost(INSN_COST);
15546 format %{ "tst $op1" %}
15547
15548 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15549
15550 ins_pipe(icmp_reg_imm);
15551 %}
15552
15553 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15554 %{
15555 match(Set cr (CmpL op1 op2));
15556
15557 effect(DEF cr, USE op1);
15558
15559 ins_cost(INSN_COST);
15560 format %{ "cmp $op1, $op2" %}
15561
15562 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15563
15564 ins_pipe(icmp_reg_imm);
15565 %}
15566
15567 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15568 %{
15569 match(Set cr (CmpL op1 op2));
15570
15571 effect(DEF cr, USE op1);
15572
15573 ins_cost(INSN_COST * 2);
15574 format %{ "cmp $op1, $op2" %}
15575
15576 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15577
15578 ins_pipe(icmp_reg_imm);
15579 %}
15580
15581 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15582 %{
15583 match(Set cr (CmpUL op1 op2));
15584
15585 effect(DEF cr, USE op1, USE op2);
15586
15587 ins_cost(INSN_COST);
15588 format %{ "cmp $op1, $op2" %}
15589
15590 ins_encode(aarch64_enc_cmp(op1, op2));
15591
15592 ins_pipe(icmp_reg_reg);
15593 %}
15594
15595 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15596 %{
15597 match(Set cr (CmpUL op1 zero));
15598
15599 effect(DEF cr, USE op1);
15600
15601 ins_cost(INSN_COST);
15602 format %{ "tst $op1" %}
15603
15604 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15605
15606 ins_pipe(icmp_reg_imm);
15607 %}
15608
15609 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15610 %{
15611 match(Set cr (CmpUL op1 op2));
15612
15613 effect(DEF cr, USE op1);
15614
15615 ins_cost(INSN_COST);
15616 format %{ "cmp $op1, $op2" %}
15617
15618 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15619
15620 ins_pipe(icmp_reg_imm);
15621 %}
15622
15623 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15624 %{
15625 match(Set cr (CmpUL op1 op2));
15626
15627 effect(DEF cr, USE op1);
15628
15629 ins_cost(INSN_COST * 2);
15630 format %{ "cmp $op1, $op2" %}
15631
15632 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15633
15634 ins_pipe(icmp_reg_imm);
15635 %}
15636
15637 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15638 %{
15639 match(Set cr (CmpP op1 op2));
15640
15641 effect(DEF cr, USE op1, USE op2);
15642
15643 ins_cost(INSN_COST);
15644 format %{ "cmp $op1, $op2\t // ptr" %}
15645
15646 ins_encode(aarch64_enc_cmpp(op1, op2));
15647
15648 ins_pipe(icmp_reg_reg);
15649 %}
15650
15651 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15652 %{
15653 match(Set cr (CmpN op1 op2));
15654
15655 effect(DEF cr, USE op1, USE op2);
15656
15657 ins_cost(INSN_COST);
15658 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15659
15660 ins_encode(aarch64_enc_cmpn(op1, op2));
15661
15662 ins_pipe(icmp_reg_reg);
15663 %}
15664
15665 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15666 %{
15667 match(Set cr (CmpP op1 zero));
15668
15669 effect(DEF cr, USE op1, USE zero);
15670
15671 ins_cost(INSN_COST);
15672 format %{ "cmp $op1, 0\t // ptr" %}
15673
15674 ins_encode(aarch64_enc_testp(op1));
15675
15676 ins_pipe(icmp_reg_imm);
15677 %}
15678
15679 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15680 %{
15681 match(Set cr (CmpN op1 zero));
15682
15683 effect(DEF cr, USE op1, USE zero);
15684
15685 ins_cost(INSN_COST);
15686 format %{ "cmp $op1, 0\t // compressed ptr" %}
15687
15688 ins_encode(aarch64_enc_testn(op1));
15689
15690 ins_pipe(icmp_reg_imm);
15691 %}
15692
15693 // FP comparisons
15694 //
15695 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15696 // using normal cmpOp. See declaration of rFlagsReg for details.
15697
15698 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15699 %{
15700 match(Set cr (CmpF src1 src2));
15701
15702 ins_cost(3 * INSN_COST);
15703 format %{ "fcmps $src1, $src2" %}
15704
15705 ins_encode %{
15706 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15707 %}
15708
15709 ins_pipe(pipe_class_compare);
15710 %}
15711
15712 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15713 %{
15714 match(Set cr (CmpF src1 src2));
15715
15716 ins_cost(3 * INSN_COST);
15717 format %{ "fcmps $src1, 0.0" %}
15718
15719 ins_encode %{
15720 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15721 %}
15722
15723 ins_pipe(pipe_class_compare);
15724 %}
15725 // FROM HERE
15726
15727 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15728 %{
15729 match(Set cr (CmpD src1 src2));
15730
15731 ins_cost(3 * INSN_COST);
15732 format %{ "fcmpd $src1, $src2" %}
15733
15734 ins_encode %{
15735 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15736 %}
15737
15738 ins_pipe(pipe_class_compare);
15739 %}
15740
15741 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15742 %{
15743 match(Set cr (CmpD src1 src2));
15744
15745 ins_cost(3 * INSN_COST);
15746 format %{ "fcmpd $src1, 0.0" %}
15747
15748 ins_encode %{
15749 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15750 %}
15751
15752 ins_pipe(pipe_class_compare);
15753 %}
15754
15755 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15756 %{
15757 match(Set dst (CmpF3 src1 src2));
15758 effect(KILL cr);
15759
15760 ins_cost(5 * INSN_COST);
15761 format %{ "fcmps $src1, $src2\n\t"
15762 "csinvw($dst, zr, zr, eq\n\t"
15763 "csnegw($dst, $dst, $dst, lt)"
15764 %}
15765
15766 ins_encode %{
15767 Label done;
15768 FloatRegister s1 = as_FloatRegister($src1$$reg);
15769 FloatRegister s2 = as_FloatRegister($src2$$reg);
15770 Register d = as_Register($dst$$reg);
15771 __ fcmps(s1, s2);
15772 // installs 0 if EQ else -1
15773 __ csinvw(d, zr, zr, Assembler::EQ);
15774 // keeps -1 if less or unordered else installs 1
15775 __ csnegw(d, d, d, Assembler::LT);
15776 __ bind(done);
15777 %}
15778
15779 ins_pipe(pipe_class_default);
15780
15781 %}
15782
15783 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15784 %{
15785 match(Set dst (CmpD3 src1 src2));
15786 effect(KILL cr);
15787
15788 ins_cost(5 * INSN_COST);
15789 format %{ "fcmpd $src1, $src2\n\t"
15790 "csinvw($dst, zr, zr, eq\n\t"
15791 "csnegw($dst, $dst, $dst, lt)"
15792 %}
15793
15794 ins_encode %{
15795 Label done;
15796 FloatRegister s1 = as_FloatRegister($src1$$reg);
15797 FloatRegister s2 = as_FloatRegister($src2$$reg);
15798 Register d = as_Register($dst$$reg);
15799 __ fcmpd(s1, s2);
15800 // installs 0 if EQ else -1
15801 __ csinvw(d, zr, zr, Assembler::EQ);
15802 // keeps -1 if less or unordered else installs 1
15803 __ csnegw(d, d, d, Assembler::LT);
15804 __ bind(done);
15805 %}
15806 ins_pipe(pipe_class_default);
15807
15808 %}
15809
15810 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15811 %{
15812 match(Set dst (CmpF3 src1 zero));
15813 effect(KILL cr);
15814
15815 ins_cost(5 * INSN_COST);
15816 format %{ "fcmps $src1, 0.0\n\t"
15817 "csinvw($dst, zr, zr, eq\n\t"
15818 "csnegw($dst, $dst, $dst, lt)"
15819 %}
15820
15821 ins_encode %{
15822 Label done;
15823 FloatRegister s1 = as_FloatRegister($src1$$reg);
15824 Register d = as_Register($dst$$reg);
15825 __ fcmps(s1, 0.0);
15826 // installs 0 if EQ else -1
15827 __ csinvw(d, zr, zr, Assembler::EQ);
15828 // keeps -1 if less or unordered else installs 1
15829 __ csnegw(d, d, d, Assembler::LT);
15830 __ bind(done);
15831 %}
15832
15833 ins_pipe(pipe_class_default);
15834
15835 %}
15836
15837 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15838 %{
15839 match(Set dst (CmpD3 src1 zero));
15840 effect(KILL cr);
15841
15842 ins_cost(5 * INSN_COST);
15843 format %{ "fcmpd $src1, 0.0\n\t"
15844 "csinvw($dst, zr, zr, eq\n\t"
15845 "csnegw($dst, $dst, $dst, lt)"
15846 %}
15847
15848 ins_encode %{
15849 Label done;
15850 FloatRegister s1 = as_FloatRegister($src1$$reg);
15851 Register d = as_Register($dst$$reg);
15852 __ fcmpd(s1, 0.0);
15853 // installs 0 if EQ else -1
15854 __ csinvw(d, zr, zr, Assembler::EQ);
15855 // keeps -1 if less or unordered else installs 1
15856 __ csnegw(d, d, d, Assembler::LT);
15857 __ bind(done);
15858 %}
15859 ins_pipe(pipe_class_default);
15860
15861 %}
15862
15863 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15864 %{
15865 match(Set dst (CmpLTMask p q));
15866 effect(KILL cr);
15867
15868 ins_cost(3 * INSN_COST);
15869
15870 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15871 "csetw $dst, lt\n\t"
15872 "subw $dst, zr, $dst"
15873 %}
15874
15875 ins_encode %{
15876 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15877 __ csetw(as_Register($dst$$reg), Assembler::LT);
15878 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15879 %}
15880
15881 ins_pipe(ialu_reg_reg);
15882 %}
15883
15884 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15885 %{
15886 match(Set dst (CmpLTMask src zero));
15887 effect(KILL cr);
15888
15889 ins_cost(INSN_COST);
15890
15891 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15892
15893 ins_encode %{
15894 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15895 %}
15896
15897 ins_pipe(ialu_reg_shift);
15898 %}
15899
15900 // ============================================================================
15901 // Max and Min
15902
15903 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15904
15905 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15906 %{
15907 effect(DEF cr, USE src);
15908 ins_cost(INSN_COST);
15909 format %{ "cmpw $src, 0" %}
15910
15911 ins_encode %{
15912 __ cmpw($src$$Register, 0);
15913 %}
15914 ins_pipe(icmp_reg_imm);
15915 %}
15916
15917 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15918 %{
15919 match(Set dst (MinI src1 src2));
15920 ins_cost(INSN_COST * 3);
15921
15922 expand %{
15923 rFlagsReg cr;
15924 compI_reg_reg(cr, src1, src2);
15925 cmovI_reg_reg_lt(dst, src1, src2, cr);
15926 %}
15927 %}
15928
15929 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15930 %{
15931 match(Set dst (MaxI src1 src2));
15932 ins_cost(INSN_COST * 3);
15933
15934 expand %{
15935 rFlagsReg cr;
15936 compI_reg_reg(cr, src1, src2);
15937 cmovI_reg_reg_gt(dst, src1, src2, cr);
15938 %}
15939 %}
15940
15941
15942 // ============================================================================
15943 // Branch Instructions
15944
15945 // Direct Branch.
15946 instruct branch(label lbl)
15947 %{
15948 match(Goto);
15949
15950 effect(USE lbl);
15951
15952 ins_cost(BRANCH_COST);
15953 format %{ "b $lbl" %}
15954
15955 ins_encode(aarch64_enc_b(lbl));
15956
15957 ins_pipe(pipe_branch);
15958 %}
15959
15960 // Conditional Near Branch
15961 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15962 %{
15963 // Same match rule as `branchConFar'.
15964 match(If cmp cr);
15965
15966 effect(USE lbl);
15967
15968 ins_cost(BRANCH_COST);
15969 // If set to 1 this indicates that the current instruction is a
15970 // short variant of a long branch. This avoids using this
15971 // instruction in first-pass matching. It will then only be used in
15972 // the `Shorten_branches' pass.
15973 // ins_short_branch(1);
15974 format %{ "b$cmp $lbl" %}
15975
15976 ins_encode(aarch64_enc_br_con(cmp, lbl));
15977
15978 ins_pipe(pipe_branch_cond);
15979 %}
15980
15981 // Conditional Near Branch Unsigned
15982 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15983 %{
15984 // Same match rule as `branchConFar'.
15985 match(If cmp cr);
15986
15987 effect(USE lbl);
15988
15989 ins_cost(BRANCH_COST);
15990 // If set to 1 this indicates that the current instruction is a
15991 // short variant of a long branch. This avoids using this
15992 // instruction in first-pass matching. It will then only be used in
15993 // the `Shorten_branches' pass.
15994 // ins_short_branch(1);
15995 format %{ "b$cmp $lbl\t# unsigned" %}
15996
15997 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15998
15999 ins_pipe(pipe_branch_cond);
16000 %}
16001
16002 // Make use of CBZ and CBNZ. These instructions, as well as being
16003 // shorter than (cmp; branch), have the additional benefit of not
16004 // killing the flags.
16005
16006 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16007 match(If cmp (CmpI op1 op2));
16008 effect(USE labl);
16009
16010 ins_cost(BRANCH_COST);
16011 format %{ "cbw$cmp $op1, $labl" %}
16012 ins_encode %{
16013 Label* L = $labl$$label;
16014 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16015 if (cond == Assembler::EQ)
16016 __ cbzw($op1$$Register, *L);
16017 else
16018 __ cbnzw($op1$$Register, *L);
16019 %}
16020 ins_pipe(pipe_cmp_branch);
16021 %}
16022
16023 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16024 match(If cmp (CmpL op1 op2));
16025 effect(USE labl);
16026
16027 ins_cost(BRANCH_COST);
16028 format %{ "cb$cmp $op1, $labl" %}
16029 ins_encode %{
16030 Label* L = $labl$$label;
16031 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16032 if (cond == Assembler::EQ)
16033 __ cbz($op1$$Register, *L);
16034 else
16035 __ cbnz($op1$$Register, *L);
16036 %}
16037 ins_pipe(pipe_cmp_branch);
16038 %}
16039
16040 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16041 match(If cmp (CmpP op1 op2));
16042 effect(USE labl);
16043
16044 ins_cost(BRANCH_COST);
16045 format %{ "cb$cmp $op1, $labl" %}
16046 ins_encode %{
16047 Label* L = $labl$$label;
16048 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16049 if (cond == Assembler::EQ)
16050 __ cbz($op1$$Register, *L);
16051 else
16052 __ cbnz($op1$$Register, *L);
16053 %}
16054 ins_pipe(pipe_cmp_branch);
16055 %}
16056
16057 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16058 match(If cmp (CmpN op1 op2));
16059 effect(USE labl);
16060
16061 ins_cost(BRANCH_COST);
16062 format %{ "cbw$cmp $op1, $labl" %}
16063 ins_encode %{
16064 Label* L = $labl$$label;
16065 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16066 if (cond == Assembler::EQ)
16067 __ cbzw($op1$$Register, *L);
16068 else
16069 __ cbnzw($op1$$Register, *L);
16070 %}
16071 ins_pipe(pipe_cmp_branch);
16072 %}
16073
16074 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16075 match(If cmp (CmpP (DecodeN oop) zero));
16076 effect(USE labl);
16077
16078 ins_cost(BRANCH_COST);
16079 format %{ "cb$cmp $oop, $labl" %}
16080 ins_encode %{
16081 Label* L = $labl$$label;
16082 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16083 if (cond == Assembler::EQ)
16084 __ cbzw($oop$$Register, *L);
16085 else
16086 __ cbnzw($oop$$Register, *L);
16087 %}
16088 ins_pipe(pipe_cmp_branch);
16089 %}
16090
16091 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16092 match(If cmp (CmpU op1 op2));
16093 effect(USE labl);
16094
16095 ins_cost(BRANCH_COST);
16096 format %{ "cbw$cmp $op1, $labl" %}
16097 ins_encode %{
16098 Label* L = $labl$$label;
16099 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16100 if (cond == Assembler::EQ || cond == Assembler::LS) {
16101 __ cbzw($op1$$Register, *L);
16102 } else {
16103 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16104 __ cbnzw($op1$$Register, *L);
16105 }
16106 %}
16107 ins_pipe(pipe_cmp_branch);
16108 %}
16109
16110 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
16111 match(If cmp (CmpUL op1 op2));
16112 effect(USE labl);
16113
16114 ins_cost(BRANCH_COST);
16115 format %{ "cb$cmp $op1, $labl" %}
16116 ins_encode %{
16117 Label* L = $labl$$label;
16118 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16119 if (cond == Assembler::EQ || cond == Assembler::LS) {
16120 __ cbz($op1$$Register, *L);
16121 } else {
16122 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16123 __ cbnz($op1$$Register, *L);
16124 }
16125 %}
16126 ins_pipe(pipe_cmp_branch);
16127 %}
16128
16129 // Test bit and Branch
16130
16131 // Patterns for short (< 32KiB) variants
16132 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16133 match(If cmp (CmpL op1 op2));
16134 effect(USE labl);
16135
16136 ins_cost(BRANCH_COST);
16137 format %{ "cb$cmp $op1, $labl # long" %}
16138 ins_encode %{
16139 Label* L = $labl$$label;
16140 Assembler::Condition cond =
16141 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16142 __ tbr(cond, $op1$$Register, 63, *L);
16143 %}
16144 ins_pipe(pipe_cmp_branch);
16145 ins_short_branch(1);
16146 %}
16147
16148 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16149 match(If cmp (CmpI op1 op2));
16150 effect(USE labl);
16151
16152 ins_cost(BRANCH_COST);
16153 format %{ "cb$cmp $op1, $labl # int" %}
16154 ins_encode %{
16155 Label* L = $labl$$label;
16156 Assembler::Condition cond =
16157 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16158 __ tbr(cond, $op1$$Register, 31, *L);
16159 %}
16160 ins_pipe(pipe_cmp_branch);
16161 ins_short_branch(1);
16162 %}
16163
16164 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16165 match(If cmp (CmpL (AndL op1 op2) op3));
16166 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16167 effect(USE labl);
16168
16169 ins_cost(BRANCH_COST);
16170 format %{ "tb$cmp $op1, $op2, $labl" %}
16171 ins_encode %{
16172 Label* L = $labl$$label;
16173 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16174 int bit = exact_log2_long($op2$$constant);
16175 __ tbr(cond, $op1$$Register, bit, *L);
16176 %}
16177 ins_pipe(pipe_cmp_branch);
16178 ins_short_branch(1);
16179 %}
16180
16181 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16182 match(If cmp (CmpI (AndI op1 op2) op3));
16183 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16184 effect(USE labl);
16185
16186 ins_cost(BRANCH_COST);
16187 format %{ "tb$cmp $op1, $op2, $labl" %}
16188 ins_encode %{
16189 Label* L = $labl$$label;
16190 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16191 int bit = exact_log2((juint)$op2$$constant);
16192 __ tbr(cond, $op1$$Register, bit, *L);
16193 %}
16194 ins_pipe(pipe_cmp_branch);
16195 ins_short_branch(1);
16196 %}
16197
16198 // And far variants
16199 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16200 match(If cmp (CmpL op1 op2));
16201 effect(USE labl);
16202
16203 ins_cost(BRANCH_COST);
16204 format %{ "cb$cmp $op1, $labl # long" %}
16205 ins_encode %{
16206 Label* L = $labl$$label;
16207 Assembler::Condition cond =
16208 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16209 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16210 %}
16211 ins_pipe(pipe_cmp_branch);
16212 %}
16213
16214 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16215 match(If cmp (CmpI op1 op2));
16216 effect(USE labl);
16217
16218 ins_cost(BRANCH_COST);
16219 format %{ "cb$cmp $op1, $labl # int" %}
16220 ins_encode %{
16221 Label* L = $labl$$label;
16222 Assembler::Condition cond =
16223 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16224 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16225 %}
16226 ins_pipe(pipe_cmp_branch);
16227 %}
16228
16229 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16230 match(If cmp (CmpL (AndL op1 op2) op3));
16231 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16232 effect(USE labl);
16233
16234 ins_cost(BRANCH_COST);
16235 format %{ "tb$cmp $op1, $op2, $labl" %}
16236 ins_encode %{
16237 Label* L = $labl$$label;
16238 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16239 int bit = exact_log2_long($op2$$constant);
16240 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16241 %}
16242 ins_pipe(pipe_cmp_branch);
16243 %}
16244
16245 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16246 match(If cmp (CmpI (AndI op1 op2) op3));
16247 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16248 effect(USE labl);
16249
16250 ins_cost(BRANCH_COST);
16251 format %{ "tb$cmp $op1, $op2, $labl" %}
16252 ins_encode %{
16253 Label* L = $labl$$label;
16254 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16255 int bit = exact_log2((juint)$op2$$constant);
16256 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16257 %}
16258 ins_pipe(pipe_cmp_branch);
16259 %}
16260
16261 // Test bits
16262
16263 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16264 match(Set cr (CmpL (AndL op1 op2) op3));
16265 predicate(Assembler::operand_valid_for_logical_immediate
16266 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16267
16268 ins_cost(INSN_COST);
16269 format %{ "tst $op1, $op2 # long" %}
16270 ins_encode %{
16271 __ tst($op1$$Register, $op2$$constant);
16272 %}
16273 ins_pipe(ialu_reg_reg);
16274 %}
16275
16276 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16277 match(Set cr (CmpI (AndI op1 op2) op3));
16278 predicate(Assembler::operand_valid_for_logical_immediate
16279 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16280
16281 ins_cost(INSN_COST);
16282 format %{ "tst $op1, $op2 # int" %}
16283 ins_encode %{
16284 __ tstw($op1$$Register, $op2$$constant);
16285 %}
16286 ins_pipe(ialu_reg_reg);
16287 %}
16288
16289 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16290 match(Set cr (CmpL (AndL op1 op2) op3));
16291
16292 ins_cost(INSN_COST);
16293 format %{ "tst $op1, $op2 # long" %}
16294 ins_encode %{
16295 __ tst($op1$$Register, $op2$$Register);
16296 %}
16297 ins_pipe(ialu_reg_reg);
16298 %}
16299
16300 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16301 match(Set cr (CmpI (AndI op1 op2) op3));
16302
16303 ins_cost(INSN_COST);
16304 format %{ "tstw $op1, $op2 # int" %}
16305 ins_encode %{
16306 __ tstw($op1$$Register, $op2$$Register);
16307 %}
16308 ins_pipe(ialu_reg_reg);
16309 %}
16310
16311
16312 // Conditional Far Branch
16313 // Conditional Far Branch Unsigned
16314 // TODO: fixme
16315
16316 // counted loop end branch near
16317 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16318 %{
16319 match(CountedLoopEnd cmp cr);
16320
16321 effect(USE lbl);
16322
16323 ins_cost(BRANCH_COST);
16324 // short variant.
16325 // ins_short_branch(1);
16326 format %{ "b$cmp $lbl \t// counted loop end" %}
16327
16328 ins_encode(aarch64_enc_br_con(cmp, lbl));
16329
16330 ins_pipe(pipe_branch);
16331 %}
16332
16333 // counted loop end branch far
16334 // TODO: fixme
16335
16336 // ============================================================================
16337 // inlined locking and unlocking
16338
16339 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16340 %{
16341 match(Set cr (FastLock object box));
16342 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16343
16344 ins_cost(5 * INSN_COST);
16345 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16346
16347 ins_encode %{
16348 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16349 %}
16350
16351 ins_pipe(pipe_serial);
16352 %}
16353
16354 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16355 %{
16356 match(Set cr (FastUnlock object box));
16357 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16358
16359 ins_cost(5 * INSN_COST);
16360 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16361
16362 ins_encode %{
16363 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16364 %}
16365
16366 ins_pipe(pipe_serial);
16367 %}
16368
16369 // ============================================================================
16370 // Safepoint Instructions
16371
16372 // TODO
16373 // provide a near and far version of this code
16374
16375 instruct safePoint(rFlagsReg cr, iRegP poll)
16376 %{
16377 match(SafePoint poll);
16378 effect(KILL cr);
16379
16380 format %{
16381 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16382 %}
16383 ins_encode %{
16384 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16385 %}
16386 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16387 %}
16388
16389
16390 // ============================================================================
16391 // Procedure Call/Return Instructions
16392
16393 // Call Java Static Instruction
16394
16395 instruct CallStaticJavaDirect(method meth)
16396 %{
16397 match(CallStaticJava);
16398
16399 effect(USE meth);
16400
16401 ins_cost(CALL_COST);
16402
16403 format %{ "call,static $meth \t// ==> " %}
16404
16405 ins_encode(aarch64_enc_java_static_call(meth),
16406 aarch64_enc_call_epilog);
16407
16408 ins_pipe(pipe_class_call);
16409 %}
16410
16411 // TO HERE
16412
16413 // Call Java Dynamic Instruction
16414 instruct CallDynamicJavaDirect(method meth)
16415 %{
16416 match(CallDynamicJava);
16417
16418 effect(USE meth);
16419
16420 ins_cost(CALL_COST);
16421
16422 format %{ "CALL,dynamic $meth \t// ==> " %}
16423
16424 ins_encode(aarch64_enc_java_dynamic_call(meth),
16425 aarch64_enc_call_epilog);
16426
16427 ins_pipe(pipe_class_call);
16428 %}
16429
16430 // Call Runtime Instruction
16431
16432 instruct CallRuntimeDirect(method meth)
16433 %{
16434 match(CallRuntime);
16435
16436 effect(USE meth);
16437
16438 ins_cost(CALL_COST);
16439
16440 format %{ "CALL, runtime $meth" %}
16441
16442 ins_encode( aarch64_enc_java_to_runtime(meth) );
16443
16444 ins_pipe(pipe_class_call);
16445 %}
16446
16447 // Call Runtime Instruction
16448
16449 instruct CallLeafDirect(method meth)
16450 %{
16451 match(CallLeaf);
16452
16453 effect(USE meth);
16454
16455 ins_cost(CALL_COST);
16456
16457 format %{ "CALL, runtime leaf $meth" %}
16458
16459 ins_encode( aarch64_enc_java_to_runtime(meth) );
16460
16461 ins_pipe(pipe_class_call);
16462 %}
16463
16464 // Call Runtime Instruction without safepoint and with vector arguments
16465 instruct CallLeafDirectVector(method meth)
16466 %{
16467 match(CallLeafVector);
16468
16469 effect(USE meth);
16470
16471 ins_cost(CALL_COST);
16472
16473 format %{ "CALL, runtime leaf vector $meth" %}
16474
16475 ins_encode(aarch64_enc_java_to_runtime(meth));
16476
16477 ins_pipe(pipe_class_call);
16478 %}
16479
16480 // Call Runtime Instruction
16481
16482 instruct CallLeafNoFPDirect(method meth)
16483 %{
16484 match(CallLeafNoFP);
16485
16486 effect(USE meth);
16487
16488 ins_cost(CALL_COST);
16489
16490 format %{ "CALL, runtime leaf nofp $meth" %}
16491
16492 ins_encode( aarch64_enc_java_to_runtime(meth) );
16493
16494 ins_pipe(pipe_class_call);
16495 %}
16496
16497 // Tail Call; Jump from runtime stub to Java code.
16498 // Also known as an 'interprocedural jump'.
16499 // Target of jump will eventually return to caller.
16500 // TailJump below removes the return address.
16501 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16502 // emitted just above the TailCall which has reset rfp to the caller state.
16503 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16504 %{
16505 match(TailCall jump_target method_ptr);
16506
16507 ins_cost(CALL_COST);
16508
16509 format %{ "br $jump_target\t# $method_ptr holds method" %}
16510
16511 ins_encode(aarch64_enc_tail_call(jump_target));
16512
16513 ins_pipe(pipe_class_call);
16514 %}
16515
16516 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16517 %{
16518 match(TailJump jump_target ex_oop);
16519
16520 ins_cost(CALL_COST);
16521
16522 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16523
16524 ins_encode(aarch64_enc_tail_jmp(jump_target));
16525
16526 ins_pipe(pipe_class_call);
16527 %}
16528
16529 // Forward exception.
16530 instruct ForwardExceptionjmp()
16531 %{
16532 match(ForwardException);
16533 ins_cost(CALL_COST);
16534
16535 format %{ "b forward_exception_stub" %}
16536 ins_encode %{
16537 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16538 %}
16539 ins_pipe(pipe_class_call);
16540 %}
16541
16542 // Create exception oop: created by stack-crawling runtime code.
16543 // Created exception is now available to this handler, and is setup
16544 // just prior to jumping to this handler. No code emitted.
16545 // TODO check
16546 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16547 instruct CreateException(iRegP_R0 ex_oop)
16548 %{
16549 match(Set ex_oop (CreateEx));
16550
16551 format %{ " -- \t// exception oop; no code emitted" %}
16552
16553 size(0);
16554
16555 ins_encode( /*empty*/ );
16556
16557 ins_pipe(pipe_class_empty);
16558 %}
16559
16560 // Rethrow exception: The exception oop will come in the first
16561 // argument position. Then JUMP (not call) to the rethrow stub code.
16562 instruct RethrowException() %{
16563 match(Rethrow);
16564 ins_cost(CALL_COST);
16565
16566 format %{ "b rethrow_stub" %}
16567
16568 ins_encode( aarch64_enc_rethrow() );
16569
16570 ins_pipe(pipe_class_call);
16571 %}
16572
16573
16574 // Return Instruction
16575 // epilog node loads ret address into lr as part of frame pop
16576 instruct Ret()
16577 %{
16578 match(Return);
16579
16580 format %{ "ret\t// return register" %}
16581
16582 ins_encode( aarch64_enc_ret() );
16583
16584 ins_pipe(pipe_branch);
16585 %}
16586
16587 // Die now.
16588 instruct ShouldNotReachHere() %{
16589 match(Halt);
16590
16591 ins_cost(CALL_COST);
16592 format %{ "ShouldNotReachHere" %}
16593
16594 ins_encode %{
16595 if (is_reachable()) {
16596 const char* str = __ code_string(_halt_reason);
16597 __ stop(str);
16598 }
16599 %}
16600
16601 ins_pipe(pipe_class_default);
16602 %}
16603
16604 // ============================================================================
16605 // Partial Subtype Check
16606 //
16607 // superklass array for an instance of the superklass. Set a hidden
16608 // internal cache on a hit (cache is checked with exposed code in
16609 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16610 // encoding ALSO sets flags.
16611
16612 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16613 %{
16614 match(Set result (PartialSubtypeCheck sub super));
16615 predicate(!UseSecondarySupersTable);
16616 effect(KILL cr, KILL temp);
16617
16618 ins_cost(20 * INSN_COST); // slightly larger than the next version
16619 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16620
16621 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16622
16623 opcode(0x1); // Force zero of result reg on hit
16624
16625 ins_pipe(pipe_class_memory);
16626 %}
16627
16628 // Two versions of partialSubtypeCheck, both used when we need to
16629 // search for a super class in the secondary supers array. The first
16630 // is used when we don't know _a priori_ the class being searched
16631 // for. The second, far more common, is used when we do know: this is
16632 // used for instanceof, checkcast, and any case where C2 can determine
16633 // it by constant propagation.
16634
16635 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16636 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16637 rFlagsReg cr)
16638 %{
16639 match(Set result (PartialSubtypeCheck sub super));
16640 predicate(UseSecondarySupersTable);
16641 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16642
16643 ins_cost(10 * INSN_COST); // slightly larger than the next version
16644 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16645
16646 ins_encode %{
16647 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16648 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16649 $vtemp$$FloatRegister,
16650 $result$$Register, /*L_success*/nullptr);
16651 %}
16652
16653 ins_pipe(pipe_class_memory);
16654 %}
16655
16656 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16657 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16658 rFlagsReg cr)
16659 %{
16660 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16661 predicate(UseSecondarySupersTable);
16662 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16663
16664 ins_cost(5 * INSN_COST); // smaller than the next version
16665 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16666
16667 ins_encode %{
16668 bool success = false;
16669 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16670 if (InlineSecondarySupersTest) {
16671 success =
16672 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16673 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16674 $vtemp$$FloatRegister,
16675 $result$$Register,
16676 super_klass_slot);
16677 } else {
16678 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16679 success = (call != nullptr);
16680 }
16681 if (!success) {
16682 ciEnv::current()->record_failure("CodeCache is full");
16683 return;
16684 }
16685 %}
16686
16687 ins_pipe(pipe_class_memory);
16688 %}
16689
16690 // Intrisics for String.compareTo()
16691
16692 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16693 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16694 %{
16695 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16696 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16697 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16698
16699 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16700 ins_encode %{
16701 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16702 __ string_compare($str1$$Register, $str2$$Register,
16703 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16704 $tmp1$$Register, $tmp2$$Register,
16705 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16706 %}
16707 ins_pipe(pipe_class_memory);
16708 %}
16709
16710 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16711 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16712 %{
16713 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16714 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16715 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16716
16717 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16718 ins_encode %{
16719 __ string_compare($str1$$Register, $str2$$Register,
16720 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16721 $tmp1$$Register, $tmp2$$Register,
16722 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16723 %}
16724 ins_pipe(pipe_class_memory);
16725 %}
16726
16727 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16728 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16729 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16730 %{
16731 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16732 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16733 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16734 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16735
16736 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16737 ins_encode %{
16738 __ string_compare($str1$$Register, $str2$$Register,
16739 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16740 $tmp1$$Register, $tmp2$$Register,
16741 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16742 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16743 %}
16744 ins_pipe(pipe_class_memory);
16745 %}
16746
16747 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16748 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16749 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16750 %{
16751 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16752 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16753 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16754 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16755
16756 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16757 ins_encode %{
16758 __ string_compare($str1$$Register, $str2$$Register,
16759 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16760 $tmp1$$Register, $tmp2$$Register,
16761 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16762 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16763 %}
16764 ins_pipe(pipe_class_memory);
16765 %}
16766
16767 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16768 // these string_compare variants as NEON register type for convenience so that the prototype of
16769 // string_compare can be shared with all variants.
16770
16771 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16772 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16773 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16774 pRegGov_P1 pgtmp2, rFlagsReg cr)
16775 %{
16776 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16777 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16778 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16779 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16780
16781 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16782 ins_encode %{
16783 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16784 __ string_compare($str1$$Register, $str2$$Register,
16785 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16786 $tmp1$$Register, $tmp2$$Register,
16787 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16788 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16789 StrIntrinsicNode::LL);
16790 %}
16791 ins_pipe(pipe_class_memory);
16792 %}
16793
16794 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16795 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16796 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16797 pRegGov_P1 pgtmp2, rFlagsReg cr)
16798 %{
16799 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16800 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16801 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16802 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16803
16804 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16805 ins_encode %{
16806 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16807 __ string_compare($str1$$Register, $str2$$Register,
16808 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16809 $tmp1$$Register, $tmp2$$Register,
16810 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16811 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16812 StrIntrinsicNode::LU);
16813 %}
16814 ins_pipe(pipe_class_memory);
16815 %}
16816
16817 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16818 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16819 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16820 pRegGov_P1 pgtmp2, rFlagsReg cr)
16821 %{
16822 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16823 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16824 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16825 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16826
16827 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16828 ins_encode %{
16829 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16830 __ string_compare($str1$$Register, $str2$$Register,
16831 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16832 $tmp1$$Register, $tmp2$$Register,
16833 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16834 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16835 StrIntrinsicNode::UL);
16836 %}
16837 ins_pipe(pipe_class_memory);
16838 %}
16839
16840 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16841 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16842 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16843 pRegGov_P1 pgtmp2, rFlagsReg cr)
16844 %{
16845 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16846 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16847 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16848 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16849
16850 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16851 ins_encode %{
16852 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16853 __ string_compare($str1$$Register, $str2$$Register,
16854 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16855 $tmp1$$Register, $tmp2$$Register,
16856 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16857 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16858 StrIntrinsicNode::UU);
16859 %}
16860 ins_pipe(pipe_class_memory);
16861 %}
16862
16863 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16864 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16865 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16866 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16867 %{
16868 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16869 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16870 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16871 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16872 TEMP vtmp0, TEMP vtmp1, KILL cr);
16873 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16874 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16875
16876 ins_encode %{
16877 __ string_indexof($str1$$Register, $str2$$Register,
16878 $cnt1$$Register, $cnt2$$Register,
16879 $tmp1$$Register, $tmp2$$Register,
16880 $tmp3$$Register, $tmp4$$Register,
16881 $tmp5$$Register, $tmp6$$Register,
16882 -1, $result$$Register, StrIntrinsicNode::UU);
16883 %}
16884 ins_pipe(pipe_class_memory);
16885 %}
16886
16887 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16888 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16889 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16890 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16891 %{
16892 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16893 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16894 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16895 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16896 TEMP vtmp0, TEMP vtmp1, KILL cr);
16897 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16898 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16899
16900 ins_encode %{
16901 __ string_indexof($str1$$Register, $str2$$Register,
16902 $cnt1$$Register, $cnt2$$Register,
16903 $tmp1$$Register, $tmp2$$Register,
16904 $tmp3$$Register, $tmp4$$Register,
16905 $tmp5$$Register, $tmp6$$Register,
16906 -1, $result$$Register, StrIntrinsicNode::LL);
16907 %}
16908 ins_pipe(pipe_class_memory);
16909 %}
16910
16911 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16912 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16913 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16914 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16915 %{
16916 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16917 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16918 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16919 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16920 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16921 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16922 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16923
16924 ins_encode %{
16925 __ string_indexof($str1$$Register, $str2$$Register,
16926 $cnt1$$Register, $cnt2$$Register,
16927 $tmp1$$Register, $tmp2$$Register,
16928 $tmp3$$Register, $tmp4$$Register,
16929 $tmp5$$Register, $tmp6$$Register,
16930 -1, $result$$Register, StrIntrinsicNode::UL);
16931 %}
16932 ins_pipe(pipe_class_memory);
16933 %}
16934
16935 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16936 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16937 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16938 %{
16939 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16940 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16941 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16942 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16943 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16944 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16945
16946 ins_encode %{
16947 int icnt2 = (int)$int_cnt2$$constant;
16948 __ string_indexof($str1$$Register, $str2$$Register,
16949 $cnt1$$Register, zr,
16950 $tmp1$$Register, $tmp2$$Register,
16951 $tmp3$$Register, $tmp4$$Register, zr, zr,
16952 icnt2, $result$$Register, StrIntrinsicNode::UU);
16953 %}
16954 ins_pipe(pipe_class_memory);
16955 %}
16956
16957 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16958 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16959 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16960 %{
16961 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16962 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16963 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16964 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16965 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16966 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16967
16968 ins_encode %{
16969 int icnt2 = (int)$int_cnt2$$constant;
16970 __ string_indexof($str1$$Register, $str2$$Register,
16971 $cnt1$$Register, zr,
16972 $tmp1$$Register, $tmp2$$Register,
16973 $tmp3$$Register, $tmp4$$Register, zr, zr,
16974 icnt2, $result$$Register, StrIntrinsicNode::LL);
16975 %}
16976 ins_pipe(pipe_class_memory);
16977 %}
16978
16979 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16980 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16981 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16982 %{
16983 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16984 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16985 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16986 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16987 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16988 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16989
16990 ins_encode %{
16991 int icnt2 = (int)$int_cnt2$$constant;
16992 __ string_indexof($str1$$Register, $str2$$Register,
16993 $cnt1$$Register, zr,
16994 $tmp1$$Register, $tmp2$$Register,
16995 $tmp3$$Register, $tmp4$$Register, zr, zr,
16996 icnt2, $result$$Register, StrIntrinsicNode::UL);
16997 %}
16998 ins_pipe(pipe_class_memory);
16999 %}
17000
17001 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17002 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17003 iRegINoSp tmp3, rFlagsReg cr)
17004 %{
17005 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17006 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17007 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17008 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17009
17010 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17011
17012 ins_encode %{
17013 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17014 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17015 $tmp3$$Register);
17016 %}
17017 ins_pipe(pipe_class_memory);
17018 %}
17019
17020 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17021 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17022 iRegINoSp tmp3, rFlagsReg cr)
17023 %{
17024 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17025 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17026 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17027 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17028
17029 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17030
17031 ins_encode %{
17032 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17033 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17034 $tmp3$$Register);
17035 %}
17036 ins_pipe(pipe_class_memory);
17037 %}
17038
17039 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17040 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17041 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17042 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17043 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17044 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17045 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17046 ins_encode %{
17047 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17048 $result$$Register, $ztmp1$$FloatRegister,
17049 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17050 $ptmp$$PRegister, true /* isL */);
17051 %}
17052 ins_pipe(pipe_class_memory);
17053 %}
17054
17055 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17056 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17057 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17058 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17059 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17060 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17061 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17062 ins_encode %{
17063 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17064 $result$$Register, $ztmp1$$FloatRegister,
17065 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17066 $ptmp$$PRegister, false /* isL */);
17067 %}
17068 ins_pipe(pipe_class_memory);
17069 %}
17070
17071 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17072 iRegI_R0 result, rFlagsReg cr)
17073 %{
17074 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17075 match(Set result (StrEquals (Binary str1 str2) cnt));
17076 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17077
17078 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17079 ins_encode %{
17080 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17081 __ string_equals($str1$$Register, $str2$$Register,
17082 $result$$Register, $cnt$$Register);
17083 %}
17084 ins_pipe(pipe_class_memory);
17085 %}
17086
17087 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17088 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17089 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17090 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17091 iRegP_R10 tmp, rFlagsReg cr)
17092 %{
17093 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17094 match(Set result (AryEq ary1 ary2));
17095 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17096 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17097 TEMP vtmp6, TEMP vtmp7, KILL cr);
17098
17099 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17100 ins_encode %{
17101 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17102 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17103 $result$$Register, $tmp$$Register, 1);
17104 if (tpc == nullptr) {
17105 ciEnv::current()->record_failure("CodeCache is full");
17106 return;
17107 }
17108 %}
17109 ins_pipe(pipe_class_memory);
17110 %}
17111
17112 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17113 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17114 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17115 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17116 iRegP_R10 tmp, rFlagsReg cr)
17117 %{
17118 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17119 match(Set result (AryEq ary1 ary2));
17120 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17121 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17122 TEMP vtmp6, TEMP vtmp7, KILL cr);
17123
17124 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17125 ins_encode %{
17126 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17127 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17128 $result$$Register, $tmp$$Register, 2);
17129 if (tpc == nullptr) {
17130 ciEnv::current()->record_failure("CodeCache is full");
17131 return;
17132 }
17133 %}
17134 ins_pipe(pipe_class_memory);
17135 %}
17136
17137 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17138 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17139 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17140 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17141 %{
17142 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17143 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17144 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17145
17146 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
17147 ins_encode %{
17148 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17149 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17150 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17151 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17152 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17153 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17154 (BasicType)$basic_type$$constant);
17155 if (tpc == nullptr) {
17156 ciEnv::current()->record_failure("CodeCache is full");
17157 return;
17158 }
17159 %}
17160 ins_pipe(pipe_class_memory);
17161 %}
17162
17163 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17164 %{
17165 match(Set result (CountPositives ary1 len));
17166 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17167 format %{ "count positives byte[] $ary1,$len -> $result" %}
17168 ins_encode %{
17169 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17170 if (tpc == nullptr) {
17171 ciEnv::current()->record_failure("CodeCache is full");
17172 return;
17173 }
17174 %}
17175 ins_pipe( pipe_slow );
17176 %}
17177
17178 // fast char[] to byte[] compression
17179 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17180 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17181 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17182 iRegI_R0 result, rFlagsReg cr)
17183 %{
17184 match(Set result (StrCompressedCopy src (Binary dst len)));
17185 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17186 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17187
17188 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17189 ins_encode %{
17190 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17191 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17192 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17193 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17194 %}
17195 ins_pipe(pipe_slow);
17196 %}
17197
17198 // fast byte[] to char[] inflation
17199 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17200 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17201 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17202 %{
17203 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17204 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17205 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17206 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17207
17208 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17209 ins_encode %{
17210 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17211 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17212 $vtmp2$$FloatRegister, $tmp$$Register);
17213 if (tpc == nullptr) {
17214 ciEnv::current()->record_failure("CodeCache is full");
17215 return;
17216 }
17217 %}
17218 ins_pipe(pipe_class_memory);
17219 %}
17220
17221 // encode char[] to byte[] in ISO_8859_1
17222 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17223 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17224 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17225 iRegI_R0 result, rFlagsReg cr)
17226 %{
17227 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17228 match(Set result (EncodeISOArray src (Binary dst len)));
17229 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17230 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17231
17232 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17233 ins_encode %{
17234 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17235 $result$$Register, false,
17236 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17237 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17238 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17239 %}
17240 ins_pipe(pipe_class_memory);
17241 %}
17242
17243 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17244 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17245 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17246 iRegI_R0 result, rFlagsReg cr)
17247 %{
17248 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17249 match(Set result (EncodeISOArray src (Binary dst len)));
17250 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17251 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17252
17253 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17254 ins_encode %{
17255 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17256 $result$$Register, true,
17257 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17258 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17259 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17260 %}
17261 ins_pipe(pipe_class_memory);
17262 %}
17263
17264 //----------------------------- CompressBits/ExpandBits ------------------------
17265
17266 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17267 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17268 match(Set dst (CompressBits src mask));
17269 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17270 format %{ "mov $tsrc, $src\n\t"
17271 "mov $tmask, $mask\n\t"
17272 "bext $tdst, $tsrc, $tmask\n\t"
17273 "mov $dst, $tdst"
17274 %}
17275 ins_encode %{
17276 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17277 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17278 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17279 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17280 %}
17281 ins_pipe(pipe_slow);
17282 %}
17283
17284 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17285 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17286 match(Set dst (CompressBits (LoadI mem) mask));
17287 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17288 format %{ "ldrs $tsrc, $mem\n\t"
17289 "ldrs $tmask, $mask\n\t"
17290 "bext $tdst, $tsrc, $tmask\n\t"
17291 "mov $dst, $tdst"
17292 %}
17293 ins_encode %{
17294 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17295 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17296 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17297 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17298 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17299 %}
17300 ins_pipe(pipe_slow);
17301 %}
17302
17303 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17304 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17305 match(Set dst (CompressBits src mask));
17306 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17307 format %{ "mov $tsrc, $src\n\t"
17308 "mov $tmask, $mask\n\t"
17309 "bext $tdst, $tsrc, $tmask\n\t"
17310 "mov $dst, $tdst"
17311 %}
17312 ins_encode %{
17313 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17314 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17315 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17316 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17317 %}
17318 ins_pipe(pipe_slow);
17319 %}
17320
17321 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17322 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17323 match(Set dst (CompressBits (LoadL mem) mask));
17324 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17325 format %{ "ldrd $tsrc, $mem\n\t"
17326 "ldrd $tmask, $mask\n\t"
17327 "bext $tdst, $tsrc, $tmask\n\t"
17328 "mov $dst, $tdst"
17329 %}
17330 ins_encode %{
17331 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17332 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17333 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17334 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17335 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17336 %}
17337 ins_pipe(pipe_slow);
17338 %}
17339
17340 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17341 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17342 match(Set dst (ExpandBits src mask));
17343 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17344 format %{ "mov $tsrc, $src\n\t"
17345 "mov $tmask, $mask\n\t"
17346 "bdep $tdst, $tsrc, $tmask\n\t"
17347 "mov $dst, $tdst"
17348 %}
17349 ins_encode %{
17350 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17351 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17352 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17353 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17354 %}
17355 ins_pipe(pipe_slow);
17356 %}
17357
17358 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17359 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17360 match(Set dst (ExpandBits (LoadI mem) mask));
17361 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17362 format %{ "ldrs $tsrc, $mem\n\t"
17363 "ldrs $tmask, $mask\n\t"
17364 "bdep $tdst, $tsrc, $tmask\n\t"
17365 "mov $dst, $tdst"
17366 %}
17367 ins_encode %{
17368 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17369 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17370 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17371 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17372 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17373 %}
17374 ins_pipe(pipe_slow);
17375 %}
17376
17377 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17378 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17379 match(Set dst (ExpandBits src mask));
17380 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17381 format %{ "mov $tsrc, $src\n\t"
17382 "mov $tmask, $mask\n\t"
17383 "bdep $tdst, $tsrc, $tmask\n\t"
17384 "mov $dst, $tdst"
17385 %}
17386 ins_encode %{
17387 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17388 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17389 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17390 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17391 %}
17392 ins_pipe(pipe_slow);
17393 %}
17394
17395
17396 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17397 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17398 match(Set dst (ExpandBits (LoadL mem) mask));
17399 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17400 format %{ "ldrd $tsrc, $mem\n\t"
17401 "ldrd $tmask, $mask\n\t"
17402 "bdep $tdst, $tsrc, $tmask\n\t"
17403 "mov $dst, $tdst"
17404 %}
17405 ins_encode %{
17406 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17407 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17408 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17409 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17410 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17411 %}
17412 ins_pipe(pipe_slow);
17413 %}
17414
17415 //----------------------------- Reinterpret ----------------------------------
17416 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17417 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17418 match(Set dst (ReinterpretHF2S src));
17419 format %{ "reinterpretHF2S $dst, $src" %}
17420 ins_encode %{
17421 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17422 %}
17423 ins_pipe(pipe_slow);
17424 %}
17425
17426 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17427 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17428 match(Set dst (ReinterpretS2HF src));
17429 format %{ "reinterpretS2HF $dst, $src" %}
17430 ins_encode %{
17431 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17432 %}
17433 ins_pipe(pipe_slow);
17434 %}
17435
17436 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17437 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17438 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
17439 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
17440 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
17441 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17442 // can be omitted in this pattern, resulting in -
17443 // fcvt $dst, $src // Convert float to half-precision float
17444 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17445 %{
17446 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17447 format %{ "convF2HFAndS2HF $dst, $src" %}
17448 ins_encode %{
17449 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17450 %}
17451 ins_pipe(pipe_slow);
17452 %}
17453
17454 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17455 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17456 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
17457 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17458 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
17459 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17460 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17461 // resulting in -
17462 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
17463 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17464 %{
17465 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17466 format %{ "convHF2SAndHF2F $dst, $src" %}
17467 ins_encode %{
17468 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17469 %}
17470 ins_pipe(pipe_slow);
17471 %}
17472
17473 // ============================================================================
17474 // This name is KNOWN by the ADLC and cannot be changed.
17475 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17476 // for this guy.
17477 instruct tlsLoadP(thread_RegP dst)
17478 %{
17479 match(Set dst (ThreadLocal));
17480
17481 ins_cost(0);
17482
17483 format %{ " -- \t// $dst=Thread::current(), empty" %}
17484
17485 size(0);
17486
17487 ins_encode( /*empty*/ );
17488
17489 ins_pipe(pipe_class_empty);
17490 %}
17491
17492 //----------PEEPHOLE RULES-----------------------------------------------------
17493 // These must follow all instruction definitions as they use the names
17494 // defined in the instructions definitions.
17495 //
17496 // peepmatch ( root_instr_name [preceding_instruction]* );
17497 //
17498 // peepconstraint %{
17499 // (instruction_number.operand_name relational_op instruction_number.operand_name
17500 // [, ...] );
17501 // // instruction numbers are zero-based using left to right order in peepmatch
17502 //
17503 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17504 // // provide an instruction_number.operand_name for each operand that appears
17505 // // in the replacement instruction's match rule
17506 //
17507 // ---------VM FLAGS---------------------------------------------------------
17508 //
17509 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17510 //
17511 // Each peephole rule is given an identifying number starting with zero and
17512 // increasing by one in the order seen by the parser. An individual peephole
17513 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17514 // on the command-line.
17515 //
17516 // ---------CURRENT LIMITATIONS----------------------------------------------
17517 //
17518 // Only match adjacent instructions in same basic block
17519 // Only equality constraints
17520 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17521 // Only one replacement instruction
17522 //
17523 // ---------EXAMPLE----------------------------------------------------------
17524 //
17525 // // pertinent parts of existing instructions in architecture description
17526 // instruct movI(iRegINoSp dst, iRegI src)
17527 // %{
17528 // match(Set dst (CopyI src));
17529 // %}
17530 //
17531 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17532 // %{
17533 // match(Set dst (AddI dst src));
17534 // effect(KILL cr);
17535 // %}
17536 //
17537 // // Change (inc mov) to lea
17538 // peephole %{
17539 // // increment preceded by register-register move
17540 // peepmatch ( incI_iReg movI );
17541 // // require that the destination register of the increment
17542 // // match the destination register of the move
17543 // peepconstraint ( 0.dst == 1.dst );
17544 // // construct a replacement instruction that sets
17545 // // the destination to ( move's source register + one )
17546 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17547 // %}
17548 //
17549
17550 // Implementation no longer uses movX instructions since
17551 // machine-independent system no longer uses CopyX nodes.
17552 //
17553 // peephole
17554 // %{
17555 // peepmatch (incI_iReg movI);
17556 // peepconstraint (0.dst == 1.dst);
17557 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17558 // %}
17559
17560 // peephole
17561 // %{
17562 // peepmatch (decI_iReg movI);
17563 // peepconstraint (0.dst == 1.dst);
17564 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17565 // %}
17566
17567 // peephole
17568 // %{
17569 // peepmatch (addI_iReg_imm movI);
17570 // peepconstraint (0.dst == 1.dst);
17571 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17572 // %}
17573
17574 // peephole
17575 // %{
17576 // peepmatch (incL_iReg movL);
17577 // peepconstraint (0.dst == 1.dst);
17578 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17579 // %}
17580
17581 // peephole
17582 // %{
17583 // peepmatch (decL_iReg movL);
17584 // peepconstraint (0.dst == 1.dst);
17585 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17586 // %}
17587
17588 // peephole
17589 // %{
17590 // peepmatch (addL_iReg_imm movL);
17591 // peepconstraint (0.dst == 1.dst);
17592 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17593 // %}
17594
17595 // peephole
17596 // %{
17597 // peepmatch (addP_iReg_imm movP);
17598 // peepconstraint (0.dst == 1.dst);
17599 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17600 // %}
17601
17602 // // Change load of spilled value to only a spill
17603 // instruct storeI(memory mem, iRegI src)
17604 // %{
17605 // match(Set mem (StoreI mem src));
17606 // %}
17607 //
17608 // instruct loadI(iRegINoSp dst, memory mem)
17609 // %{
17610 // match(Set dst (LoadI mem));
17611 // %}
17612 //
17613
17614 //----------SMARTSPILL RULES---------------------------------------------------
17615 // These must follow all instruction definitions as they use the names
17616 // defined in the instructions definitions.
17617
17618 // Local Variables:
17619 // mode: c++
17620 // End: