1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return 0; // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1652 // lea(rscratch1, RuntimeAddress(addr)
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else if (_entry_point == nullptr) {
1658 // See CallLeafNoFPIndirect
1659 return 1 * NativeInstruction::instruction_size;
1660 } else {
1661 return 6 * NativeInstruction::instruction_size;
1662 }
1663 }
1664
1665 //=============================================================================
1666
1667 #ifndef PRODUCT
1668 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1669 st->print("BREAKPOINT");
1670 }
1671 #endif
1672
1673 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1674 __ brk(0);
1675 }
1676
1677 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1678 return MachNode::size(ra_);
1679 }
1680
1681 //=============================================================================
1682
1683 #ifndef PRODUCT
1684 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1685 st->print("nop \t# %d bytes pad for loops and calls", _count);
1686 }
1687 #endif
1688
1689 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1690 for (int i = 0; i < _count; i++) {
1691 __ nop();
1692 }
1693 }
1694
1695 uint MachNopNode::size(PhaseRegAlloc*) const {
1696 return _count * NativeInstruction::instruction_size;
1697 }
1698
1699 //=============================================================================
1700 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1701
1702 int ConstantTable::calculate_table_base_offset() const {
1703 return 0; // absolute addressing, no offset
1704 }
1705
1706 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1707 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1708 ShouldNotReachHere();
1709 }
1710
1711 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1712 // Empty encoding
1713 }
1714
1715 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1716 return 0;
1717 }
1718
1719 #ifndef PRODUCT
1720 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1721 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1722 }
1723 #endif
1724
1725 #ifndef PRODUCT
1726 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1727 Compile* C = ra_->C;
1728
1729 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1730
1731 if (C->output()->need_stack_bang(framesize))
1732 st->print("# stack bang size=%d\n\t", framesize);
1733
1734 if (VM_Version::use_rop_protection()) {
1735 st->print("ldr zr, [lr]\n\t");
1736 st->print("paciaz\n\t");
1737 }
1738 if (framesize < ((1 << 9) + 2 * wordSize)) {
1739 st->print("sub sp, sp, #%d\n\t", framesize);
1740 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1741 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1742 } else {
1743 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1744 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1745 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1746 st->print("sub sp, sp, rscratch1");
1747 }
1748 if (C->stub_function() == nullptr) {
1749 st->print("\n\t");
1750 st->print("ldr rscratch1, [guard]\n\t");
1751 st->print("dmb ishld\n\t");
1752 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1753 st->print("cmp rscratch1, rscratch2\n\t");
1754 st->print("b.eq skip");
1755 st->print("\n\t");
1756 st->print("blr #nmethod_entry_barrier_stub\n\t");
1757 st->print("b skip\n\t");
1758 st->print("guard: int\n\t");
1759 st->print("\n\t");
1760 st->print("skip:\n\t");
1761 }
1762 }
1763 #endif
1764
1765 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1766 Compile* C = ra_->C;
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 __ verified_entry(C, 0);
1773
1774 if (C->stub_function() == nullptr) {
1775 __ entry_barrier();
1776 }
1777
1778 if (!Compile::current()->output()->in_scratch_emit_size()) {
1779 __ bind(*_verified_entry);
1780 }
1781
1782 if (VerifyStackAtCalls) {
1783 Unimplemented();
1784 }
1785
1786 C->output()->set_frame_complete(__ offset());
1787
1788 if (C->has_mach_constant_base_node()) {
1789 // NOTE: We set the table base offset here because users might be
1790 // emitted before MachConstantBaseNode.
1791 ConstantTable& constant_table = C->output()->constant_table();
1792 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1793 }
1794 }
1795
1796 int MachPrologNode::reloc() const
1797 {
1798 return 0;
1799 }
1800
1801 //=============================================================================
1802
1803 #ifndef PRODUCT
1804 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1805 Compile* C = ra_->C;
1806 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1807
1808 st->print("# pop frame %d\n\t",framesize);
1809
1810 if (framesize == 0) {
1811 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1812 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1813 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1814 st->print("add sp, sp, #%d\n\t", framesize);
1815 } else {
1816 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1817 st->print("add sp, sp, rscratch1\n\t");
1818 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1819 }
1820 if (VM_Version::use_rop_protection()) {
1821 st->print("autiaz\n\t");
1822 st->print("ldr zr, [lr]\n\t");
1823 }
1824
1825 if (do_polling() && C->is_method_compilation()) {
1826 st->print("# test polling word\n\t");
1827 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1828 st->print("cmp sp, rscratch1\n\t");
1829 st->print("bhi #slow_path");
1830 }
1831 }
1832 #endif
1833
1834 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1835 Compile* C = ra_->C;
1836 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1837
1838 __ remove_frame(framesize, C->needs_stack_repair());
1839
1840 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1841 __ reserved_stack_check();
1842 }
1843
1844 if (do_polling() && C->is_method_compilation()) {
1845 Label dummy_label;
1846 Label* code_stub = &dummy_label;
1847 if (!C->output()->in_scratch_emit_size()) {
1848 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1849 C->output()->add_stub(stub);
1850 code_stub = &stub->entry();
1851 }
1852 __ relocate(relocInfo::poll_return_type);
1853 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1854 }
1855 }
1856
1857 int MachEpilogNode::reloc() const {
1858 // Return number of relocatable values contained in this instruction.
1859 return 1; // 1 for polling page.
1860 }
1861
1862 const Pipeline * MachEpilogNode::pipeline() const {
1863 return MachNode::pipeline_class();
1864 }
1865
1866 //=============================================================================
1867
1868 static enum RC rc_class(OptoReg::Name reg) {
1869
1870 if (reg == OptoReg::Bad) {
1871 return rc_bad;
1872 }
1873
1874 // we have 32 int registers * 2 halves
1875 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1876
1877 if (reg < slots_of_int_registers) {
1878 return rc_int;
1879 }
1880
1881 // we have 32 float register * 8 halves
1882 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1883 if (reg < slots_of_int_registers + slots_of_float_registers) {
1884 return rc_float;
1885 }
1886
1887 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1888 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1889 return rc_predicate;
1890 }
1891
1892 // Between predicate regs & stack is the flags.
1893 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1894
1895 return rc_stack;
1896 }
1897
1898 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1899 Compile* C = ra_->C;
1900
1901 // Get registers to move.
1902 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1903 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1904 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1905 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1906
1907 enum RC src_hi_rc = rc_class(src_hi);
1908 enum RC src_lo_rc = rc_class(src_lo);
1909 enum RC dst_hi_rc = rc_class(dst_hi);
1910 enum RC dst_lo_rc = rc_class(dst_lo);
1911
1912 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1913
1914 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1915 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1916 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1917 "expected aligned-adjacent pairs");
1918 }
1919
1920 if (src_lo == dst_lo && src_hi == dst_hi) {
1921 return 0; // Self copy, no move.
1922 }
1923
1924 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1925 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1926 int src_offset = ra_->reg2offset(src_lo);
1927 int dst_offset = ra_->reg2offset(dst_lo);
1928
1929 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1930 uint ireg = ideal_reg();
1931 if (ireg == Op_VecA && masm) {
1932 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1933 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1934 // stack->stack
1935 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1936 sve_vector_reg_size_in_bytes);
1937 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1938 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1939 sve_vector_reg_size_in_bytes);
1940 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1941 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1942 sve_vector_reg_size_in_bytes);
1943 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1944 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1945 as_FloatRegister(Matcher::_regEncode[src_lo]),
1946 as_FloatRegister(Matcher::_regEncode[src_lo]));
1947 } else {
1948 ShouldNotReachHere();
1949 }
1950 } else if (masm) {
1951 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1952 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1953 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1954 // stack->stack
1955 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1956 if (ireg == Op_VecD) {
1957 __ unspill(rscratch1, true, src_offset);
1958 __ spill(rscratch1, true, dst_offset);
1959 } else {
1960 __ spill_copy128(src_offset, dst_offset);
1961 }
1962 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1963 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1964 ireg == Op_VecD ? __ T8B : __ T16B,
1965 as_FloatRegister(Matcher::_regEncode[src_lo]));
1966 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1967 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1968 ireg == Op_VecD ? __ D : __ Q,
1969 ra_->reg2offset(dst_lo));
1970 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1971 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1972 ireg == Op_VecD ? __ D : __ Q,
1973 ra_->reg2offset(src_lo));
1974 } else {
1975 ShouldNotReachHere();
1976 }
1977 }
1978 } else if (masm) {
1979 switch (src_lo_rc) {
1980 case rc_int:
1981 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
1982 if (is64) {
1983 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1984 as_Register(Matcher::_regEncode[src_lo]));
1985 } else {
1986 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1987 as_Register(Matcher::_regEncode[src_lo]));
1988 }
1989 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1990 if (is64) {
1991 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1992 as_Register(Matcher::_regEncode[src_lo]));
1993 } else {
1994 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1995 as_Register(Matcher::_regEncode[src_lo]));
1996 }
1997 } else { // gpr --> stack spill
1998 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1999 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2000 }
2001 break;
2002 case rc_float:
2003 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2004 if (is64) {
2005 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2006 as_FloatRegister(Matcher::_regEncode[src_lo]));
2007 } else {
2008 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2009 as_FloatRegister(Matcher::_regEncode[src_lo]));
2010 }
2011 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2012 if (is64) {
2013 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2014 as_FloatRegister(Matcher::_regEncode[src_lo]));
2015 } else {
2016 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 as_FloatRegister(Matcher::_regEncode[src_lo]));
2018 }
2019 } else { // fpr --> stack spill
2020 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2021 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2022 is64 ? __ D : __ S, dst_offset);
2023 }
2024 break;
2025 case rc_stack:
2026 if (dst_lo_rc == rc_int) { // stack --> gpr load
2027 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2028 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2029 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2030 is64 ? __ D : __ S, src_offset);
2031 } else if (dst_lo_rc == rc_predicate) {
2032 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2033 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2034 } else { // stack --> stack copy
2035 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2036 if (ideal_reg() == Op_RegVectMask) {
2037 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2038 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2039 } else {
2040 __ unspill(rscratch1, is64, src_offset);
2041 __ spill(rscratch1, is64, dst_offset);
2042 }
2043 }
2044 break;
2045 case rc_predicate:
2046 if (dst_lo_rc == rc_predicate) {
2047 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2048 } else if (dst_lo_rc == rc_stack) {
2049 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2050 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2051 } else {
2052 assert(false, "bad src and dst rc_class combination.");
2053 ShouldNotReachHere();
2054 }
2055 break;
2056 default:
2057 assert(false, "bad rc_class for spill");
2058 ShouldNotReachHere();
2059 }
2060 }
2061
2062 if (st) {
2063 st->print("spill ");
2064 if (src_lo_rc == rc_stack) {
2065 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2066 } else {
2067 st->print("%s -> ", Matcher::regName[src_lo]);
2068 }
2069 if (dst_lo_rc == rc_stack) {
2070 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2071 } else {
2072 st->print("%s", Matcher::regName[dst_lo]);
2073 }
2074 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2075 int vsize = 0;
2076 switch (ideal_reg()) {
2077 case Op_VecD:
2078 vsize = 64;
2079 break;
2080 case Op_VecX:
2081 vsize = 128;
2082 break;
2083 case Op_VecA:
2084 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2085 break;
2086 default:
2087 assert(false, "bad register type for spill");
2088 ShouldNotReachHere();
2089 }
2090 st->print("\t# vector spill size = %d", vsize);
2091 } else if (ideal_reg() == Op_RegVectMask) {
2092 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2093 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2094 st->print("\t# predicate spill size = %d", vsize);
2095 } else {
2096 st->print("\t# spill size = %d", is64 ? 64 : 32);
2097 }
2098 }
2099
2100 return 0;
2101
2102 }
2103
2104 #ifndef PRODUCT
2105 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2106 if (!ra_)
2107 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2108 else
2109 implementation(nullptr, ra_, false, st);
2110 }
2111 #endif
2112
2113 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2114 implementation(masm, ra_, false, nullptr);
2115 }
2116
2117 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2118 return MachNode::size(ra_);
2119 }
2120
2121 //=============================================================================
2122
2123 #ifndef PRODUCT
2124 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2125 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2126 int reg = ra_->get_reg_first(this);
2127 st->print("add %s, rsp, #%d]\t# box lock",
2128 Matcher::regName[reg], offset);
2129 }
2130 #endif
2131
2132 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2133 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2134 int reg = ra_->get_encode(this);
2135
2136 // This add will handle any 24-bit signed offset. 24 bits allows an
2137 // 8 megabyte stack frame.
2138 __ add(as_Register(reg), sp, offset);
2139 }
2140
2141 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2142 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2143 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2144
2145 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2146 return NativeInstruction::instruction_size;
2147 } else {
2148 return 2 * NativeInstruction::instruction_size;
2149 }
2150 }
2151
2152 ///=============================================================================
2153 #ifndef PRODUCT
2154 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2155 {
2156 st->print_cr("# MachVEPNode");
2157 if (!_verified) {
2158 st->print_cr("\t load_class");
2159 } else {
2160 st->print_cr("\t unpack_inline_arg");
2161 }
2162 }
2163 #endif
2164
2165 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2166 {
2167 if (!_verified) {
2168 __ ic_check(1);
2169 } else {
2170 // insert a nop at the start of the prolog so we can patch in a
2171 // branch if we need to invalidate the method later
2172 __ nop();
2173
2174 // TODO 8284443 Avoid creation of temporary frame
2175 if (ra_->C->stub_function() == nullptr) {
2176 __ verified_entry(ra_->C, 0);
2177 __ entry_barrier();
2178 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2179 __ remove_frame(framesize, false);
2180 }
2181 // Unpack inline type args passed as oop and then jump to
2182 // the verified entry point (skipping the unverified entry).
2183 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2184 // Emit code for verified entry and save increment for stack repair on return
2185 __ verified_entry(ra_->C, sp_inc);
2186 if (Compile::current()->output()->in_scratch_emit_size()) {
2187 Label dummy_verified_entry;
2188 __ b(dummy_verified_entry);
2189 } else {
2190 __ b(*_verified_entry);
2191 }
2192 }
2193 }
2194
2195 //=============================================================================
2196 #ifndef PRODUCT
2197 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2198 {
2199 st->print_cr("# MachUEPNode");
2200 if (UseCompressedClassPointers) {
2201 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2202 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2203 st->print_cr("\tcmpw rscratch1, r10");
2204 } else {
2205 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2206 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2207 st->print_cr("\tcmp rscratch1, r10");
2208 }
2209 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2210 }
2211 #endif
2212
2213 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2214 {
2215 __ ic_check(InteriorEntryAlignment);
2216 }
2217
2218 // REQUIRED EMIT CODE
2219
2220 //=============================================================================
2221
2222 // Emit exception handler code.
2223 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2224 {
2225 // mov rscratch1 #exception_blob_entry_point
2226 // br rscratch1
2227 // Note that the code buffer's insts_mark is always relative to insts.
2228 // That's why we must use the macroassembler to generate a handler.
2229 address base = __ start_a_stub(size_exception_handler());
2230 if (base == nullptr) {
2231 ciEnv::current()->record_failure("CodeCache is full");
2232 return 0; // CodeBuffer::expand failed
2233 }
2234 int offset = __ offset();
2235 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2236 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2237 __ end_a_stub();
2238 return offset;
2239 }
2240
2241 // Emit deopt handler code.
2242 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2243 {
2244 // Note that the code buffer's insts_mark is always relative to insts.
2245 // That's why we must use the macroassembler to generate a handler.
2246 address base = __ start_a_stub(size_deopt_handler());
2247 if (base == nullptr) {
2248 ciEnv::current()->record_failure("CodeCache is full");
2249 return 0; // CodeBuffer::expand failed
2250 }
2251 int offset = __ offset();
2252
2253 __ adr(lr, __ pc());
2254 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2255
2256 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2257 __ end_a_stub();
2258 return offset;
2259 }
2260
2261 // REQUIRED MATCHER CODE
2262
2263 //=============================================================================
2264
2265 bool Matcher::match_rule_supported(int opcode) {
2266 if (!has_match_rule(opcode))
2267 return false;
2268
2269 switch (opcode) {
2270 case Op_OnSpinWait:
2271 return VM_Version::supports_on_spin_wait();
2272 case Op_CacheWB:
2273 case Op_CacheWBPreSync:
2274 case Op_CacheWBPostSync:
2275 if (!VM_Version::supports_data_cache_line_flush()) {
2276 return false;
2277 }
2278 break;
2279 case Op_ExpandBits:
2280 case Op_CompressBits:
2281 if (!VM_Version::supports_svebitperm()) {
2282 return false;
2283 }
2284 break;
2285 case Op_FmaF:
2286 case Op_FmaD:
2287 case Op_FmaVF:
2288 case Op_FmaVD:
2289 if (!UseFMA) {
2290 return false;
2291 }
2292 break;
2293 case Op_FmaHF:
2294 // UseFMA flag also needs to be checked along with FEAT_FP16
2295 if (!UseFMA || !is_feat_fp16_supported()) {
2296 return false;
2297 }
2298 break;
2299 case Op_AddHF:
2300 case Op_SubHF:
2301 case Op_MulHF:
2302 case Op_DivHF:
2303 case Op_MinHF:
2304 case Op_MaxHF:
2305 case Op_SqrtHF:
2306 // Half-precision floating point scalar operations require FEAT_FP16
2307 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2308 // features are supported.
2309 if (!is_feat_fp16_supported()) {
2310 return false;
2311 }
2312 break;
2313 }
2314
2315 return true; // Per default match rules are supported.
2316 }
2317
2318 const RegMask* Matcher::predicate_reg_mask(void) {
2319 return &_PR_REG_mask;
2320 }
2321
2322 bool Matcher::supports_vector_calling_convention(void) {
2323 return EnableVectorSupport;
2324 }
2325
2326 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2327 assert(EnableVectorSupport, "sanity");
2328 int lo = V0_num;
2329 int hi = V0_H_num;
2330 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2331 hi = V0_K_num;
2332 }
2333 return OptoRegPair(hi, lo);
2334 }
2335
2336 // Is this branch offset short enough that a short branch can be used?
2337 //
2338 // NOTE: If the platform does not provide any short branch variants, then
2339 // this method should return false for offset 0.
2340 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2341 // The passed offset is relative to address of the branch.
2342
2343 return (-32768 <= offset && offset < 32768);
2344 }
2345
2346 // Vector width in bytes.
2347 int Matcher::vector_width_in_bytes(BasicType bt) {
2348 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2349 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2350 // Minimum 2 values in vector
2351 if (size < 2*type2aelembytes(bt)) size = 0;
2352 // But never < 4
2353 if (size < 4) size = 0;
2354 return size;
2355 }
2356
2357 // Limits on vector size (number of elements) loaded into vector.
2358 int Matcher::max_vector_size(const BasicType bt) {
2359 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2360 }
2361
2362 int Matcher::min_vector_size(const BasicType bt) {
2363 int max_size = max_vector_size(bt);
2364 // Limit the min vector size to 8 bytes.
2365 int size = 8 / type2aelembytes(bt);
2366 if (bt == T_BYTE) {
2367 // To support vector api shuffle/rearrange.
2368 size = 4;
2369 } else if (bt == T_BOOLEAN) {
2370 // To support vector api load/store mask.
2371 size = 2;
2372 }
2373 if (size < 2) size = 2;
2374 return MIN2(size, max_size);
2375 }
2376
2377 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2378 return Matcher::max_vector_size(bt);
2379 }
2380
2381 // Actual max scalable vector register length.
2382 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2383 return Matcher::max_vector_size(bt);
2384 }
2385
2386 // Vector ideal reg.
2387 uint Matcher::vector_ideal_reg(int len) {
2388 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2389 return Op_VecA;
2390 }
2391 switch(len) {
2392 // For 16-bit/32-bit mask vector, reuse VecD.
2393 case 2:
2394 case 4:
2395 case 8: return Op_VecD;
2396 case 16: return Op_VecX;
2397 }
2398 ShouldNotReachHere();
2399 return 0;
2400 }
2401
2402 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2403 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2404 switch (ideal_reg) {
2405 case Op_VecA: return new vecAOper();
2406 case Op_VecD: return new vecDOper();
2407 case Op_VecX: return new vecXOper();
2408 }
2409 ShouldNotReachHere();
2410 return nullptr;
2411 }
2412
2413 bool Matcher::is_reg2reg_move(MachNode* m) {
2414 return false;
2415 }
2416
2417 bool Matcher::is_generic_vector(MachOper* opnd) {
2418 return opnd->opcode() == VREG;
2419 }
2420
2421 // Return whether or not this register is ever used as an argument.
2422 // This function is used on startup to build the trampoline stubs in
2423 // generateOptoStub. Registers not mentioned will be killed by the VM
2424 // call in the trampoline, and arguments in those registers not be
2425 // available to the callee.
2426 bool Matcher::can_be_java_arg(int reg)
2427 {
2428 return
2429 reg == R0_num || reg == R0_H_num ||
2430 reg == R1_num || reg == R1_H_num ||
2431 reg == R2_num || reg == R2_H_num ||
2432 reg == R3_num || reg == R3_H_num ||
2433 reg == R4_num || reg == R4_H_num ||
2434 reg == R5_num || reg == R5_H_num ||
2435 reg == R6_num || reg == R6_H_num ||
2436 reg == R7_num || reg == R7_H_num ||
2437 reg == V0_num || reg == V0_H_num ||
2438 reg == V1_num || reg == V1_H_num ||
2439 reg == V2_num || reg == V2_H_num ||
2440 reg == V3_num || reg == V3_H_num ||
2441 reg == V4_num || reg == V4_H_num ||
2442 reg == V5_num || reg == V5_H_num ||
2443 reg == V6_num || reg == V6_H_num ||
2444 reg == V7_num || reg == V7_H_num;
2445 }
2446
2447 bool Matcher::is_spillable_arg(int reg)
2448 {
2449 return can_be_java_arg(reg);
2450 }
2451
2452 uint Matcher::int_pressure_limit()
2453 {
2454 // JDK-8183543: When taking the number of available registers as int
2455 // register pressure threshold, the jtreg test:
2456 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2457 // failed due to C2 compilation failure with
2458 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2459 //
2460 // A derived pointer is live at CallNode and then is flagged by RA
2461 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2462 // derived pointers and lastly fail to spill after reaching maximum
2463 // number of iterations. Lowering the default pressure threshold to
2464 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2465 // a high register pressure area of the code so that split_DEF can
2466 // generate DefinitionSpillCopy for the derived pointer.
2467 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2468 if (!PreserveFramePointer) {
2469 // When PreserveFramePointer is off, frame pointer is allocatable,
2470 // but different from other SOC registers, it is excluded from
2471 // fatproj's mask because its save type is No-Save. Decrease 1 to
2472 // ensure high pressure at fatproj when PreserveFramePointer is off.
2473 // See check_pressure_at_fatproj().
2474 default_int_pressure_threshold--;
2475 }
2476 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2477 }
2478
2479 uint Matcher::float_pressure_limit()
2480 {
2481 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2482 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2483 }
2484
2485 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2486 return false;
2487 }
2488
2489 RegMask Matcher::divI_proj_mask() {
2490 ShouldNotReachHere();
2491 return RegMask();
2492 }
2493
2494 // Register for MODI projection of divmodI.
2495 RegMask Matcher::modI_proj_mask() {
2496 ShouldNotReachHere();
2497 return RegMask();
2498 }
2499
2500 // Register for DIVL projection of divmodL.
2501 RegMask Matcher::divL_proj_mask() {
2502 ShouldNotReachHere();
2503 return RegMask();
2504 }
2505
2506 // Register for MODL projection of divmodL.
2507 RegMask Matcher::modL_proj_mask() {
2508 ShouldNotReachHere();
2509 return RegMask();
2510 }
2511
2512 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2513 return FP_REG_mask();
2514 }
2515
2516 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2517 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2518 Node* u = addp->fast_out(i);
2519 if (u->is_LoadStore()) {
2520 // On AArch64, LoadStoreNodes (i.e. compare and swap
2521 // instructions) only take register indirect as an operand, so
2522 // any attempt to use an AddPNode as an input to a LoadStoreNode
2523 // must fail.
2524 return false;
2525 }
2526 if (u->is_Mem()) {
2527 int opsize = u->as_Mem()->memory_size();
2528 assert(opsize > 0, "unexpected memory operand size");
2529 if (u->as_Mem()->memory_size() != (1<<shift)) {
2530 return false;
2531 }
2532 }
2533 }
2534 return true;
2535 }
2536
2537 // Convert BootTest condition to Assembler condition.
2538 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2539 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2540 Assembler::Condition result;
2541 switch(cond) {
2542 case BoolTest::eq:
2543 result = Assembler::EQ; break;
2544 case BoolTest::ne:
2545 result = Assembler::NE; break;
2546 case BoolTest::le:
2547 result = Assembler::LE; break;
2548 case BoolTest::ge:
2549 result = Assembler::GE; break;
2550 case BoolTest::lt:
2551 result = Assembler::LT; break;
2552 case BoolTest::gt:
2553 result = Assembler::GT; break;
2554 case BoolTest::ule:
2555 result = Assembler::LS; break;
2556 case BoolTest::uge:
2557 result = Assembler::HS; break;
2558 case BoolTest::ult:
2559 result = Assembler::LO; break;
2560 case BoolTest::ugt:
2561 result = Assembler::HI; break;
2562 case BoolTest::overflow:
2563 result = Assembler::VS; break;
2564 case BoolTest::no_overflow:
2565 result = Assembler::VC; break;
2566 default:
2567 ShouldNotReachHere();
2568 return Assembler::Condition(-1);
2569 }
2570
2571 // Check conversion
2572 if (cond & BoolTest::unsigned_compare) {
2573 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2574 } else {
2575 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2576 }
2577
2578 return result;
2579 }
2580
2581 // Binary src (Replicate con)
2582 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2583 if (n == nullptr || m == nullptr) {
2584 return false;
2585 }
2586
2587 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2588 return false;
2589 }
2590
2591 Node* imm_node = m->in(1);
2592 if (!imm_node->is_Con()) {
2593 return false;
2594 }
2595
2596 const Type* t = imm_node->bottom_type();
2597 if (!(t->isa_int() || t->isa_long())) {
2598 return false;
2599 }
2600
2601 switch (n->Opcode()) {
2602 case Op_AndV:
2603 case Op_OrV:
2604 case Op_XorV: {
2605 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2606 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2607 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2608 }
2609 case Op_AddVB:
2610 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2611 case Op_AddVS:
2612 case Op_AddVI:
2613 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2614 case Op_AddVL:
2615 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2616 default:
2617 return false;
2618 }
2619 }
2620
2621 // (XorV src (Replicate m1))
2622 // (XorVMask src (MaskAll m1))
2623 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2624 if (n != nullptr && m != nullptr) {
2625 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2626 VectorNode::is_all_ones_vector(m);
2627 }
2628 return false;
2629 }
2630
2631 // Should the matcher clone input 'm' of node 'n'?
2632 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2633 if (is_vshift_con_pattern(n, m) ||
2634 is_vector_bitwise_not_pattern(n, m) ||
2635 is_valid_sve_arith_imm_pattern(n, m) ||
2636 is_encode_and_store_pattern(n, m)) {
2637 mstack.push(m, Visit);
2638 return true;
2639 }
2640 return false;
2641 }
2642
2643 // Should the Matcher clone shifts on addressing modes, expecting them
2644 // to be subsumed into complex addressing expressions or compute them
2645 // into registers?
2646 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2647
2648 // Loads and stores with indirect memory input (e.g., volatile loads and
2649 // stores) do not subsume the input into complex addressing expressions. If
2650 // the addressing expression is input to at least one such load or store, do
2651 // not clone the addressing expression. Query needs_acquiring_load and
2652 // needs_releasing_store as a proxy for indirect memory input, as it is not
2653 // possible to directly query for indirect memory input at this stage.
2654 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2655 Node* n = m->fast_out(i);
2656 if (n->is_Load() && needs_acquiring_load(n)) {
2657 return false;
2658 }
2659 if (n->is_Store() && needs_releasing_store(n)) {
2660 return false;
2661 }
2662 }
2663
2664 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2665 return true;
2666 }
2667
2668 Node *off = m->in(AddPNode::Offset);
2669 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2670 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2671 // Are there other uses besides address expressions?
2672 !is_visited(off)) {
2673 address_visited.set(off->_idx); // Flag as address_visited
2674 mstack.push(off->in(2), Visit);
2675 Node *conv = off->in(1);
2676 if (conv->Opcode() == Op_ConvI2L &&
2677 // Are there other uses besides address expressions?
2678 !is_visited(conv)) {
2679 address_visited.set(conv->_idx); // Flag as address_visited
2680 mstack.push(conv->in(1), Pre_Visit);
2681 } else {
2682 mstack.push(conv, Pre_Visit);
2683 }
2684 address_visited.test_set(m->_idx); // Flag as address_visited
2685 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2686 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2687 return true;
2688 } else if (off->Opcode() == Op_ConvI2L &&
2689 // Are there other uses besides address expressions?
2690 !is_visited(off)) {
2691 address_visited.test_set(m->_idx); // Flag as address_visited
2692 address_visited.set(off->_idx); // Flag as address_visited
2693 mstack.push(off->in(1), Pre_Visit);
2694 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2695 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2696 return true;
2697 }
2698 return false;
2699 }
2700
2701 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2702 { \
2703 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2704 guarantee(DISP == 0, "mode not permitted for volatile"); \
2705 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2706 __ INSN(REG, as_Register(BASE)); \
2707 }
2708
2709
2710 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2711 {
2712 Address::extend scale;
2713
2714 // Hooboy, this is fugly. We need a way to communicate to the
2715 // encoder that the index needs to be sign extended, so we have to
2716 // enumerate all the cases.
2717 switch (opcode) {
2718 case INDINDEXSCALEDI2L:
2719 case INDINDEXSCALEDI2LN:
2720 case INDINDEXI2L:
2721 case INDINDEXI2LN:
2722 scale = Address::sxtw(size);
2723 break;
2724 default:
2725 scale = Address::lsl(size);
2726 }
2727
2728 if (index == -1) {
2729 return Address(base, disp);
2730 } else {
2731 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2732 return Address(base, as_Register(index), scale);
2733 }
2734 }
2735
2736
2737 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2738 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2739 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2740 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2741 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2742
2743 // Used for all non-volatile memory accesses. The use of
2744 // $mem->opcode() to discover whether this pattern uses sign-extended
2745 // offsets is something of a kludge.
2746 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2747 Register reg, int opcode,
2748 Register base, int index, int scale, int disp,
2749 int size_in_memory)
2750 {
2751 Address addr = mem2address(opcode, base, index, scale, disp);
2752 if (addr.getMode() == Address::base_plus_offset) {
2753 /* Fix up any out-of-range offsets. */
2754 assert_different_registers(rscratch1, base);
2755 assert_different_registers(rscratch1, reg);
2756 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2757 }
2758 (masm->*insn)(reg, addr);
2759 }
2760
2761 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2762 FloatRegister reg, int opcode,
2763 Register base, int index, int size, int disp,
2764 int size_in_memory)
2765 {
2766 Address::extend scale;
2767
2768 switch (opcode) {
2769 case INDINDEXSCALEDI2L:
2770 case INDINDEXSCALEDI2LN:
2771 scale = Address::sxtw(size);
2772 break;
2773 default:
2774 scale = Address::lsl(size);
2775 }
2776
2777 if (index == -1) {
2778 // Fix up any out-of-range offsets.
2779 assert_different_registers(rscratch1, base);
2780 Address addr = Address(base, disp);
2781 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2782 (masm->*insn)(reg, addr);
2783 } else {
2784 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2785 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2786 }
2787 }
2788
2789 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2790 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2791 int opcode, Register base, int index, int size, int disp)
2792 {
2793 if (index == -1) {
2794 (masm->*insn)(reg, T, Address(base, disp));
2795 } else {
2796 assert(disp == 0, "unsupported address mode");
2797 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2798 }
2799 }
2800
2801 %}
2802
2803
2804
2805 //----------ENCODING BLOCK-----------------------------------------------------
2806 // This block specifies the encoding classes used by the compiler to
2807 // output byte streams. Encoding classes are parameterized macros
2808 // used by Machine Instruction Nodes in order to generate the bit
2809 // encoding of the instruction. Operands specify their base encoding
2810 // interface with the interface keyword. There are currently
2811 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2812 // COND_INTER. REG_INTER causes an operand to generate a function
2813 // which returns its register number when queried. CONST_INTER causes
2814 // an operand to generate a function which returns the value of the
2815 // constant when queried. MEMORY_INTER causes an operand to generate
2816 // four functions which return the Base Register, the Index Register,
2817 // the Scale Value, and the Offset Value of the operand when queried.
2818 // COND_INTER causes an operand to generate six functions which return
2819 // the encoding code (ie - encoding bits for the instruction)
2820 // associated with each basic boolean condition for a conditional
2821 // instruction.
2822 //
2823 // Instructions specify two basic values for encoding. Again, a
2824 // function is available to check if the constant displacement is an
2825 // oop. They use the ins_encode keyword to specify their encoding
2826 // classes (which must be a sequence of enc_class names, and their
2827 // parameters, specified in the encoding block), and they use the
2828 // opcode keyword to specify, in order, their primary, secondary, and
2829 // tertiary opcode. Only the opcode sections which a particular
2830 // instruction needs for encoding need to be specified.
2831 encode %{
2832 // Build emit functions for each basic byte or larger field in the
2833 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2834 // from C++ code in the enc_class source block. Emit functions will
2835 // live in the main source block for now. In future, we can
2836 // generalize this by adding a syntax that specifies the sizes of
2837 // fields in an order, so that the adlc can build the emit functions
2838 // automagically
2839
2840 // catch all for unimplemented encodings
2841 enc_class enc_unimplemented %{
2842 __ unimplemented("C2 catch all");
2843 %}
2844
2845 // BEGIN Non-volatile memory access
2846
2847 // This encoding class is generated automatically from ad_encode.m4.
2848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2849 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2850 Register dst_reg = as_Register($dst$$reg);
2851 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2852 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2853 %}
2854
2855 // This encoding class is generated automatically from ad_encode.m4.
2856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2857 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2858 Register dst_reg = as_Register($dst$$reg);
2859 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2860 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2861 %}
2862
2863 // This encoding class is generated automatically from ad_encode.m4.
2864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2865 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2866 Register dst_reg = as_Register($dst$$reg);
2867 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2868 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2869 %}
2870
2871 // This encoding class is generated automatically from ad_encode.m4.
2872 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2873 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2874 Register dst_reg = as_Register($dst$$reg);
2875 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2876 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2877 %}
2878
2879 // This encoding class is generated automatically from ad_encode.m4.
2880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2881 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2882 Register dst_reg = as_Register($dst$$reg);
2883 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2884 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2885 %}
2886
2887 // This encoding class is generated automatically from ad_encode.m4.
2888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2889 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2890 Register dst_reg = as_Register($dst$$reg);
2891 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2892 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2893 %}
2894
2895 // This encoding class is generated automatically from ad_encode.m4.
2896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2897 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2898 Register dst_reg = as_Register($dst$$reg);
2899 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2900 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2901 %}
2902
2903 // This encoding class is generated automatically from ad_encode.m4.
2904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2905 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2906 Register dst_reg = as_Register($dst$$reg);
2907 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2908 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2909 %}
2910
2911 // This encoding class is generated automatically from ad_encode.m4.
2912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2913 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2914 Register dst_reg = as_Register($dst$$reg);
2915 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2916 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2917 %}
2918
2919 // This encoding class is generated automatically from ad_encode.m4.
2920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2921 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2922 Register dst_reg = as_Register($dst$$reg);
2923 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2924 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2925 %}
2926
2927 // This encoding class is generated automatically from ad_encode.m4.
2928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2929 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2930 Register dst_reg = as_Register($dst$$reg);
2931 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2932 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2933 %}
2934
2935 // This encoding class is generated automatically from ad_encode.m4.
2936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2937 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2938 Register dst_reg = as_Register($dst$$reg);
2939 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2940 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2941 %}
2942
2943 // This encoding class is generated automatically from ad_encode.m4.
2944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2945 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2946 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2947 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2948 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2949 %}
2950
2951 // This encoding class is generated automatically from ad_encode.m4.
2952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2953 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2954 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2955 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2956 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2957 %}
2958
2959 // This encoding class is generated automatically from ad_encode.m4.
2960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2961 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2962 Register src_reg = as_Register($src$$reg);
2963 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2964 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2965 %}
2966
2967 // This encoding class is generated automatically from ad_encode.m4.
2968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2969 enc_class aarch64_enc_strb0(memory1 mem) %{
2970 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2971 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2972 %}
2973
2974 // This encoding class is generated automatically from ad_encode.m4.
2975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2976 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2977 Register src_reg = as_Register($src$$reg);
2978 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2979 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2980 %}
2981
2982 // This encoding class is generated automatically from ad_encode.m4.
2983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2984 enc_class aarch64_enc_strh0(memory2 mem) %{
2985 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2986 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2987 %}
2988
2989 // This encoding class is generated automatically from ad_encode.m4.
2990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2991 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2992 Register src_reg = as_Register($src$$reg);
2993 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
2994 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2995 %}
2996
2997 // This encoding class is generated automatically from ad_encode.m4.
2998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2999 enc_class aarch64_enc_strw0(memory4 mem) %{
3000 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3001 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3002 %}
3003
3004 // This encoding class is generated automatically from ad_encode.m4.
3005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3006 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3007 Register src_reg = as_Register($src$$reg);
3008 // we sometimes get asked to store the stack pointer into the
3009 // current thread -- we cannot do that directly on AArch64
3010 if (src_reg == r31_sp) {
3011 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3012 __ mov(rscratch2, sp);
3013 src_reg = rscratch2;
3014 }
3015 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3016 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3017 %}
3018
3019 // This encoding class is generated automatically from ad_encode.m4.
3020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3021 enc_class aarch64_enc_str0(memory8 mem) %{
3022 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3023 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3024 %}
3025
3026 // This encoding class is generated automatically from ad_encode.m4.
3027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3028 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3029 FloatRegister src_reg = as_FloatRegister($src$$reg);
3030 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3031 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3032 %}
3033
3034 // This encoding class is generated automatically from ad_encode.m4.
3035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3036 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3037 FloatRegister src_reg = as_FloatRegister($src$$reg);
3038 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3039 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3040 %}
3041
3042 // This encoding class is generated automatically from ad_encode.m4.
3043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3044 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3045 __ membar(Assembler::StoreStore);
3046 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3047 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3048 %}
3049
3050 // END Non-volatile memory access
3051
3052 // Vector loads and stores
3053 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3054 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3055 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3056 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3057 %}
3058
3059 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3060 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3061 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3062 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3063 %}
3064
3065 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3066 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3067 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3068 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3069 %}
3070
3071 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3072 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3073 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3074 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3075 %}
3076
3077 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3078 FloatRegister src_reg = as_FloatRegister($src$$reg);
3079 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3080 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3081 %}
3082
3083 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3084 FloatRegister src_reg = as_FloatRegister($src$$reg);
3085 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3086 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3087 %}
3088
3089 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3090 FloatRegister src_reg = as_FloatRegister($src$$reg);
3091 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3092 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3093 %}
3094
3095 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3096 FloatRegister src_reg = as_FloatRegister($src$$reg);
3097 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3098 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3099 %}
3100
3101 // volatile loads and stores
3102
3103 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3104 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3105 rscratch1, stlrb);
3106 %}
3107
3108 enc_class aarch64_enc_stlrb0(memory mem) %{
3109 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3110 rscratch1, stlrb);
3111 %}
3112
3113 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3114 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3115 rscratch1, stlrh);
3116 %}
3117
3118 enc_class aarch64_enc_stlrh0(memory mem) %{
3119 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3120 rscratch1, stlrh);
3121 %}
3122
3123 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3124 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3125 rscratch1, stlrw);
3126 %}
3127
3128 enc_class aarch64_enc_stlrw0(memory mem) %{
3129 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3130 rscratch1, stlrw);
3131 %}
3132
3133 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3134 Register dst_reg = as_Register($dst$$reg);
3135 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3136 rscratch1, ldarb);
3137 __ sxtbw(dst_reg, dst_reg);
3138 %}
3139
3140 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3141 Register dst_reg = as_Register($dst$$reg);
3142 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3143 rscratch1, ldarb);
3144 __ sxtb(dst_reg, dst_reg);
3145 %}
3146
3147 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3148 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3149 rscratch1, ldarb);
3150 %}
3151
3152 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3153 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3154 rscratch1, ldarb);
3155 %}
3156
3157 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3158 Register dst_reg = as_Register($dst$$reg);
3159 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, ldarh);
3161 __ sxthw(dst_reg, dst_reg);
3162 %}
3163
3164 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3165 Register dst_reg = as_Register($dst$$reg);
3166 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3167 rscratch1, ldarh);
3168 __ sxth(dst_reg, dst_reg);
3169 %}
3170
3171 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3172 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3173 rscratch1, ldarh);
3174 %}
3175
3176 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3177 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3178 rscratch1, ldarh);
3179 %}
3180
3181 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3182 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3183 rscratch1, ldarw);
3184 %}
3185
3186 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3187 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3188 rscratch1, ldarw);
3189 %}
3190
3191 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3192 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3193 rscratch1, ldar);
3194 %}
3195
3196 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3197 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3198 rscratch1, ldarw);
3199 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3200 %}
3201
3202 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3203 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3204 rscratch1, ldar);
3205 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3206 %}
3207
3208 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3209 Register src_reg = as_Register($src$$reg);
3210 // we sometimes get asked to store the stack pointer into the
3211 // current thread -- we cannot do that directly on AArch64
3212 if (src_reg == r31_sp) {
3213 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3214 __ mov(rscratch2, sp);
3215 src_reg = rscratch2;
3216 }
3217 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3218 rscratch1, stlr);
3219 %}
3220
3221 enc_class aarch64_enc_stlr0(memory mem) %{
3222 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3223 rscratch1, stlr);
3224 %}
3225
3226 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3227 {
3228 FloatRegister src_reg = as_FloatRegister($src$$reg);
3229 __ fmovs(rscratch2, src_reg);
3230 }
3231 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3232 rscratch1, stlrw);
3233 %}
3234
3235 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3236 {
3237 FloatRegister src_reg = as_FloatRegister($src$$reg);
3238 __ fmovd(rscratch2, src_reg);
3239 }
3240 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3241 rscratch1, stlr);
3242 %}
3243
3244 // synchronized read/update encodings
3245
3246 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3247 Register dst_reg = as_Register($dst$$reg);
3248 Register base = as_Register($mem$$base);
3249 int index = $mem$$index;
3250 int scale = $mem$$scale;
3251 int disp = $mem$$disp;
3252 if (index == -1) {
3253 if (disp != 0) {
3254 __ lea(rscratch1, Address(base, disp));
3255 __ ldaxr(dst_reg, rscratch1);
3256 } else {
3257 // TODO
3258 // should we ever get anything other than this case?
3259 __ ldaxr(dst_reg, base);
3260 }
3261 } else {
3262 Register index_reg = as_Register(index);
3263 if (disp == 0) {
3264 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3265 __ ldaxr(dst_reg, rscratch1);
3266 } else {
3267 __ lea(rscratch1, Address(base, disp));
3268 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3269 __ ldaxr(dst_reg, rscratch1);
3270 }
3271 }
3272 %}
3273
3274 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3275 Register src_reg = as_Register($src$$reg);
3276 Register base = as_Register($mem$$base);
3277 int index = $mem$$index;
3278 int scale = $mem$$scale;
3279 int disp = $mem$$disp;
3280 if (index == -1) {
3281 if (disp != 0) {
3282 __ lea(rscratch2, Address(base, disp));
3283 __ stlxr(rscratch1, src_reg, rscratch2);
3284 } else {
3285 // TODO
3286 // should we ever get anything other than this case?
3287 __ stlxr(rscratch1, src_reg, base);
3288 }
3289 } else {
3290 Register index_reg = as_Register(index);
3291 if (disp == 0) {
3292 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3293 __ stlxr(rscratch1, src_reg, rscratch2);
3294 } else {
3295 __ lea(rscratch2, Address(base, disp));
3296 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3297 __ stlxr(rscratch1, src_reg, rscratch2);
3298 }
3299 }
3300 __ cmpw(rscratch1, zr);
3301 %}
3302
3303 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3304 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3305 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3306 Assembler::xword, /*acquire*/ false, /*release*/ true,
3307 /*weak*/ false, noreg);
3308 %}
3309
3310 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3311 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3312 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3313 Assembler::word, /*acquire*/ false, /*release*/ true,
3314 /*weak*/ false, noreg);
3315 %}
3316
3317 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3318 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3319 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3320 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3321 /*weak*/ false, noreg);
3322 %}
3323
3324 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3325 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3326 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3327 Assembler::byte, /*acquire*/ false, /*release*/ true,
3328 /*weak*/ false, noreg);
3329 %}
3330
3331
3332 // The only difference between aarch64_enc_cmpxchg and
3333 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3334 // CompareAndSwap sequence to serve as a barrier on acquiring a
3335 // lock.
3336 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3337 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3338 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3339 Assembler::xword, /*acquire*/ true, /*release*/ true,
3340 /*weak*/ false, noreg);
3341 %}
3342
3343 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3344 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3345 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3346 Assembler::word, /*acquire*/ true, /*release*/ true,
3347 /*weak*/ false, noreg);
3348 %}
3349
3350 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3351 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3352 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3353 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3354 /*weak*/ false, noreg);
3355 %}
3356
3357 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3358 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3359 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3360 Assembler::byte, /*acquire*/ true, /*release*/ true,
3361 /*weak*/ false, noreg);
3362 %}
3363
3364 // auxiliary used for CompareAndSwapX to set result register
3365 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3366 Register res_reg = as_Register($res$$reg);
3367 __ cset(res_reg, Assembler::EQ);
3368 %}
3369
3370 // prefetch encodings
3371
3372 enc_class aarch64_enc_prefetchw(memory mem) %{
3373 Register base = as_Register($mem$$base);
3374 int index = $mem$$index;
3375 int scale = $mem$$scale;
3376 int disp = $mem$$disp;
3377 if (index == -1) {
3378 // Fix up any out-of-range offsets.
3379 assert_different_registers(rscratch1, base);
3380 Address addr = Address(base, disp);
3381 addr = __ legitimize_address(addr, 8, rscratch1);
3382 __ prfm(addr, PSTL1KEEP);
3383 } else {
3384 Register index_reg = as_Register(index);
3385 if (disp == 0) {
3386 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3387 } else {
3388 __ lea(rscratch1, Address(base, disp));
3389 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3390 }
3391 }
3392 %}
3393
3394 // mov encodings
3395
3396 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3397 uint32_t con = (uint32_t)$src$$constant;
3398 Register dst_reg = as_Register($dst$$reg);
3399 if (con == 0) {
3400 __ movw(dst_reg, zr);
3401 } else {
3402 __ movw(dst_reg, con);
3403 }
3404 %}
3405
3406 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3407 Register dst_reg = as_Register($dst$$reg);
3408 uint64_t con = (uint64_t)$src$$constant;
3409 if (con == 0) {
3410 __ mov(dst_reg, zr);
3411 } else {
3412 __ mov(dst_reg, con);
3413 }
3414 %}
3415
3416 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3417 Register dst_reg = as_Register($dst$$reg);
3418 address con = (address)$src$$constant;
3419 if (con == nullptr || con == (address)1) {
3420 ShouldNotReachHere();
3421 } else {
3422 relocInfo::relocType rtype = $src->constant_reloc();
3423 if (rtype == relocInfo::oop_type) {
3424 __ movoop(dst_reg, (jobject)con);
3425 } else if (rtype == relocInfo::metadata_type) {
3426 __ mov_metadata(dst_reg, (Metadata*)con);
3427 } else {
3428 assert(rtype == relocInfo::none, "unexpected reloc type");
3429 if (! __ is_valid_AArch64_address(con) ||
3430 con < (address)(uintptr_t)os::vm_page_size()) {
3431 __ mov(dst_reg, con);
3432 } else {
3433 uint64_t offset;
3434 __ adrp(dst_reg, con, offset);
3435 __ add(dst_reg, dst_reg, offset);
3436 }
3437 }
3438 }
3439 %}
3440
3441 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3442 Register dst_reg = as_Register($dst$$reg);
3443 __ mov(dst_reg, zr);
3444 %}
3445
3446 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3447 Register dst_reg = as_Register($dst$$reg);
3448 __ mov(dst_reg, (uint64_t)1);
3449 %}
3450
3451 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3452 __ load_byte_map_base($dst$$Register);
3453 %}
3454
3455 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3456 Register dst_reg = as_Register($dst$$reg);
3457 address con = (address)$src$$constant;
3458 if (con == nullptr) {
3459 ShouldNotReachHere();
3460 } else {
3461 relocInfo::relocType rtype = $src->constant_reloc();
3462 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3463 __ set_narrow_oop(dst_reg, (jobject)con);
3464 }
3465 %}
3466
3467 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3468 Register dst_reg = as_Register($dst$$reg);
3469 __ mov(dst_reg, zr);
3470 %}
3471
3472 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3473 Register dst_reg = as_Register($dst$$reg);
3474 address con = (address)$src$$constant;
3475 if (con == nullptr) {
3476 ShouldNotReachHere();
3477 } else {
3478 relocInfo::relocType rtype = $src->constant_reloc();
3479 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3480 __ set_narrow_klass(dst_reg, (Klass *)con);
3481 }
3482 %}
3483
3484 // arithmetic encodings
3485
3486 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3487 Register dst_reg = as_Register($dst$$reg);
3488 Register src_reg = as_Register($src1$$reg);
3489 int32_t con = (int32_t)$src2$$constant;
3490 // add has primary == 0, subtract has primary == 1
3491 if ($primary) { con = -con; }
3492 if (con < 0) {
3493 __ subw(dst_reg, src_reg, -con);
3494 } else {
3495 __ addw(dst_reg, src_reg, con);
3496 }
3497 %}
3498
3499 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3500 Register dst_reg = as_Register($dst$$reg);
3501 Register src_reg = as_Register($src1$$reg);
3502 int32_t con = (int32_t)$src2$$constant;
3503 // add has primary == 0, subtract has primary == 1
3504 if ($primary) { con = -con; }
3505 if (con < 0) {
3506 __ sub(dst_reg, src_reg, -con);
3507 } else {
3508 __ add(dst_reg, src_reg, con);
3509 }
3510 %}
3511
3512 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3513 Register dst_reg = as_Register($dst$$reg);
3514 Register src1_reg = as_Register($src1$$reg);
3515 Register src2_reg = as_Register($src2$$reg);
3516 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3517 %}
3518
3519 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3520 Register dst_reg = as_Register($dst$$reg);
3521 Register src1_reg = as_Register($src1$$reg);
3522 Register src2_reg = as_Register($src2$$reg);
3523 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3524 %}
3525
3526 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3527 Register dst_reg = as_Register($dst$$reg);
3528 Register src1_reg = as_Register($src1$$reg);
3529 Register src2_reg = as_Register($src2$$reg);
3530 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3531 %}
3532
3533 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3534 Register dst_reg = as_Register($dst$$reg);
3535 Register src1_reg = as_Register($src1$$reg);
3536 Register src2_reg = as_Register($src2$$reg);
3537 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3538 %}
3539
3540 // compare instruction encodings
3541
3542 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3543 Register reg1 = as_Register($src1$$reg);
3544 Register reg2 = as_Register($src2$$reg);
3545 __ cmpw(reg1, reg2);
3546 %}
3547
3548 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3549 Register reg = as_Register($src1$$reg);
3550 int32_t val = $src2$$constant;
3551 if (val >= 0) {
3552 __ subsw(zr, reg, val);
3553 } else {
3554 __ addsw(zr, reg, -val);
3555 }
3556 %}
3557
3558 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3559 Register reg1 = as_Register($src1$$reg);
3560 uint32_t val = (uint32_t)$src2$$constant;
3561 __ movw(rscratch1, val);
3562 __ cmpw(reg1, rscratch1);
3563 %}
3564
3565 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3566 Register reg1 = as_Register($src1$$reg);
3567 Register reg2 = as_Register($src2$$reg);
3568 __ cmp(reg1, reg2);
3569 %}
3570
3571 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3572 Register reg = as_Register($src1$$reg);
3573 int64_t val = $src2$$constant;
3574 if (val >= 0) {
3575 __ subs(zr, reg, val);
3576 } else if (val != -val) {
3577 __ adds(zr, reg, -val);
3578 } else {
3579 // aargh, Long.MIN_VALUE is a special case
3580 __ orr(rscratch1, zr, (uint64_t)val);
3581 __ subs(zr, reg, rscratch1);
3582 }
3583 %}
3584
3585 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3586 Register reg1 = as_Register($src1$$reg);
3587 uint64_t val = (uint64_t)$src2$$constant;
3588 __ mov(rscratch1, val);
3589 __ cmp(reg1, rscratch1);
3590 %}
3591
3592 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3593 Register reg1 = as_Register($src1$$reg);
3594 Register reg2 = as_Register($src2$$reg);
3595 __ cmp(reg1, reg2);
3596 %}
3597
3598 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3599 Register reg1 = as_Register($src1$$reg);
3600 Register reg2 = as_Register($src2$$reg);
3601 __ cmpw(reg1, reg2);
3602 %}
3603
3604 enc_class aarch64_enc_testp(iRegP src) %{
3605 Register reg = as_Register($src$$reg);
3606 __ cmp(reg, zr);
3607 %}
3608
3609 enc_class aarch64_enc_testn(iRegN src) %{
3610 Register reg = as_Register($src$$reg);
3611 __ cmpw(reg, zr);
3612 %}
3613
3614 enc_class aarch64_enc_b(label lbl) %{
3615 Label *L = $lbl$$label;
3616 __ b(*L);
3617 %}
3618
3619 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3620 Label *L = $lbl$$label;
3621 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3622 %}
3623
3624 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3625 Label *L = $lbl$$label;
3626 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3627 %}
3628
3629 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3630 %{
3631 Register sub_reg = as_Register($sub$$reg);
3632 Register super_reg = as_Register($super$$reg);
3633 Register temp_reg = as_Register($temp$$reg);
3634 Register result_reg = as_Register($result$$reg);
3635
3636 Label miss;
3637 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3638 nullptr, &miss,
3639 /*set_cond_codes:*/ true);
3640 if ($primary) {
3641 __ mov(result_reg, zr);
3642 }
3643 __ bind(miss);
3644 %}
3645
3646 enc_class aarch64_enc_java_static_call(method meth) %{
3647 address addr = (address)$meth$$method;
3648 address call;
3649 if (!_method) {
3650 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3651 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3652 if (call == nullptr) {
3653 ciEnv::current()->record_failure("CodeCache is full");
3654 return;
3655 }
3656 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3657 // The NOP here is purely to ensure that eliding a call to
3658 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3659 __ nop();
3660 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3661 } else {
3662 int method_index = resolved_method_index(masm);
3663 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3664 : static_call_Relocation::spec(method_index);
3665 call = __ trampoline_call(Address(addr, rspec));
3666 if (call == nullptr) {
3667 ciEnv::current()->record_failure("CodeCache is full");
3668 return;
3669 }
3670 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3671 // Calls of the same statically bound method can share
3672 // a stub to the interpreter.
3673 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3674 } else {
3675 // Emit stub for static call
3676 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3677 if (stub == nullptr) {
3678 ciEnv::current()->record_failure("CodeCache is full");
3679 return;
3680 }
3681 }
3682 }
3683
3684 __ post_call_nop();
3685
3686 // Only non uncommon_trap calls need to reinitialize ptrue.
3687 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3688 __ reinitialize_ptrue();
3689 }
3690 %}
3691
3692 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3693 int method_index = resolved_method_index(masm);
3694 address call = __ ic_call((address)$meth$$method, method_index);
3695 if (call == nullptr) {
3696 ciEnv::current()->record_failure("CodeCache is full");
3697 return;
3698 }
3699 __ post_call_nop();
3700 if (Compile::current()->max_vector_size() > 0) {
3701 __ reinitialize_ptrue();
3702 }
3703 %}
3704
3705 enc_class aarch64_enc_call_epilog() %{
3706 if (VerifyStackAtCalls) {
3707 // Check that stack depth is unchanged: find majik cookie on stack
3708 __ call_Unimplemented();
3709 }
3710 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
3711 // The last return value is not set by the callee but used to pass the null marker to compiled code.
3712 // Search for the corresponding projection, get the register and emit code that initialized it.
3713 uint con = (tf()->range_cc()->cnt() - 1);
3714 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3715 ProjNode* proj = fast_out(i)->as_Proj();
3716 if (proj->_con == con) {
3717 // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
3718 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3719 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3720 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3721 __ cmp(r0, zr);
3722 __ cset(toReg, Assembler::NE);
3723 if (reg->is_stack()) {
3724 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3725 __ str(toReg, Address(sp, st_off));
3726 }
3727 break;
3728 }
3729 }
3730 if (return_value_is_used()) {
3731 // An inline type is returned as fields in multiple registers.
3732 // R0 either contains an oop if the inline type is buffered or a pointer
3733 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3734 // if the lowest bit is set to allow C2 to use the oop after null checking.
3735 // r0 &= (r0 & 1) - 1
3736 __ andr(rscratch1, r0, 0x1);
3737 __ sub(rscratch1, rscratch1, 0x1);
3738 __ andr(r0, r0, rscratch1);
3739 }
3740 }
3741 %}
3742
3743 enc_class aarch64_enc_java_to_runtime(method meth) %{
3744 // some calls to generated routines (arraycopy code) are scheduled
3745 // by C2 as runtime calls. if so we can call them using a br (they
3746 // will be in a reachable segment) otherwise we have to use a blr
3747 // which loads the absolute address into a register.
3748 address entry = (address)$meth$$method;
3749 CodeBlob *cb = CodeCache::find_blob(entry);
3750 if (cb) {
3751 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3752 if (call == nullptr) {
3753 ciEnv::current()->record_failure("CodeCache is full");
3754 return;
3755 }
3756 __ post_call_nop();
3757 } else {
3758 Label retaddr;
3759 // Make the anchor frame walkable
3760 __ adr(rscratch2, retaddr);
3761 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3762 __ lea(rscratch1, RuntimeAddress(entry));
3763 __ blr(rscratch1);
3764 __ bind(retaddr);
3765 __ post_call_nop();
3766 }
3767 if (Compile::current()->max_vector_size() > 0) {
3768 __ reinitialize_ptrue();
3769 }
3770 %}
3771
3772 enc_class aarch64_enc_rethrow() %{
3773 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3774 %}
3775
3776 enc_class aarch64_enc_ret() %{
3777 #ifdef ASSERT
3778 if (Compile::current()->max_vector_size() > 0) {
3779 __ verify_ptrue();
3780 }
3781 #endif
3782 __ ret(lr);
3783 %}
3784
3785 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3786 Register target_reg = as_Register($jump_target$$reg);
3787 __ br(target_reg);
3788 %}
3789
3790 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3791 Register target_reg = as_Register($jump_target$$reg);
3792 // exception oop should be in r0
3793 // ret addr has been popped into lr
3794 // callee expects it in r3
3795 __ mov(r3, lr);
3796 __ br(target_reg);
3797 %}
3798
3799 %}
3800
3801 //----------FRAME--------------------------------------------------------------
3802 // Definition of frame structure and management information.
3803 //
3804 // S T A C K L A Y O U T Allocators stack-slot number
3805 // | (to get allocators register number
3806 // G Owned by | | v add OptoReg::stack0())
3807 // r CALLER | |
3808 // o | +--------+ pad to even-align allocators stack-slot
3809 // w V | pad0 | numbers; owned by CALLER
3810 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3811 // h ^ | in | 5
3812 // | | args | 4 Holes in incoming args owned by SELF
3813 // | | | | 3
3814 // | | +--------+
3815 // V | | old out| Empty on Intel, window on Sparc
3816 // | old |preserve| Must be even aligned.
3817 // | SP-+--------+----> Matcher::_old_SP, even aligned
3818 // | | in | 3 area for Intel ret address
3819 // Owned by |preserve| Empty on Sparc.
3820 // SELF +--------+
3821 // | | pad2 | 2 pad to align old SP
3822 // | +--------+ 1
3823 // | | locks | 0
3824 // | +--------+----> OptoReg::stack0(), even aligned
3825 // | | pad1 | 11 pad to align new SP
3826 // | +--------+
3827 // | | | 10
3828 // | | spills | 9 spills
3829 // V | | 8 (pad0 slot for callee)
3830 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3831 // ^ | out | 7
3832 // | | args | 6 Holes in outgoing args owned by CALLEE
3833 // Owned by +--------+
3834 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3835 // | new |preserve| Must be even-aligned.
3836 // | SP-+--------+----> Matcher::_new_SP, even aligned
3837 // | | |
3838 //
3839 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3840 // known from SELF's arguments and the Java calling convention.
3841 // Region 6-7 is determined per call site.
3842 // Note 2: If the calling convention leaves holes in the incoming argument
3843 // area, those holes are owned by SELF. Holes in the outgoing area
3844 // are owned by the CALLEE. Holes should not be necessary in the
3845 // incoming area, as the Java calling convention is completely under
3846 // the control of the AD file. Doubles can be sorted and packed to
3847 // avoid holes. Holes in the outgoing arguments may be necessary for
3848 // varargs C calling conventions.
3849 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3850 // even aligned with pad0 as needed.
3851 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3852 // (the latter is true on Intel but is it false on AArch64?)
3853 // region 6-11 is even aligned; it may be padded out more so that
3854 // the region from SP to FP meets the minimum stack alignment.
3855 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3856 // alignment. Region 11, pad1, may be dynamically extended so that
3857 // SP meets the minimum alignment.
3858
3859 frame %{
3860 // These three registers define part of the calling convention
3861 // between compiled code and the interpreter.
3862
3863 // Inline Cache Register or Method for I2C.
3864 inline_cache_reg(R12);
3865
3866 // Number of stack slots consumed by locking an object
3867 sync_stack_slots(2);
3868
3869 // Compiled code's Frame Pointer
3870 frame_pointer(R31);
3871
3872 // Interpreter stores its frame pointer in a register which is
3873 // stored to the stack by I2CAdaptors.
3874 // I2CAdaptors convert from interpreted java to compiled java.
3875 interpreter_frame_pointer(R29);
3876
3877 // Stack alignment requirement
3878 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3879
3880 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3881 // for calls to C. Supports the var-args backing area for register parms.
3882 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3883
3884 // The after-PROLOG location of the return address. Location of
3885 // return address specifies a type (REG or STACK) and a number
3886 // representing the register number (i.e. - use a register name) or
3887 // stack slot.
3888 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3889 // Otherwise, it is above the locks and verification slot and alignment word
3890 // TODO this may well be correct but need to check why that - 2 is there
3891 // ppc port uses 0 but we definitely need to allow for fixed_slots
3892 // which folds in the space used for monitors
3893 return_addr(STACK - 2 +
3894 align_up((Compile::current()->in_preserve_stack_slots() +
3895 Compile::current()->fixed_slots()),
3896 stack_alignment_in_slots()));
3897
3898 // Location of compiled Java return values. Same as C for now.
3899 return_value
3900 %{
3901 // TODO do we allow ideal_reg == Op_RegN???
3902 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3903 "only return normal values");
3904
3905 static const int lo[Op_RegL + 1] = { // enum name
3906 0, // Op_Node
3907 0, // Op_Set
3908 R0_num, // Op_RegN
3909 R0_num, // Op_RegI
3910 R0_num, // Op_RegP
3911 V0_num, // Op_RegF
3912 V0_num, // Op_RegD
3913 R0_num // Op_RegL
3914 };
3915
3916 static const int hi[Op_RegL + 1] = { // enum name
3917 0, // Op_Node
3918 0, // Op_Set
3919 OptoReg::Bad, // Op_RegN
3920 OptoReg::Bad, // Op_RegI
3921 R0_H_num, // Op_RegP
3922 OptoReg::Bad, // Op_RegF
3923 V0_H_num, // Op_RegD
3924 R0_H_num // Op_RegL
3925 };
3926
3927 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3928 %}
3929 %}
3930
3931 //----------ATTRIBUTES---------------------------------------------------------
3932 //----------Operand Attributes-------------------------------------------------
3933 op_attrib op_cost(1); // Required cost attribute
3934
3935 //----------Instruction Attributes---------------------------------------------
3936 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3937 ins_attrib ins_size(32); // Required size attribute (in bits)
3938 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3939 // a non-matching short branch variant
3940 // of some long branch?
3941 ins_attrib ins_alignment(4); // Required alignment attribute (must
3942 // be a power of 2) specifies the
3943 // alignment that some part of the
3944 // instruction (not necessarily the
3945 // start) requires. If > 1, a
3946 // compute_padding() function must be
3947 // provided for the instruction
3948
3949 //----------OPERANDS-----------------------------------------------------------
3950 // Operand definitions must precede instruction definitions for correct parsing
3951 // in the ADLC because operands constitute user defined types which are used in
3952 // instruction definitions.
3953
3954 //----------Simple Operands----------------------------------------------------
3955
3956 // Integer operands 32 bit
3957 // 32 bit immediate
3958 operand immI()
3959 %{
3960 match(ConI);
3961
3962 op_cost(0);
3963 format %{ %}
3964 interface(CONST_INTER);
3965 %}
3966
3967 // 32 bit zero
3968 operand immI0()
3969 %{
3970 predicate(n->get_int() == 0);
3971 match(ConI);
3972
3973 op_cost(0);
3974 format %{ %}
3975 interface(CONST_INTER);
3976 %}
3977
3978 // 32 bit unit increment
3979 operand immI_1()
3980 %{
3981 predicate(n->get_int() == 1);
3982 match(ConI);
3983
3984 op_cost(0);
3985 format %{ %}
3986 interface(CONST_INTER);
3987 %}
3988
3989 // 32 bit unit decrement
3990 operand immI_M1()
3991 %{
3992 predicate(n->get_int() == -1);
3993 match(ConI);
3994
3995 op_cost(0);
3996 format %{ %}
3997 interface(CONST_INTER);
3998 %}
3999
4000 // Shift values for add/sub extension shift
4001 operand immIExt()
4002 %{
4003 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4004 match(ConI);
4005
4006 op_cost(0);
4007 format %{ %}
4008 interface(CONST_INTER);
4009 %}
4010
4011 operand immI_gt_1()
4012 %{
4013 predicate(n->get_int() > 1);
4014 match(ConI);
4015
4016 op_cost(0);
4017 format %{ %}
4018 interface(CONST_INTER);
4019 %}
4020
4021 operand immI_le_4()
4022 %{
4023 predicate(n->get_int() <= 4);
4024 match(ConI);
4025
4026 op_cost(0);
4027 format %{ %}
4028 interface(CONST_INTER);
4029 %}
4030
4031 operand immI_16()
4032 %{
4033 predicate(n->get_int() == 16);
4034 match(ConI);
4035
4036 op_cost(0);
4037 format %{ %}
4038 interface(CONST_INTER);
4039 %}
4040
4041 operand immI_24()
4042 %{
4043 predicate(n->get_int() == 24);
4044 match(ConI);
4045
4046 op_cost(0);
4047 format %{ %}
4048 interface(CONST_INTER);
4049 %}
4050
4051 operand immI_32()
4052 %{
4053 predicate(n->get_int() == 32);
4054 match(ConI);
4055
4056 op_cost(0);
4057 format %{ %}
4058 interface(CONST_INTER);
4059 %}
4060
4061 operand immI_48()
4062 %{
4063 predicate(n->get_int() == 48);
4064 match(ConI);
4065
4066 op_cost(0);
4067 format %{ %}
4068 interface(CONST_INTER);
4069 %}
4070
4071 operand immI_56()
4072 %{
4073 predicate(n->get_int() == 56);
4074 match(ConI);
4075
4076 op_cost(0);
4077 format %{ %}
4078 interface(CONST_INTER);
4079 %}
4080
4081 operand immI_255()
4082 %{
4083 predicate(n->get_int() == 255);
4084 match(ConI);
4085
4086 op_cost(0);
4087 format %{ %}
4088 interface(CONST_INTER);
4089 %}
4090
4091 operand immI_65535()
4092 %{
4093 predicate(n->get_int() == 65535);
4094 match(ConI);
4095
4096 op_cost(0);
4097 format %{ %}
4098 interface(CONST_INTER);
4099 %}
4100
4101 operand immI_positive()
4102 %{
4103 predicate(n->get_int() > 0);
4104 match(ConI);
4105
4106 op_cost(0);
4107 format %{ %}
4108 interface(CONST_INTER);
4109 %}
4110
4111 // BoolTest condition for signed compare
4112 operand immI_cmp_cond()
4113 %{
4114 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4115 match(ConI);
4116
4117 op_cost(0);
4118 format %{ %}
4119 interface(CONST_INTER);
4120 %}
4121
4122 // BoolTest condition for unsigned compare
4123 operand immI_cmpU_cond()
4124 %{
4125 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4126 match(ConI);
4127
4128 op_cost(0);
4129 format %{ %}
4130 interface(CONST_INTER);
4131 %}
4132
4133 operand immL_255()
4134 %{
4135 predicate(n->get_long() == 255L);
4136 match(ConL);
4137
4138 op_cost(0);
4139 format %{ %}
4140 interface(CONST_INTER);
4141 %}
4142
4143 operand immL_65535()
4144 %{
4145 predicate(n->get_long() == 65535L);
4146 match(ConL);
4147
4148 op_cost(0);
4149 format %{ %}
4150 interface(CONST_INTER);
4151 %}
4152
4153 operand immL_4294967295()
4154 %{
4155 predicate(n->get_long() == 4294967295L);
4156 match(ConL);
4157
4158 op_cost(0);
4159 format %{ %}
4160 interface(CONST_INTER);
4161 %}
4162
4163 operand immL_bitmask()
4164 %{
4165 predicate((n->get_long() != 0)
4166 && ((n->get_long() & 0xc000000000000000l) == 0)
4167 && is_power_of_2(n->get_long() + 1));
4168 match(ConL);
4169
4170 op_cost(0);
4171 format %{ %}
4172 interface(CONST_INTER);
4173 %}
4174
4175 operand immI_bitmask()
4176 %{
4177 predicate((n->get_int() != 0)
4178 && ((n->get_int() & 0xc0000000) == 0)
4179 && is_power_of_2(n->get_int() + 1));
4180 match(ConI);
4181
4182 op_cost(0);
4183 format %{ %}
4184 interface(CONST_INTER);
4185 %}
4186
4187 operand immL_positive_bitmaskI()
4188 %{
4189 predicate((n->get_long() != 0)
4190 && ((julong)n->get_long() < 0x80000000ULL)
4191 && is_power_of_2(n->get_long() + 1));
4192 match(ConL);
4193
4194 op_cost(0);
4195 format %{ %}
4196 interface(CONST_INTER);
4197 %}
4198
4199 // Scale values for scaled offset addressing modes (up to long but not quad)
4200 operand immIScale()
4201 %{
4202 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4203 match(ConI);
4204
4205 op_cost(0);
4206 format %{ %}
4207 interface(CONST_INTER);
4208 %}
4209
4210 // 5 bit signed integer
4211 operand immI5()
4212 %{
4213 predicate(Assembler::is_simm(n->get_int(), 5));
4214 match(ConI);
4215
4216 op_cost(0);
4217 format %{ %}
4218 interface(CONST_INTER);
4219 %}
4220
4221 // 7 bit unsigned integer
4222 operand immIU7()
4223 %{
4224 predicate(Assembler::is_uimm(n->get_int(), 7));
4225 match(ConI);
4226
4227 op_cost(0);
4228 format %{ %}
4229 interface(CONST_INTER);
4230 %}
4231
4232 // Offset for scaled or unscaled immediate loads and stores
4233 operand immIOffset()
4234 %{
4235 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4236 match(ConI);
4237
4238 op_cost(0);
4239 format %{ %}
4240 interface(CONST_INTER);
4241 %}
4242
4243 operand immIOffset1()
4244 %{
4245 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4246 match(ConI);
4247
4248 op_cost(0);
4249 format %{ %}
4250 interface(CONST_INTER);
4251 %}
4252
4253 operand immIOffset2()
4254 %{
4255 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4256 match(ConI);
4257
4258 op_cost(0);
4259 format %{ %}
4260 interface(CONST_INTER);
4261 %}
4262
4263 operand immIOffset4()
4264 %{
4265 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4266 match(ConI);
4267
4268 op_cost(0);
4269 format %{ %}
4270 interface(CONST_INTER);
4271 %}
4272
4273 operand immIOffset8()
4274 %{
4275 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4276 match(ConI);
4277
4278 op_cost(0);
4279 format %{ %}
4280 interface(CONST_INTER);
4281 %}
4282
4283 operand immIOffset16()
4284 %{
4285 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4286 match(ConI);
4287
4288 op_cost(0);
4289 format %{ %}
4290 interface(CONST_INTER);
4291 %}
4292
4293 operand immLOffset()
4294 %{
4295 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4296 match(ConL);
4297
4298 op_cost(0);
4299 format %{ %}
4300 interface(CONST_INTER);
4301 %}
4302
4303 operand immLoffset1()
4304 %{
4305 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4306 match(ConL);
4307
4308 op_cost(0);
4309 format %{ %}
4310 interface(CONST_INTER);
4311 %}
4312
4313 operand immLoffset2()
4314 %{
4315 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4316 match(ConL);
4317
4318 op_cost(0);
4319 format %{ %}
4320 interface(CONST_INTER);
4321 %}
4322
4323 operand immLoffset4()
4324 %{
4325 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4326 match(ConL);
4327
4328 op_cost(0);
4329 format %{ %}
4330 interface(CONST_INTER);
4331 %}
4332
4333 operand immLoffset8()
4334 %{
4335 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4336 match(ConL);
4337
4338 op_cost(0);
4339 format %{ %}
4340 interface(CONST_INTER);
4341 %}
4342
4343 operand immLoffset16()
4344 %{
4345 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4346 match(ConL);
4347
4348 op_cost(0);
4349 format %{ %}
4350 interface(CONST_INTER);
4351 %}
4352
4353 // 5 bit signed long integer
4354 operand immL5()
4355 %{
4356 predicate(Assembler::is_simm(n->get_long(), 5));
4357 match(ConL);
4358
4359 op_cost(0);
4360 format %{ %}
4361 interface(CONST_INTER);
4362 %}
4363
4364 // 7 bit unsigned long integer
4365 operand immLU7()
4366 %{
4367 predicate(Assembler::is_uimm(n->get_long(), 7));
4368 match(ConL);
4369
4370 op_cost(0);
4371 format %{ %}
4372 interface(CONST_INTER);
4373 %}
4374
4375 // 8 bit signed value.
4376 operand immI8()
4377 %{
4378 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4379 match(ConI);
4380
4381 op_cost(0);
4382 format %{ %}
4383 interface(CONST_INTER);
4384 %}
4385
4386 // 8 bit signed value (simm8), or #simm8 LSL 8.
4387 operand immI8_shift8()
4388 %{
4389 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4390 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4391 match(ConI);
4392
4393 op_cost(0);
4394 format %{ %}
4395 interface(CONST_INTER);
4396 %}
4397
4398 // 8 bit signed value (simm8), or #simm8 LSL 8.
4399 operand immL8_shift8()
4400 %{
4401 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4402 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4403 match(ConL);
4404
4405 op_cost(0);
4406 format %{ %}
4407 interface(CONST_INTER);
4408 %}
4409
4410 // 8 bit integer valid for vector add sub immediate
4411 operand immBAddSubV()
4412 %{
4413 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4414 match(ConI);
4415
4416 op_cost(0);
4417 format %{ %}
4418 interface(CONST_INTER);
4419 %}
4420
4421 // 32 bit integer valid for add sub immediate
4422 operand immIAddSub()
4423 %{
4424 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4425 match(ConI);
4426 op_cost(0);
4427 format %{ %}
4428 interface(CONST_INTER);
4429 %}
4430
4431 // 32 bit integer valid for vector add sub immediate
4432 operand immIAddSubV()
4433 %{
4434 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4435 match(ConI);
4436
4437 op_cost(0);
4438 format %{ %}
4439 interface(CONST_INTER);
4440 %}
4441
4442 // 32 bit unsigned integer valid for logical immediate
4443
4444 operand immBLog()
4445 %{
4446 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4447 match(ConI);
4448
4449 op_cost(0);
4450 format %{ %}
4451 interface(CONST_INTER);
4452 %}
4453
4454 operand immSLog()
4455 %{
4456 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4457 match(ConI);
4458
4459 op_cost(0);
4460 format %{ %}
4461 interface(CONST_INTER);
4462 %}
4463
4464 operand immILog()
4465 %{
4466 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4467 match(ConI);
4468
4469 op_cost(0);
4470 format %{ %}
4471 interface(CONST_INTER);
4472 %}
4473
4474 // Integer operands 64 bit
4475 // 64 bit immediate
4476 operand immL()
4477 %{
4478 match(ConL);
4479
4480 op_cost(0);
4481 format %{ %}
4482 interface(CONST_INTER);
4483 %}
4484
4485 // 64 bit zero
4486 operand immL0()
4487 %{
4488 predicate(n->get_long() == 0);
4489 match(ConL);
4490
4491 op_cost(0);
4492 format %{ %}
4493 interface(CONST_INTER);
4494 %}
4495
4496 // 64 bit unit decrement
4497 operand immL_M1()
4498 %{
4499 predicate(n->get_long() == -1);
4500 match(ConL);
4501
4502 op_cost(0);
4503 format %{ %}
4504 interface(CONST_INTER);
4505 %}
4506
4507 // 64 bit integer valid for add sub immediate
4508 operand immLAddSub()
4509 %{
4510 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4511 match(ConL);
4512 op_cost(0);
4513 format %{ %}
4514 interface(CONST_INTER);
4515 %}
4516
4517 // 64 bit integer valid for addv subv immediate
4518 operand immLAddSubV()
4519 %{
4520 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4521 match(ConL);
4522
4523 op_cost(0);
4524 format %{ %}
4525 interface(CONST_INTER);
4526 %}
4527
4528 // 64 bit integer valid for logical immediate
4529 operand immLLog()
4530 %{
4531 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4532 match(ConL);
4533 op_cost(0);
4534 format %{ %}
4535 interface(CONST_INTER);
4536 %}
4537
4538 // Long Immediate: low 32-bit mask
4539 operand immL_32bits()
4540 %{
4541 predicate(n->get_long() == 0xFFFFFFFFL);
4542 match(ConL);
4543 op_cost(0);
4544 format %{ %}
4545 interface(CONST_INTER);
4546 %}
4547
4548 // Pointer operands
4549 // Pointer Immediate
4550 operand immP()
4551 %{
4552 match(ConP);
4553
4554 op_cost(0);
4555 format %{ %}
4556 interface(CONST_INTER);
4557 %}
4558
4559 // nullptr Pointer Immediate
4560 operand immP0()
4561 %{
4562 predicate(n->get_ptr() == 0);
4563 match(ConP);
4564
4565 op_cost(0);
4566 format %{ %}
4567 interface(CONST_INTER);
4568 %}
4569
4570 // Pointer Immediate One
4571 // this is used in object initialization (initial object header)
4572 operand immP_1()
4573 %{
4574 predicate(n->get_ptr() == 1);
4575 match(ConP);
4576
4577 op_cost(0);
4578 format %{ %}
4579 interface(CONST_INTER);
4580 %}
4581
4582 // Card Table Byte Map Base
4583 operand immByteMapBase()
4584 %{
4585 // Get base of card map
4586 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4587 SHENANDOAHGC_ONLY(!BarrierSet::barrier_set()->is_a(BarrierSet::ShenandoahBarrierSet) &&)
4588 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4589 match(ConP);
4590
4591 op_cost(0);
4592 format %{ %}
4593 interface(CONST_INTER);
4594 %}
4595
4596 // Float and Double operands
4597 // Double Immediate
4598 operand immD()
4599 %{
4600 match(ConD);
4601 op_cost(0);
4602 format %{ %}
4603 interface(CONST_INTER);
4604 %}
4605
4606 // Double Immediate: +0.0d
4607 operand immD0()
4608 %{
4609 predicate(jlong_cast(n->getd()) == 0);
4610 match(ConD);
4611
4612 op_cost(0);
4613 format %{ %}
4614 interface(CONST_INTER);
4615 %}
4616
4617 // constant 'double +0.0'.
4618 operand immDPacked()
4619 %{
4620 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4621 match(ConD);
4622 op_cost(0);
4623 format %{ %}
4624 interface(CONST_INTER);
4625 %}
4626
4627 // Float Immediate
4628 operand immF()
4629 %{
4630 match(ConF);
4631 op_cost(0);
4632 format %{ %}
4633 interface(CONST_INTER);
4634 %}
4635
4636 // Float Immediate: +0.0f.
4637 operand immF0()
4638 %{
4639 predicate(jint_cast(n->getf()) == 0);
4640 match(ConF);
4641
4642 op_cost(0);
4643 format %{ %}
4644 interface(CONST_INTER);
4645 %}
4646
4647 // Half Float (FP16) Immediate
4648 operand immH()
4649 %{
4650 match(ConH);
4651 op_cost(0);
4652 format %{ %}
4653 interface(CONST_INTER);
4654 %}
4655
4656 //
4657 operand immFPacked()
4658 %{
4659 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4660 match(ConF);
4661 op_cost(0);
4662 format %{ %}
4663 interface(CONST_INTER);
4664 %}
4665
4666 // Narrow pointer operands
4667 // Narrow Pointer Immediate
4668 operand immN()
4669 %{
4670 match(ConN);
4671
4672 op_cost(0);
4673 format %{ %}
4674 interface(CONST_INTER);
4675 %}
4676
4677 // Narrow nullptr Pointer Immediate
4678 operand immN0()
4679 %{
4680 predicate(n->get_narrowcon() == 0);
4681 match(ConN);
4682
4683 op_cost(0);
4684 format %{ %}
4685 interface(CONST_INTER);
4686 %}
4687
4688 operand immNKlass()
4689 %{
4690 match(ConNKlass);
4691
4692 op_cost(0);
4693 format %{ %}
4694 interface(CONST_INTER);
4695 %}
4696
4697 // Integer 32 bit Register Operands
4698 // Integer 32 bitRegister (excludes SP)
4699 operand iRegI()
4700 %{
4701 constraint(ALLOC_IN_RC(any_reg32));
4702 match(RegI);
4703 match(iRegINoSp);
4704 op_cost(0);
4705 format %{ %}
4706 interface(REG_INTER);
4707 %}
4708
4709 // Integer 32 bit Register not Special
4710 operand iRegINoSp()
4711 %{
4712 constraint(ALLOC_IN_RC(no_special_reg32));
4713 match(RegI);
4714 op_cost(0);
4715 format %{ %}
4716 interface(REG_INTER);
4717 %}
4718
4719 // Integer 64 bit Register Operands
4720 // Integer 64 bit Register (includes SP)
4721 operand iRegL()
4722 %{
4723 constraint(ALLOC_IN_RC(any_reg));
4724 match(RegL);
4725 match(iRegLNoSp);
4726 op_cost(0);
4727 format %{ %}
4728 interface(REG_INTER);
4729 %}
4730
4731 // Integer 64 bit Register not Special
4732 operand iRegLNoSp()
4733 %{
4734 constraint(ALLOC_IN_RC(no_special_reg));
4735 match(RegL);
4736 match(iRegL_R0);
4737 format %{ %}
4738 interface(REG_INTER);
4739 %}
4740
4741 // Pointer Register Operands
4742 // Pointer Register
4743 operand iRegP()
4744 %{
4745 constraint(ALLOC_IN_RC(ptr_reg));
4746 match(RegP);
4747 match(iRegPNoSp);
4748 match(iRegP_R0);
4749 //match(iRegP_R2);
4750 //match(iRegP_R4);
4751 match(iRegP_R5);
4752 match(thread_RegP);
4753 op_cost(0);
4754 format %{ %}
4755 interface(REG_INTER);
4756 %}
4757
4758 // Pointer 64 bit Register not Special
4759 operand iRegPNoSp()
4760 %{
4761 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4762 match(RegP);
4763 // match(iRegP);
4764 // match(iRegP_R0);
4765 // match(iRegP_R2);
4766 // match(iRegP_R4);
4767 // match(iRegP_R5);
4768 // match(thread_RegP);
4769 op_cost(0);
4770 format %{ %}
4771 interface(REG_INTER);
4772 %}
4773
4774 // This operand is not allowed to use rfp even if
4775 // rfp is not used to hold the frame pointer.
4776 operand iRegPNoSpNoRfp()
4777 %{
4778 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4779 match(RegP);
4780 match(iRegPNoSp);
4781 op_cost(0);
4782 format %{ %}
4783 interface(REG_INTER);
4784 %}
4785
4786 // Pointer 64 bit Register R0 only
4787 operand iRegP_R0()
4788 %{
4789 constraint(ALLOC_IN_RC(r0_reg));
4790 match(RegP);
4791 // match(iRegP);
4792 match(iRegPNoSp);
4793 op_cost(0);
4794 format %{ %}
4795 interface(REG_INTER);
4796 %}
4797
4798 // Pointer 64 bit Register R1 only
4799 operand iRegP_R1()
4800 %{
4801 constraint(ALLOC_IN_RC(r1_reg));
4802 match(RegP);
4803 // match(iRegP);
4804 match(iRegPNoSp);
4805 op_cost(0);
4806 format %{ %}
4807 interface(REG_INTER);
4808 %}
4809
4810 // Pointer 64 bit Register R2 only
4811 operand iRegP_R2()
4812 %{
4813 constraint(ALLOC_IN_RC(r2_reg));
4814 match(RegP);
4815 // match(iRegP);
4816 match(iRegPNoSp);
4817 op_cost(0);
4818 format %{ %}
4819 interface(REG_INTER);
4820 %}
4821
4822 // Pointer 64 bit Register R3 only
4823 operand iRegP_R3()
4824 %{
4825 constraint(ALLOC_IN_RC(r3_reg));
4826 match(RegP);
4827 // match(iRegP);
4828 match(iRegPNoSp);
4829 op_cost(0);
4830 format %{ %}
4831 interface(REG_INTER);
4832 %}
4833
4834 // Pointer 64 bit Register R4 only
4835 operand iRegP_R4()
4836 %{
4837 constraint(ALLOC_IN_RC(r4_reg));
4838 match(RegP);
4839 // match(iRegP);
4840 match(iRegPNoSp);
4841 op_cost(0);
4842 format %{ %}
4843 interface(REG_INTER);
4844 %}
4845
4846 // Pointer 64 bit Register R5 only
4847 operand iRegP_R5()
4848 %{
4849 constraint(ALLOC_IN_RC(r5_reg));
4850 match(RegP);
4851 // match(iRegP);
4852 match(iRegPNoSp);
4853 op_cost(0);
4854 format %{ %}
4855 interface(REG_INTER);
4856 %}
4857
4858 // Pointer 64 bit Register R10 only
4859 operand iRegP_R10()
4860 %{
4861 constraint(ALLOC_IN_RC(r10_reg));
4862 match(RegP);
4863 // match(iRegP);
4864 match(iRegPNoSp);
4865 op_cost(0);
4866 format %{ %}
4867 interface(REG_INTER);
4868 %}
4869
4870 // Long 64 bit Register R0 only
4871 operand iRegL_R0()
4872 %{
4873 constraint(ALLOC_IN_RC(r0_reg));
4874 match(RegL);
4875 match(iRegLNoSp);
4876 op_cost(0);
4877 format %{ %}
4878 interface(REG_INTER);
4879 %}
4880
4881 // Long 64 bit Register R11 only
4882 operand iRegL_R11()
4883 %{
4884 constraint(ALLOC_IN_RC(r11_reg));
4885 match(RegL);
4886 match(iRegLNoSp);
4887 op_cost(0);
4888 format %{ %}
4889 interface(REG_INTER);
4890 %}
4891
4892 // Register R0 only
4893 operand iRegI_R0()
4894 %{
4895 constraint(ALLOC_IN_RC(int_r0_reg));
4896 match(RegI);
4897 match(iRegINoSp);
4898 op_cost(0);
4899 format %{ %}
4900 interface(REG_INTER);
4901 %}
4902
4903 // Register R2 only
4904 operand iRegI_R2()
4905 %{
4906 constraint(ALLOC_IN_RC(int_r2_reg));
4907 match(RegI);
4908 match(iRegINoSp);
4909 op_cost(0);
4910 format %{ %}
4911 interface(REG_INTER);
4912 %}
4913
4914 // Register R3 only
4915 operand iRegI_R3()
4916 %{
4917 constraint(ALLOC_IN_RC(int_r3_reg));
4918 match(RegI);
4919 match(iRegINoSp);
4920 op_cost(0);
4921 format %{ %}
4922 interface(REG_INTER);
4923 %}
4924
4925
4926 // Register R4 only
4927 operand iRegI_R4()
4928 %{
4929 constraint(ALLOC_IN_RC(int_r4_reg));
4930 match(RegI);
4931 match(iRegINoSp);
4932 op_cost(0);
4933 format %{ %}
4934 interface(REG_INTER);
4935 %}
4936
4937
4938 // Pointer Register Operands
4939 // Narrow Pointer Register
4940 operand iRegN()
4941 %{
4942 constraint(ALLOC_IN_RC(any_reg32));
4943 match(RegN);
4944 match(iRegNNoSp);
4945 op_cost(0);
4946 format %{ %}
4947 interface(REG_INTER);
4948 %}
4949
4950 // Integer 64 bit Register not Special
4951 operand iRegNNoSp()
4952 %{
4953 constraint(ALLOC_IN_RC(no_special_reg32));
4954 match(RegN);
4955 op_cost(0);
4956 format %{ %}
4957 interface(REG_INTER);
4958 %}
4959
4960 // Float Register
4961 // Float register operands
4962 operand vRegF()
4963 %{
4964 constraint(ALLOC_IN_RC(float_reg));
4965 match(RegF);
4966
4967 op_cost(0);
4968 format %{ %}
4969 interface(REG_INTER);
4970 %}
4971
4972 // Double Register
4973 // Double register operands
4974 operand vRegD()
4975 %{
4976 constraint(ALLOC_IN_RC(double_reg));
4977 match(RegD);
4978
4979 op_cost(0);
4980 format %{ %}
4981 interface(REG_INTER);
4982 %}
4983
4984 // Generic vector class. This will be used for
4985 // all vector operands, including NEON and SVE.
4986 operand vReg()
4987 %{
4988 constraint(ALLOC_IN_RC(dynamic));
4989 match(VecA);
4990 match(VecD);
4991 match(VecX);
4992
4993 op_cost(0);
4994 format %{ %}
4995 interface(REG_INTER);
4996 %}
4997
4998 operand vecA()
4999 %{
5000 constraint(ALLOC_IN_RC(vectora_reg));
5001 match(VecA);
5002
5003 op_cost(0);
5004 format %{ %}
5005 interface(REG_INTER);
5006 %}
5007
5008 operand vecD()
5009 %{
5010 constraint(ALLOC_IN_RC(vectord_reg));
5011 match(VecD);
5012
5013 op_cost(0);
5014 format %{ %}
5015 interface(REG_INTER);
5016 %}
5017
5018 operand vecX()
5019 %{
5020 constraint(ALLOC_IN_RC(vectorx_reg));
5021 match(VecX);
5022
5023 op_cost(0);
5024 format %{ %}
5025 interface(REG_INTER);
5026 %}
5027
5028 operand vRegD_V0()
5029 %{
5030 constraint(ALLOC_IN_RC(v0_reg));
5031 match(RegD);
5032 op_cost(0);
5033 format %{ %}
5034 interface(REG_INTER);
5035 %}
5036
5037 operand vRegD_V1()
5038 %{
5039 constraint(ALLOC_IN_RC(v1_reg));
5040 match(RegD);
5041 op_cost(0);
5042 format %{ %}
5043 interface(REG_INTER);
5044 %}
5045
5046 operand vRegD_V2()
5047 %{
5048 constraint(ALLOC_IN_RC(v2_reg));
5049 match(RegD);
5050 op_cost(0);
5051 format %{ %}
5052 interface(REG_INTER);
5053 %}
5054
5055 operand vRegD_V3()
5056 %{
5057 constraint(ALLOC_IN_RC(v3_reg));
5058 match(RegD);
5059 op_cost(0);
5060 format %{ %}
5061 interface(REG_INTER);
5062 %}
5063
5064 operand vRegD_V4()
5065 %{
5066 constraint(ALLOC_IN_RC(v4_reg));
5067 match(RegD);
5068 op_cost(0);
5069 format %{ %}
5070 interface(REG_INTER);
5071 %}
5072
5073 operand vRegD_V5()
5074 %{
5075 constraint(ALLOC_IN_RC(v5_reg));
5076 match(RegD);
5077 op_cost(0);
5078 format %{ %}
5079 interface(REG_INTER);
5080 %}
5081
5082 operand vRegD_V6()
5083 %{
5084 constraint(ALLOC_IN_RC(v6_reg));
5085 match(RegD);
5086 op_cost(0);
5087 format %{ %}
5088 interface(REG_INTER);
5089 %}
5090
5091 operand vRegD_V7()
5092 %{
5093 constraint(ALLOC_IN_RC(v7_reg));
5094 match(RegD);
5095 op_cost(0);
5096 format %{ %}
5097 interface(REG_INTER);
5098 %}
5099
5100 operand vRegD_V12()
5101 %{
5102 constraint(ALLOC_IN_RC(v12_reg));
5103 match(RegD);
5104 op_cost(0);
5105 format %{ %}
5106 interface(REG_INTER);
5107 %}
5108
5109 operand vRegD_V13()
5110 %{
5111 constraint(ALLOC_IN_RC(v13_reg));
5112 match(RegD);
5113 op_cost(0);
5114 format %{ %}
5115 interface(REG_INTER);
5116 %}
5117
5118 operand pReg()
5119 %{
5120 constraint(ALLOC_IN_RC(pr_reg));
5121 match(RegVectMask);
5122 match(pRegGov);
5123 op_cost(0);
5124 format %{ %}
5125 interface(REG_INTER);
5126 %}
5127
5128 operand pRegGov()
5129 %{
5130 constraint(ALLOC_IN_RC(gov_pr));
5131 match(RegVectMask);
5132 match(pReg);
5133 op_cost(0);
5134 format %{ %}
5135 interface(REG_INTER);
5136 %}
5137
5138 operand pRegGov_P0()
5139 %{
5140 constraint(ALLOC_IN_RC(p0_reg));
5141 match(RegVectMask);
5142 op_cost(0);
5143 format %{ %}
5144 interface(REG_INTER);
5145 %}
5146
5147 operand pRegGov_P1()
5148 %{
5149 constraint(ALLOC_IN_RC(p1_reg));
5150 match(RegVectMask);
5151 op_cost(0);
5152 format %{ %}
5153 interface(REG_INTER);
5154 %}
5155
5156 // Flags register, used as output of signed compare instructions
5157
5158 // note that on AArch64 we also use this register as the output for
5159 // for floating point compare instructions (CmpF CmpD). this ensures
5160 // that ordered inequality tests use GT, GE, LT or LE none of which
5161 // pass through cases where the result is unordered i.e. one or both
5162 // inputs to the compare is a NaN. this means that the ideal code can
5163 // replace e.g. a GT with an LE and not end up capturing the NaN case
5164 // (where the comparison should always fail). EQ and NE tests are
5165 // always generated in ideal code so that unordered folds into the NE
5166 // case, matching the behaviour of AArch64 NE.
5167 //
5168 // This differs from x86 where the outputs of FP compares use a
5169 // special FP flags registers and where compares based on this
5170 // register are distinguished into ordered inequalities (cmpOpUCF) and
5171 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5172 // to explicitly handle the unordered case in branches. x86 also has
5173 // to include extra CMoveX rules to accept a cmpOpUCF input.
5174
5175 operand rFlagsReg()
5176 %{
5177 constraint(ALLOC_IN_RC(int_flags));
5178 match(RegFlags);
5179
5180 op_cost(0);
5181 format %{ "RFLAGS" %}
5182 interface(REG_INTER);
5183 %}
5184
5185 // Flags register, used as output of unsigned compare instructions
5186 operand rFlagsRegU()
5187 %{
5188 constraint(ALLOC_IN_RC(int_flags));
5189 match(RegFlags);
5190
5191 op_cost(0);
5192 format %{ "RFLAGSU" %}
5193 interface(REG_INTER);
5194 %}
5195
5196 // Special Registers
5197
5198 // Method Register
5199 operand inline_cache_RegP(iRegP reg)
5200 %{
5201 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5202 match(reg);
5203 match(iRegPNoSp);
5204 op_cost(0);
5205 format %{ %}
5206 interface(REG_INTER);
5207 %}
5208
5209 // Thread Register
5210 operand thread_RegP(iRegP reg)
5211 %{
5212 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5213 match(reg);
5214 op_cost(0);
5215 format %{ %}
5216 interface(REG_INTER);
5217 %}
5218
5219 //----------Memory Operands----------------------------------------------------
5220
5221 operand indirect(iRegP reg)
5222 %{
5223 constraint(ALLOC_IN_RC(ptr_reg));
5224 match(reg);
5225 op_cost(0);
5226 format %{ "[$reg]" %}
5227 interface(MEMORY_INTER) %{
5228 base($reg);
5229 index(0xffffffff);
5230 scale(0x0);
5231 disp(0x0);
5232 %}
5233 %}
5234
5235 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5236 %{
5237 constraint(ALLOC_IN_RC(ptr_reg));
5238 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5239 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5240 op_cost(0);
5241 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5242 interface(MEMORY_INTER) %{
5243 base($reg);
5244 index($ireg);
5245 scale($scale);
5246 disp(0x0);
5247 %}
5248 %}
5249
5250 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5251 %{
5252 constraint(ALLOC_IN_RC(ptr_reg));
5253 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5254 match(AddP reg (LShiftL lreg scale));
5255 op_cost(0);
5256 format %{ "$reg, $lreg lsl($scale)" %}
5257 interface(MEMORY_INTER) %{
5258 base($reg);
5259 index($lreg);
5260 scale($scale);
5261 disp(0x0);
5262 %}
5263 %}
5264
5265 operand indIndexI2L(iRegP reg, iRegI ireg)
5266 %{
5267 constraint(ALLOC_IN_RC(ptr_reg));
5268 match(AddP reg (ConvI2L ireg));
5269 op_cost(0);
5270 format %{ "$reg, $ireg, 0, I2L" %}
5271 interface(MEMORY_INTER) %{
5272 base($reg);
5273 index($ireg);
5274 scale(0x0);
5275 disp(0x0);
5276 %}
5277 %}
5278
5279 operand indIndex(iRegP reg, iRegL lreg)
5280 %{
5281 constraint(ALLOC_IN_RC(ptr_reg));
5282 match(AddP reg lreg);
5283 op_cost(0);
5284 format %{ "$reg, $lreg" %}
5285 interface(MEMORY_INTER) %{
5286 base($reg);
5287 index($lreg);
5288 scale(0x0);
5289 disp(0x0);
5290 %}
5291 %}
5292
5293 operand indOffI1(iRegP reg, immIOffset1 off)
5294 %{
5295 constraint(ALLOC_IN_RC(ptr_reg));
5296 match(AddP reg off);
5297 op_cost(0);
5298 format %{ "[$reg, $off]" %}
5299 interface(MEMORY_INTER) %{
5300 base($reg);
5301 index(0xffffffff);
5302 scale(0x0);
5303 disp($off);
5304 %}
5305 %}
5306
5307 operand indOffI2(iRegP reg, immIOffset2 off)
5308 %{
5309 constraint(ALLOC_IN_RC(ptr_reg));
5310 match(AddP reg off);
5311 op_cost(0);
5312 format %{ "[$reg, $off]" %}
5313 interface(MEMORY_INTER) %{
5314 base($reg);
5315 index(0xffffffff);
5316 scale(0x0);
5317 disp($off);
5318 %}
5319 %}
5320
5321 operand indOffI4(iRegP reg, immIOffset4 off)
5322 %{
5323 constraint(ALLOC_IN_RC(ptr_reg));
5324 match(AddP reg off);
5325 op_cost(0);
5326 format %{ "[$reg, $off]" %}
5327 interface(MEMORY_INTER) %{
5328 base($reg);
5329 index(0xffffffff);
5330 scale(0x0);
5331 disp($off);
5332 %}
5333 %}
5334
5335 operand indOffI8(iRegP reg, immIOffset8 off)
5336 %{
5337 constraint(ALLOC_IN_RC(ptr_reg));
5338 match(AddP reg off);
5339 op_cost(0);
5340 format %{ "[$reg, $off]" %}
5341 interface(MEMORY_INTER) %{
5342 base($reg);
5343 index(0xffffffff);
5344 scale(0x0);
5345 disp($off);
5346 %}
5347 %}
5348
5349 operand indOffI16(iRegP reg, immIOffset16 off)
5350 %{
5351 constraint(ALLOC_IN_RC(ptr_reg));
5352 match(AddP reg off);
5353 op_cost(0);
5354 format %{ "[$reg, $off]" %}
5355 interface(MEMORY_INTER) %{
5356 base($reg);
5357 index(0xffffffff);
5358 scale(0x0);
5359 disp($off);
5360 %}
5361 %}
5362
5363 operand indOffL1(iRegP reg, immLoffset1 off)
5364 %{
5365 constraint(ALLOC_IN_RC(ptr_reg));
5366 match(AddP reg off);
5367 op_cost(0);
5368 format %{ "[$reg, $off]" %}
5369 interface(MEMORY_INTER) %{
5370 base($reg);
5371 index(0xffffffff);
5372 scale(0x0);
5373 disp($off);
5374 %}
5375 %}
5376
5377 operand indOffL2(iRegP reg, immLoffset2 off)
5378 %{
5379 constraint(ALLOC_IN_RC(ptr_reg));
5380 match(AddP reg off);
5381 op_cost(0);
5382 format %{ "[$reg, $off]" %}
5383 interface(MEMORY_INTER) %{
5384 base($reg);
5385 index(0xffffffff);
5386 scale(0x0);
5387 disp($off);
5388 %}
5389 %}
5390
5391 operand indOffL4(iRegP reg, immLoffset4 off)
5392 %{
5393 constraint(ALLOC_IN_RC(ptr_reg));
5394 match(AddP reg off);
5395 op_cost(0);
5396 format %{ "[$reg, $off]" %}
5397 interface(MEMORY_INTER) %{
5398 base($reg);
5399 index(0xffffffff);
5400 scale(0x0);
5401 disp($off);
5402 %}
5403 %}
5404
5405 operand indOffL8(iRegP reg, immLoffset8 off)
5406 %{
5407 constraint(ALLOC_IN_RC(ptr_reg));
5408 match(AddP reg off);
5409 op_cost(0);
5410 format %{ "[$reg, $off]" %}
5411 interface(MEMORY_INTER) %{
5412 base($reg);
5413 index(0xffffffff);
5414 scale(0x0);
5415 disp($off);
5416 %}
5417 %}
5418
5419 operand indOffL16(iRegP reg, immLoffset16 off)
5420 %{
5421 constraint(ALLOC_IN_RC(ptr_reg));
5422 match(AddP reg off);
5423 op_cost(0);
5424 format %{ "[$reg, $off]" %}
5425 interface(MEMORY_INTER) %{
5426 base($reg);
5427 index(0xffffffff);
5428 scale(0x0);
5429 disp($off);
5430 %}
5431 %}
5432
5433 operand indirectX2P(iRegL reg)
5434 %{
5435 constraint(ALLOC_IN_RC(ptr_reg));
5436 match(CastX2P reg);
5437 op_cost(0);
5438 format %{ "[$reg]\t# long -> ptr" %}
5439 interface(MEMORY_INTER) %{
5440 base($reg);
5441 index(0xffffffff);
5442 scale(0x0);
5443 disp(0x0);
5444 %}
5445 %}
5446
5447 operand indOffX2P(iRegL reg, immLOffset off)
5448 %{
5449 constraint(ALLOC_IN_RC(ptr_reg));
5450 match(AddP (CastX2P reg) off);
5451 op_cost(0);
5452 format %{ "[$reg, $off]\t# long -> ptr" %}
5453 interface(MEMORY_INTER) %{
5454 base($reg);
5455 index(0xffffffff);
5456 scale(0x0);
5457 disp($off);
5458 %}
5459 %}
5460
5461 operand indirectN(iRegN reg)
5462 %{
5463 predicate(CompressedOops::shift() == 0);
5464 constraint(ALLOC_IN_RC(ptr_reg));
5465 match(DecodeN reg);
5466 op_cost(0);
5467 format %{ "[$reg]\t# narrow" %}
5468 interface(MEMORY_INTER) %{
5469 base($reg);
5470 index(0xffffffff);
5471 scale(0x0);
5472 disp(0x0);
5473 %}
5474 %}
5475
5476 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5477 %{
5478 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5479 constraint(ALLOC_IN_RC(ptr_reg));
5480 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5481 op_cost(0);
5482 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5483 interface(MEMORY_INTER) %{
5484 base($reg);
5485 index($ireg);
5486 scale($scale);
5487 disp(0x0);
5488 %}
5489 %}
5490
5491 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5492 %{
5493 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5494 constraint(ALLOC_IN_RC(ptr_reg));
5495 match(AddP (DecodeN reg) (LShiftL lreg scale));
5496 op_cost(0);
5497 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5498 interface(MEMORY_INTER) %{
5499 base($reg);
5500 index($lreg);
5501 scale($scale);
5502 disp(0x0);
5503 %}
5504 %}
5505
5506 operand indIndexI2LN(iRegN reg, iRegI ireg)
5507 %{
5508 predicate(CompressedOops::shift() == 0);
5509 constraint(ALLOC_IN_RC(ptr_reg));
5510 match(AddP (DecodeN reg) (ConvI2L ireg));
5511 op_cost(0);
5512 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5513 interface(MEMORY_INTER) %{
5514 base($reg);
5515 index($ireg);
5516 scale(0x0);
5517 disp(0x0);
5518 %}
5519 %}
5520
5521 operand indIndexN(iRegN reg, iRegL lreg)
5522 %{
5523 predicate(CompressedOops::shift() == 0);
5524 constraint(ALLOC_IN_RC(ptr_reg));
5525 match(AddP (DecodeN reg) lreg);
5526 op_cost(0);
5527 format %{ "$reg, $lreg\t# narrow" %}
5528 interface(MEMORY_INTER) %{
5529 base($reg);
5530 index($lreg);
5531 scale(0x0);
5532 disp(0x0);
5533 %}
5534 %}
5535
5536 operand indOffIN(iRegN reg, immIOffset off)
5537 %{
5538 predicate(CompressedOops::shift() == 0);
5539 constraint(ALLOC_IN_RC(ptr_reg));
5540 match(AddP (DecodeN reg) off);
5541 op_cost(0);
5542 format %{ "[$reg, $off]\t# narrow" %}
5543 interface(MEMORY_INTER) %{
5544 base($reg);
5545 index(0xffffffff);
5546 scale(0x0);
5547 disp($off);
5548 %}
5549 %}
5550
5551 operand indOffLN(iRegN reg, immLOffset off)
5552 %{
5553 predicate(CompressedOops::shift() == 0);
5554 constraint(ALLOC_IN_RC(ptr_reg));
5555 match(AddP (DecodeN reg) off);
5556 op_cost(0);
5557 format %{ "[$reg, $off]\t# narrow" %}
5558 interface(MEMORY_INTER) %{
5559 base($reg);
5560 index(0xffffffff);
5561 scale(0x0);
5562 disp($off);
5563 %}
5564 %}
5565
5566
5567 //----------Special Memory Operands--------------------------------------------
5568 // Stack Slot Operand - This operand is used for loading and storing temporary
5569 // values on the stack where a match requires a value to
5570 // flow through memory.
5571 operand stackSlotP(sRegP reg)
5572 %{
5573 constraint(ALLOC_IN_RC(stack_slots));
5574 op_cost(100);
5575 // No match rule because this operand is only generated in matching
5576 // match(RegP);
5577 format %{ "[$reg]" %}
5578 interface(MEMORY_INTER) %{
5579 base(0x1e); // RSP
5580 index(0x0); // No Index
5581 scale(0x0); // No Scale
5582 disp($reg); // Stack Offset
5583 %}
5584 %}
5585
5586 operand stackSlotI(sRegI reg)
5587 %{
5588 constraint(ALLOC_IN_RC(stack_slots));
5589 // No match rule because this operand is only generated in matching
5590 // match(RegI);
5591 format %{ "[$reg]" %}
5592 interface(MEMORY_INTER) %{
5593 base(0x1e); // RSP
5594 index(0x0); // No Index
5595 scale(0x0); // No Scale
5596 disp($reg); // Stack Offset
5597 %}
5598 %}
5599
5600 operand stackSlotF(sRegF reg)
5601 %{
5602 constraint(ALLOC_IN_RC(stack_slots));
5603 // No match rule because this operand is only generated in matching
5604 // match(RegF);
5605 format %{ "[$reg]" %}
5606 interface(MEMORY_INTER) %{
5607 base(0x1e); // RSP
5608 index(0x0); // No Index
5609 scale(0x0); // No Scale
5610 disp($reg); // Stack Offset
5611 %}
5612 %}
5613
5614 operand stackSlotD(sRegD reg)
5615 %{
5616 constraint(ALLOC_IN_RC(stack_slots));
5617 // No match rule because this operand is only generated in matching
5618 // match(RegD);
5619 format %{ "[$reg]" %}
5620 interface(MEMORY_INTER) %{
5621 base(0x1e); // RSP
5622 index(0x0); // No Index
5623 scale(0x0); // No Scale
5624 disp($reg); // Stack Offset
5625 %}
5626 %}
5627
5628 operand stackSlotL(sRegL reg)
5629 %{
5630 constraint(ALLOC_IN_RC(stack_slots));
5631 // No match rule because this operand is only generated in matching
5632 // match(RegL);
5633 format %{ "[$reg]" %}
5634 interface(MEMORY_INTER) %{
5635 base(0x1e); // RSP
5636 index(0x0); // No Index
5637 scale(0x0); // No Scale
5638 disp($reg); // Stack Offset
5639 %}
5640 %}
5641
5642 // Operands for expressing Control Flow
5643 // NOTE: Label is a predefined operand which should not be redefined in
5644 // the AD file. It is generically handled within the ADLC.
5645
5646 //----------Conditional Branch Operands----------------------------------------
5647 // Comparison Op - This is the operation of the comparison, and is limited to
5648 // the following set of codes:
5649 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5650 //
5651 // Other attributes of the comparison, such as unsignedness, are specified
5652 // by the comparison instruction that sets a condition code flags register.
5653 // That result is represented by a flags operand whose subtype is appropriate
5654 // to the unsignedness (etc.) of the comparison.
5655 //
5656 // Later, the instruction which matches both the Comparison Op (a Bool) and
5657 // the flags (produced by the Cmp) specifies the coding of the comparison op
5658 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5659
5660 // used for signed integral comparisons and fp comparisons
5661
5662 operand cmpOp()
5663 %{
5664 match(Bool);
5665
5666 format %{ "" %}
5667 interface(COND_INTER) %{
5668 equal(0x0, "eq");
5669 not_equal(0x1, "ne");
5670 less(0xb, "lt");
5671 greater_equal(0xa, "ge");
5672 less_equal(0xd, "le");
5673 greater(0xc, "gt");
5674 overflow(0x6, "vs");
5675 no_overflow(0x7, "vc");
5676 %}
5677 %}
5678
5679 // used for unsigned integral comparisons
5680
5681 operand cmpOpU()
5682 %{
5683 match(Bool);
5684
5685 format %{ "" %}
5686 interface(COND_INTER) %{
5687 equal(0x0, "eq");
5688 not_equal(0x1, "ne");
5689 less(0x3, "lo");
5690 greater_equal(0x2, "hs");
5691 less_equal(0x9, "ls");
5692 greater(0x8, "hi");
5693 overflow(0x6, "vs");
5694 no_overflow(0x7, "vc");
5695 %}
5696 %}
5697
5698 // used for certain integral comparisons which can be
5699 // converted to cbxx or tbxx instructions
5700
5701 operand cmpOpEqNe()
5702 %{
5703 match(Bool);
5704 op_cost(0);
5705 predicate(n->as_Bool()->_test._test == BoolTest::ne
5706 || n->as_Bool()->_test._test == BoolTest::eq);
5707
5708 format %{ "" %}
5709 interface(COND_INTER) %{
5710 equal(0x0, "eq");
5711 not_equal(0x1, "ne");
5712 less(0xb, "lt");
5713 greater_equal(0xa, "ge");
5714 less_equal(0xd, "le");
5715 greater(0xc, "gt");
5716 overflow(0x6, "vs");
5717 no_overflow(0x7, "vc");
5718 %}
5719 %}
5720
5721 // used for certain integral comparisons which can be
5722 // converted to cbxx or tbxx instructions
5723
5724 operand cmpOpLtGe()
5725 %{
5726 match(Bool);
5727 op_cost(0);
5728
5729 predicate(n->as_Bool()->_test._test == BoolTest::lt
5730 || n->as_Bool()->_test._test == BoolTest::ge);
5731
5732 format %{ "" %}
5733 interface(COND_INTER) %{
5734 equal(0x0, "eq");
5735 not_equal(0x1, "ne");
5736 less(0xb, "lt");
5737 greater_equal(0xa, "ge");
5738 less_equal(0xd, "le");
5739 greater(0xc, "gt");
5740 overflow(0x6, "vs");
5741 no_overflow(0x7, "vc");
5742 %}
5743 %}
5744
5745 // used for certain unsigned integral comparisons which can be
5746 // converted to cbxx or tbxx instructions
5747
5748 operand cmpOpUEqNeLeGt()
5749 %{
5750 match(Bool);
5751 op_cost(0);
5752
5753 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5754 n->as_Bool()->_test._test == BoolTest::ne ||
5755 n->as_Bool()->_test._test == BoolTest::le ||
5756 n->as_Bool()->_test._test == BoolTest::gt);
5757
5758 format %{ "" %}
5759 interface(COND_INTER) %{
5760 equal(0x0, "eq");
5761 not_equal(0x1, "ne");
5762 less(0x3, "lo");
5763 greater_equal(0x2, "hs");
5764 less_equal(0x9, "ls");
5765 greater(0x8, "hi");
5766 overflow(0x6, "vs");
5767 no_overflow(0x7, "vc");
5768 %}
5769 %}
5770
5771 // Special operand allowing long args to int ops to be truncated for free
5772
5773 operand iRegL2I(iRegL reg) %{
5774
5775 op_cost(0);
5776
5777 match(ConvL2I reg);
5778
5779 format %{ "l2i($reg)" %}
5780
5781 interface(REG_INTER)
5782 %}
5783
5784 operand iRegL2P(iRegL reg) %{
5785
5786 op_cost(0);
5787
5788 match(CastX2P reg);
5789
5790 format %{ "l2p($reg)" %}
5791
5792 interface(REG_INTER)
5793 %}
5794
5795 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5796 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5797 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5798 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5799
5800 //----------OPERAND CLASSES----------------------------------------------------
5801 // Operand Classes are groups of operands that are used as to simplify
5802 // instruction definitions by not requiring the AD writer to specify
5803 // separate instructions for every form of operand when the
5804 // instruction accepts multiple operand types with the same basic
5805 // encoding and format. The classic case of this is memory operands.
5806
5807 // memory is used to define read/write location for load/store
5808 // instruction defs. we can turn a memory op into an Address
5809
5810 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5811 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5812
5813 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5814 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5815
5816 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5817 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5818
5819 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5820 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5821
5822 // All of the memory operands. For the pipeline description.
5823 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5824 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5825 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5826
5827
5828 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5829 // operations. it allows the src to be either an iRegI or a (ConvL2I
5830 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5831 // can be elided because the 32-bit instruction will just employ the
5832 // lower 32 bits anyway.
5833 //
5834 // n.b. this does not elide all L2I conversions. if the truncated
5835 // value is consumed by more than one operation then the ConvL2I
5836 // cannot be bundled into the consuming nodes so an l2i gets planted
5837 // (actually a movw $dst $src) and the downstream instructions consume
5838 // the result of the l2i as an iRegI input. That's a shame since the
5839 // movw is actually redundant but its not too costly.
5840
5841 opclass iRegIorL2I(iRegI, iRegL2I);
5842 opclass iRegPorL2P(iRegP, iRegL2P);
5843
5844 //----------PIPELINE-----------------------------------------------------------
5845 // Rules which define the behavior of the target architectures pipeline.
5846
5847 // For specific pipelines, eg A53, define the stages of that pipeline
5848 //pipe_desc(ISS, EX1, EX2, WR);
5849 #define ISS S0
5850 #define EX1 S1
5851 #define EX2 S2
5852 #define WR S3
5853
5854 // Integer ALU reg operation
5855 pipeline %{
5856
5857 attributes %{
5858 // ARM instructions are of fixed length
5859 fixed_size_instructions; // Fixed size instructions TODO does
5860 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5861 // ARM instructions come in 32-bit word units
5862 instruction_unit_size = 4; // An instruction is 4 bytes long
5863 instruction_fetch_unit_size = 64; // The processor fetches one line
5864 instruction_fetch_units = 1; // of 64 bytes
5865
5866 // List of nop instructions
5867 nops( MachNop );
5868 %}
5869
5870 // We don't use an actual pipeline model so don't care about resources
5871 // or description. we do use pipeline classes to introduce fixed
5872 // latencies
5873
5874 //----------RESOURCES----------------------------------------------------------
5875 // Resources are the functional units available to the machine
5876
5877 resources( INS0, INS1, INS01 = INS0 | INS1,
5878 ALU0, ALU1, ALU = ALU0 | ALU1,
5879 MAC,
5880 DIV,
5881 BRANCH,
5882 LDST,
5883 NEON_FP);
5884
5885 //----------PIPELINE DESCRIPTION-----------------------------------------------
5886 // Pipeline Description specifies the stages in the machine's pipeline
5887
5888 // Define the pipeline as a generic 6 stage pipeline
5889 pipe_desc(S0, S1, S2, S3, S4, S5);
5890
5891 //----------PIPELINE CLASSES---------------------------------------------------
5892 // Pipeline Classes describe the stages in which input and output are
5893 // referenced by the hardware pipeline.
5894
5895 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5896 %{
5897 single_instruction;
5898 src1 : S1(read);
5899 src2 : S2(read);
5900 dst : S5(write);
5901 INS01 : ISS;
5902 NEON_FP : S5;
5903 %}
5904
5905 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5906 %{
5907 single_instruction;
5908 src1 : S1(read);
5909 src2 : S2(read);
5910 dst : S5(write);
5911 INS01 : ISS;
5912 NEON_FP : S5;
5913 %}
5914
5915 pipe_class fp_uop_s(vRegF dst, vRegF src)
5916 %{
5917 single_instruction;
5918 src : S1(read);
5919 dst : S5(write);
5920 INS01 : ISS;
5921 NEON_FP : S5;
5922 %}
5923
5924 pipe_class fp_uop_d(vRegD dst, vRegD src)
5925 %{
5926 single_instruction;
5927 src : S1(read);
5928 dst : S5(write);
5929 INS01 : ISS;
5930 NEON_FP : S5;
5931 %}
5932
5933 pipe_class fp_d2f(vRegF dst, vRegD src)
5934 %{
5935 single_instruction;
5936 src : S1(read);
5937 dst : S5(write);
5938 INS01 : ISS;
5939 NEON_FP : S5;
5940 %}
5941
5942 pipe_class fp_f2d(vRegD dst, vRegF src)
5943 %{
5944 single_instruction;
5945 src : S1(read);
5946 dst : S5(write);
5947 INS01 : ISS;
5948 NEON_FP : S5;
5949 %}
5950
5951 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5952 %{
5953 single_instruction;
5954 src : S1(read);
5955 dst : S5(write);
5956 INS01 : ISS;
5957 NEON_FP : S5;
5958 %}
5959
5960 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5961 %{
5962 single_instruction;
5963 src : S1(read);
5964 dst : S5(write);
5965 INS01 : ISS;
5966 NEON_FP : S5;
5967 %}
5968
5969 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5970 %{
5971 single_instruction;
5972 src : S1(read);
5973 dst : S5(write);
5974 INS01 : ISS;
5975 NEON_FP : S5;
5976 %}
5977
5978 pipe_class fp_l2f(vRegF dst, iRegL src)
5979 %{
5980 single_instruction;
5981 src : S1(read);
5982 dst : S5(write);
5983 INS01 : ISS;
5984 NEON_FP : S5;
5985 %}
5986
5987 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5988 %{
5989 single_instruction;
5990 src : S1(read);
5991 dst : S5(write);
5992 INS01 : ISS;
5993 NEON_FP : S5;
5994 %}
5995
5996 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5997 %{
5998 single_instruction;
5999 src : S1(read);
6000 dst : S5(write);
6001 INS01 : ISS;
6002 NEON_FP : S5;
6003 %}
6004
6005 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6006 %{
6007 single_instruction;
6008 src : S1(read);
6009 dst : S5(write);
6010 INS01 : ISS;
6011 NEON_FP : S5;
6012 %}
6013
6014 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6015 %{
6016 single_instruction;
6017 src : S1(read);
6018 dst : S5(write);
6019 INS01 : ISS;
6020 NEON_FP : S5;
6021 %}
6022
6023 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6024 %{
6025 single_instruction;
6026 src1 : S1(read);
6027 src2 : S2(read);
6028 dst : S5(write);
6029 INS0 : ISS;
6030 NEON_FP : S5;
6031 %}
6032
6033 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6034 %{
6035 single_instruction;
6036 src1 : S1(read);
6037 src2 : S2(read);
6038 dst : S5(write);
6039 INS0 : ISS;
6040 NEON_FP : S5;
6041 %}
6042
6043 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6044 %{
6045 single_instruction;
6046 cr : S1(read);
6047 src1 : S1(read);
6048 src2 : S1(read);
6049 dst : S3(write);
6050 INS01 : ISS;
6051 NEON_FP : S3;
6052 %}
6053
6054 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6055 %{
6056 single_instruction;
6057 cr : S1(read);
6058 src1 : S1(read);
6059 src2 : S1(read);
6060 dst : S3(write);
6061 INS01 : ISS;
6062 NEON_FP : S3;
6063 %}
6064
6065 pipe_class fp_imm_s(vRegF dst)
6066 %{
6067 single_instruction;
6068 dst : S3(write);
6069 INS01 : ISS;
6070 NEON_FP : S3;
6071 %}
6072
6073 pipe_class fp_imm_d(vRegD dst)
6074 %{
6075 single_instruction;
6076 dst : S3(write);
6077 INS01 : ISS;
6078 NEON_FP : S3;
6079 %}
6080
6081 pipe_class fp_load_constant_s(vRegF dst)
6082 %{
6083 single_instruction;
6084 dst : S4(write);
6085 INS01 : ISS;
6086 NEON_FP : S4;
6087 %}
6088
6089 pipe_class fp_load_constant_d(vRegD dst)
6090 %{
6091 single_instruction;
6092 dst : S4(write);
6093 INS01 : ISS;
6094 NEON_FP : S4;
6095 %}
6096
6097 //------- Integer ALU operations --------------------------
6098
6099 // Integer ALU reg-reg operation
6100 // Operands needed in EX1, result generated in EX2
6101 // Eg. ADD x0, x1, x2
6102 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6103 %{
6104 single_instruction;
6105 dst : EX2(write);
6106 src1 : EX1(read);
6107 src2 : EX1(read);
6108 INS01 : ISS; // Dual issue as instruction 0 or 1
6109 ALU : EX2;
6110 %}
6111
6112 // Integer ALU reg-reg operation with constant shift
6113 // Shifted register must be available in LATE_ISS instead of EX1
6114 // Eg. ADD x0, x1, x2, LSL #2
6115 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6116 %{
6117 single_instruction;
6118 dst : EX2(write);
6119 src1 : EX1(read);
6120 src2 : ISS(read);
6121 INS01 : ISS;
6122 ALU : EX2;
6123 %}
6124
6125 // Integer ALU reg operation with constant shift
6126 // Eg. LSL x0, x1, #shift
6127 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6128 %{
6129 single_instruction;
6130 dst : EX2(write);
6131 src1 : ISS(read);
6132 INS01 : ISS;
6133 ALU : EX2;
6134 %}
6135
6136 // Integer ALU reg-reg operation with variable shift
6137 // Both operands must be available in LATE_ISS instead of EX1
6138 // Result is available in EX1 instead of EX2
6139 // Eg. LSLV x0, x1, x2
6140 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6141 %{
6142 single_instruction;
6143 dst : EX1(write);
6144 src1 : ISS(read);
6145 src2 : ISS(read);
6146 INS01 : ISS;
6147 ALU : EX1;
6148 %}
6149
6150 // Integer ALU reg-reg operation with extract
6151 // As for _vshift above, but result generated in EX2
6152 // Eg. EXTR x0, x1, x2, #N
6153 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6154 %{
6155 single_instruction;
6156 dst : EX2(write);
6157 src1 : ISS(read);
6158 src2 : ISS(read);
6159 INS1 : ISS; // Can only dual issue as Instruction 1
6160 ALU : EX1;
6161 %}
6162
6163 // Integer ALU reg operation
6164 // Eg. NEG x0, x1
6165 pipe_class ialu_reg(iRegI dst, iRegI src)
6166 %{
6167 single_instruction;
6168 dst : EX2(write);
6169 src : EX1(read);
6170 INS01 : ISS;
6171 ALU : EX2;
6172 %}
6173
6174 // Integer ALU reg mmediate operation
6175 // Eg. ADD x0, x1, #N
6176 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6177 %{
6178 single_instruction;
6179 dst : EX2(write);
6180 src1 : EX1(read);
6181 INS01 : ISS;
6182 ALU : EX2;
6183 %}
6184
6185 // Integer ALU immediate operation (no source operands)
6186 // Eg. MOV x0, #N
6187 pipe_class ialu_imm(iRegI dst)
6188 %{
6189 single_instruction;
6190 dst : EX1(write);
6191 INS01 : ISS;
6192 ALU : EX1;
6193 %}
6194
6195 //------- Compare operation -------------------------------
6196
6197 // Compare reg-reg
6198 // Eg. CMP x0, x1
6199 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6200 %{
6201 single_instruction;
6202 // fixed_latency(16);
6203 cr : EX2(write);
6204 op1 : EX1(read);
6205 op2 : EX1(read);
6206 INS01 : ISS;
6207 ALU : EX2;
6208 %}
6209
6210 // Compare reg-reg
6211 // Eg. CMP x0, #N
6212 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6213 %{
6214 single_instruction;
6215 // fixed_latency(16);
6216 cr : EX2(write);
6217 op1 : EX1(read);
6218 INS01 : ISS;
6219 ALU : EX2;
6220 %}
6221
6222 //------- Conditional instructions ------------------------
6223
6224 // Conditional no operands
6225 // Eg. CSINC x0, zr, zr, <cond>
6226 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6227 %{
6228 single_instruction;
6229 cr : EX1(read);
6230 dst : EX2(write);
6231 INS01 : ISS;
6232 ALU : EX2;
6233 %}
6234
6235 // Conditional 2 operand
6236 // EG. CSEL X0, X1, X2, <cond>
6237 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6238 %{
6239 single_instruction;
6240 cr : EX1(read);
6241 src1 : EX1(read);
6242 src2 : EX1(read);
6243 dst : EX2(write);
6244 INS01 : ISS;
6245 ALU : EX2;
6246 %}
6247
6248 // Conditional 2 operand
6249 // EG. CSEL X0, X1, X2, <cond>
6250 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6251 %{
6252 single_instruction;
6253 cr : EX1(read);
6254 src : EX1(read);
6255 dst : EX2(write);
6256 INS01 : ISS;
6257 ALU : EX2;
6258 %}
6259
6260 //------- Multiply pipeline operations --------------------
6261
6262 // Multiply reg-reg
6263 // Eg. MUL w0, w1, w2
6264 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6265 %{
6266 single_instruction;
6267 dst : WR(write);
6268 src1 : ISS(read);
6269 src2 : ISS(read);
6270 INS01 : ISS;
6271 MAC : WR;
6272 %}
6273
6274 // Multiply accumulate
6275 // Eg. MADD w0, w1, w2, w3
6276 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6277 %{
6278 single_instruction;
6279 dst : WR(write);
6280 src1 : ISS(read);
6281 src2 : ISS(read);
6282 src3 : ISS(read);
6283 INS01 : ISS;
6284 MAC : WR;
6285 %}
6286
6287 // Eg. MUL w0, w1, w2
6288 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6289 %{
6290 single_instruction;
6291 fixed_latency(3); // Maximum latency for 64 bit mul
6292 dst : WR(write);
6293 src1 : ISS(read);
6294 src2 : ISS(read);
6295 INS01 : ISS;
6296 MAC : WR;
6297 %}
6298
6299 // Multiply accumulate
6300 // Eg. MADD w0, w1, w2, w3
6301 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6302 %{
6303 single_instruction;
6304 fixed_latency(3); // Maximum latency for 64 bit mul
6305 dst : WR(write);
6306 src1 : ISS(read);
6307 src2 : ISS(read);
6308 src3 : ISS(read);
6309 INS01 : ISS;
6310 MAC : WR;
6311 %}
6312
6313 //------- Divide pipeline operations --------------------
6314
6315 // Eg. SDIV w0, w1, w2
6316 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6317 %{
6318 single_instruction;
6319 fixed_latency(8); // Maximum latency for 32 bit divide
6320 dst : WR(write);
6321 src1 : ISS(read);
6322 src2 : ISS(read);
6323 INS0 : ISS; // Can only dual issue as instruction 0
6324 DIV : WR;
6325 %}
6326
6327 // Eg. SDIV x0, x1, x2
6328 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6329 %{
6330 single_instruction;
6331 fixed_latency(16); // Maximum latency for 64 bit divide
6332 dst : WR(write);
6333 src1 : ISS(read);
6334 src2 : ISS(read);
6335 INS0 : ISS; // Can only dual issue as instruction 0
6336 DIV : WR;
6337 %}
6338
6339 //------- Load pipeline operations ------------------------
6340
6341 // Load - prefetch
6342 // Eg. PFRM <mem>
6343 pipe_class iload_prefetch(memory mem)
6344 %{
6345 single_instruction;
6346 mem : ISS(read);
6347 INS01 : ISS;
6348 LDST : WR;
6349 %}
6350
6351 // Load - reg, mem
6352 // Eg. LDR x0, <mem>
6353 pipe_class iload_reg_mem(iRegI dst, memory mem)
6354 %{
6355 single_instruction;
6356 dst : WR(write);
6357 mem : ISS(read);
6358 INS01 : ISS;
6359 LDST : WR;
6360 %}
6361
6362 // Load - reg, reg
6363 // Eg. LDR x0, [sp, x1]
6364 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6365 %{
6366 single_instruction;
6367 dst : WR(write);
6368 src : ISS(read);
6369 INS01 : ISS;
6370 LDST : WR;
6371 %}
6372
6373 //------- Store pipeline operations -----------------------
6374
6375 // Store - zr, mem
6376 // Eg. STR zr, <mem>
6377 pipe_class istore_mem(memory mem)
6378 %{
6379 single_instruction;
6380 mem : ISS(read);
6381 INS01 : ISS;
6382 LDST : WR;
6383 %}
6384
6385 // Store - reg, mem
6386 // Eg. STR x0, <mem>
6387 pipe_class istore_reg_mem(iRegI src, memory mem)
6388 %{
6389 single_instruction;
6390 mem : ISS(read);
6391 src : EX2(read);
6392 INS01 : ISS;
6393 LDST : WR;
6394 %}
6395
6396 // Store - reg, reg
6397 // Eg. STR x0, [sp, x1]
6398 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6399 %{
6400 single_instruction;
6401 dst : ISS(read);
6402 src : EX2(read);
6403 INS01 : ISS;
6404 LDST : WR;
6405 %}
6406
6407 //------- Store pipeline operations -----------------------
6408
6409 // Branch
6410 pipe_class pipe_branch()
6411 %{
6412 single_instruction;
6413 INS01 : ISS;
6414 BRANCH : EX1;
6415 %}
6416
6417 // Conditional branch
6418 pipe_class pipe_branch_cond(rFlagsReg cr)
6419 %{
6420 single_instruction;
6421 cr : EX1(read);
6422 INS01 : ISS;
6423 BRANCH : EX1;
6424 %}
6425
6426 // Compare & Branch
6427 // EG. CBZ/CBNZ
6428 pipe_class pipe_cmp_branch(iRegI op1)
6429 %{
6430 single_instruction;
6431 op1 : EX1(read);
6432 INS01 : ISS;
6433 BRANCH : EX1;
6434 %}
6435
6436 //------- Synchronisation operations ----------------------
6437
6438 // Any operation requiring serialization.
6439 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6440 pipe_class pipe_serial()
6441 %{
6442 single_instruction;
6443 force_serialization;
6444 fixed_latency(16);
6445 INS01 : ISS(2); // Cannot dual issue with any other instruction
6446 LDST : WR;
6447 %}
6448
6449 // Generic big/slow expanded idiom - also serialized
6450 pipe_class pipe_slow()
6451 %{
6452 instruction_count(10);
6453 multiple_bundles;
6454 force_serialization;
6455 fixed_latency(16);
6456 INS01 : ISS(2); // Cannot dual issue with any other instruction
6457 LDST : WR;
6458 %}
6459
6460 // Empty pipeline class
6461 pipe_class pipe_class_empty()
6462 %{
6463 single_instruction;
6464 fixed_latency(0);
6465 %}
6466
6467 // Default pipeline class.
6468 pipe_class pipe_class_default()
6469 %{
6470 single_instruction;
6471 fixed_latency(2);
6472 %}
6473
6474 // Pipeline class for compares.
6475 pipe_class pipe_class_compare()
6476 %{
6477 single_instruction;
6478 fixed_latency(16);
6479 %}
6480
6481 // Pipeline class for memory operations.
6482 pipe_class pipe_class_memory()
6483 %{
6484 single_instruction;
6485 fixed_latency(16);
6486 %}
6487
6488 // Pipeline class for call.
6489 pipe_class pipe_class_call()
6490 %{
6491 single_instruction;
6492 fixed_latency(100);
6493 %}
6494
6495 // Define the class for the Nop node.
6496 define %{
6497 MachNop = pipe_class_empty;
6498 %}
6499
6500 %}
6501 //----------INSTRUCTIONS-------------------------------------------------------
6502 //
6503 // match -- States which machine-independent subtree may be replaced
6504 // by this instruction.
6505 // ins_cost -- The estimated cost of this instruction is used by instruction
6506 // selection to identify a minimum cost tree of machine
6507 // instructions that matches a tree of machine-independent
6508 // instructions.
6509 // format -- A string providing the disassembly for this instruction.
6510 // The value of an instruction's operand may be inserted
6511 // by referring to it with a '$' prefix.
6512 // opcode -- Three instruction opcodes may be provided. These are referred
6513 // to within an encode class as $primary, $secondary, and $tertiary
6514 // rrspectively. The primary opcode is commonly used to
6515 // indicate the type of machine instruction, while secondary
6516 // and tertiary are often used for prefix options or addressing
6517 // modes.
6518 // ins_encode -- A list of encode classes with parameters. The encode class
6519 // name must have been defined in an 'enc_class' specification
6520 // in the encode section of the architecture description.
6521
6522 // ============================================================================
6523 // Memory (Load/Store) Instructions
6524
6525 // Load Instructions
6526
6527 // Load Byte (8 bit signed)
6528 instruct loadB(iRegINoSp dst, memory1 mem)
6529 %{
6530 match(Set dst (LoadB mem));
6531 predicate(!needs_acquiring_load(n));
6532
6533 ins_cost(4 * INSN_COST);
6534 format %{ "ldrsbw $dst, $mem\t# byte" %}
6535
6536 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6537
6538 ins_pipe(iload_reg_mem);
6539 %}
6540
6541 // Load Byte (8 bit signed) into long
6542 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6543 %{
6544 match(Set dst (ConvI2L (LoadB mem)));
6545 predicate(!needs_acquiring_load(n->in(1)));
6546
6547 ins_cost(4 * INSN_COST);
6548 format %{ "ldrsb $dst, $mem\t# byte" %}
6549
6550 ins_encode(aarch64_enc_ldrsb(dst, mem));
6551
6552 ins_pipe(iload_reg_mem);
6553 %}
6554
6555 // Load Byte (8 bit unsigned)
6556 instruct loadUB(iRegINoSp dst, memory1 mem)
6557 %{
6558 match(Set dst (LoadUB mem));
6559 predicate(!needs_acquiring_load(n));
6560
6561 ins_cost(4 * INSN_COST);
6562 format %{ "ldrbw $dst, $mem\t# byte" %}
6563
6564 ins_encode(aarch64_enc_ldrb(dst, mem));
6565
6566 ins_pipe(iload_reg_mem);
6567 %}
6568
6569 // Load Byte (8 bit unsigned) into long
6570 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6571 %{
6572 match(Set dst (ConvI2L (LoadUB mem)));
6573 predicate(!needs_acquiring_load(n->in(1)));
6574
6575 ins_cost(4 * INSN_COST);
6576 format %{ "ldrb $dst, $mem\t# byte" %}
6577
6578 ins_encode(aarch64_enc_ldrb(dst, mem));
6579
6580 ins_pipe(iload_reg_mem);
6581 %}
6582
6583 // Load Short (16 bit signed)
6584 instruct loadS(iRegINoSp dst, memory2 mem)
6585 %{
6586 match(Set dst (LoadS mem));
6587 predicate(!needs_acquiring_load(n));
6588
6589 ins_cost(4 * INSN_COST);
6590 format %{ "ldrshw $dst, $mem\t# short" %}
6591
6592 ins_encode(aarch64_enc_ldrshw(dst, mem));
6593
6594 ins_pipe(iload_reg_mem);
6595 %}
6596
6597 // Load Short (16 bit signed) into long
6598 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6599 %{
6600 match(Set dst (ConvI2L (LoadS mem)));
6601 predicate(!needs_acquiring_load(n->in(1)));
6602
6603 ins_cost(4 * INSN_COST);
6604 format %{ "ldrsh $dst, $mem\t# short" %}
6605
6606 ins_encode(aarch64_enc_ldrsh(dst, mem));
6607
6608 ins_pipe(iload_reg_mem);
6609 %}
6610
6611 // Load Char (16 bit unsigned)
6612 instruct loadUS(iRegINoSp dst, memory2 mem)
6613 %{
6614 match(Set dst (LoadUS mem));
6615 predicate(!needs_acquiring_load(n));
6616
6617 ins_cost(4 * INSN_COST);
6618 format %{ "ldrh $dst, $mem\t# short" %}
6619
6620 ins_encode(aarch64_enc_ldrh(dst, mem));
6621
6622 ins_pipe(iload_reg_mem);
6623 %}
6624
6625 // Load Short/Char (16 bit unsigned) into long
6626 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6627 %{
6628 match(Set dst (ConvI2L (LoadUS mem)));
6629 predicate(!needs_acquiring_load(n->in(1)));
6630
6631 ins_cost(4 * INSN_COST);
6632 format %{ "ldrh $dst, $mem\t# short" %}
6633
6634 ins_encode(aarch64_enc_ldrh(dst, mem));
6635
6636 ins_pipe(iload_reg_mem);
6637 %}
6638
6639 // Load Integer (32 bit signed)
6640 instruct loadI(iRegINoSp dst, memory4 mem)
6641 %{
6642 match(Set dst (LoadI mem));
6643 predicate(!needs_acquiring_load(n));
6644
6645 ins_cost(4 * INSN_COST);
6646 format %{ "ldrw $dst, $mem\t# int" %}
6647
6648 ins_encode(aarch64_enc_ldrw(dst, mem));
6649
6650 ins_pipe(iload_reg_mem);
6651 %}
6652
6653 // Load Integer (32 bit signed) into long
6654 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6655 %{
6656 match(Set dst (ConvI2L (LoadI mem)));
6657 predicate(!needs_acquiring_load(n->in(1)));
6658
6659 ins_cost(4 * INSN_COST);
6660 format %{ "ldrsw $dst, $mem\t# int" %}
6661
6662 ins_encode(aarch64_enc_ldrsw(dst, mem));
6663
6664 ins_pipe(iload_reg_mem);
6665 %}
6666
6667 // Load Integer (32 bit unsigned) into long
6668 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6669 %{
6670 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6671 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6672
6673 ins_cost(4 * INSN_COST);
6674 format %{ "ldrw $dst, $mem\t# int" %}
6675
6676 ins_encode(aarch64_enc_ldrw(dst, mem));
6677
6678 ins_pipe(iload_reg_mem);
6679 %}
6680
6681 // Load Long (64 bit signed)
6682 instruct loadL(iRegLNoSp dst, memory8 mem)
6683 %{
6684 match(Set dst (LoadL mem));
6685 predicate(!needs_acquiring_load(n));
6686
6687 ins_cost(4 * INSN_COST);
6688 format %{ "ldr $dst, $mem\t# int" %}
6689
6690 ins_encode(aarch64_enc_ldr(dst, mem));
6691
6692 ins_pipe(iload_reg_mem);
6693 %}
6694
6695 // Load Range
6696 instruct loadRange(iRegINoSp dst, memory4 mem)
6697 %{
6698 match(Set dst (LoadRange mem));
6699
6700 ins_cost(4 * INSN_COST);
6701 format %{ "ldrw $dst, $mem\t# range" %}
6702
6703 ins_encode(aarch64_enc_ldrw(dst, mem));
6704
6705 ins_pipe(iload_reg_mem);
6706 %}
6707
6708 // Load Pointer
6709 instruct loadP(iRegPNoSp dst, memory8 mem)
6710 %{
6711 match(Set dst (LoadP mem));
6712 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6713
6714 ins_cost(4 * INSN_COST);
6715 format %{ "ldr $dst, $mem\t# ptr" %}
6716
6717 ins_encode(aarch64_enc_ldr(dst, mem));
6718
6719 ins_pipe(iload_reg_mem);
6720 %}
6721
6722 // Load Compressed Pointer
6723 instruct loadN(iRegNNoSp dst, memory4 mem)
6724 %{
6725 match(Set dst (LoadN mem));
6726 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6727
6728 ins_cost(4 * INSN_COST);
6729 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6730
6731 ins_encode(aarch64_enc_ldrw(dst, mem));
6732
6733 ins_pipe(iload_reg_mem);
6734 %}
6735
6736 // Load Klass Pointer
6737 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6738 %{
6739 match(Set dst (LoadKlass mem));
6740 predicate(!needs_acquiring_load(n));
6741
6742 ins_cost(4 * INSN_COST);
6743 format %{ "ldr $dst, $mem\t# class" %}
6744
6745 ins_encode(aarch64_enc_ldr(dst, mem));
6746
6747 ins_pipe(iload_reg_mem);
6748 %}
6749
6750 // Load Narrow Klass Pointer
6751 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6752 %{
6753 match(Set dst (LoadNKlass mem));
6754 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6755
6756 ins_cost(4 * INSN_COST);
6757 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6758
6759 ins_encode(aarch64_enc_ldrw(dst, mem));
6760
6761 ins_pipe(iload_reg_mem);
6762 %}
6763
6764 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6765 %{
6766 match(Set dst (LoadNKlass mem));
6767 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6768
6769 ins_cost(4 * INSN_COST);
6770 format %{
6771 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6772 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6773 %}
6774 ins_encode %{
6775 // inlined aarch64_enc_ldrw
6776 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6777 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6778 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6779 %}
6780 ins_pipe(iload_reg_mem);
6781 %}
6782
6783 // Load Float
6784 instruct loadF(vRegF dst, memory4 mem)
6785 %{
6786 match(Set dst (LoadF mem));
6787 predicate(!needs_acquiring_load(n));
6788
6789 ins_cost(4 * INSN_COST);
6790 format %{ "ldrs $dst, $mem\t# float" %}
6791
6792 ins_encode( aarch64_enc_ldrs(dst, mem) );
6793
6794 ins_pipe(pipe_class_memory);
6795 %}
6796
6797 // Load Double
6798 instruct loadD(vRegD dst, memory8 mem)
6799 %{
6800 match(Set dst (LoadD mem));
6801 predicate(!needs_acquiring_load(n));
6802
6803 ins_cost(4 * INSN_COST);
6804 format %{ "ldrd $dst, $mem\t# double" %}
6805
6806 ins_encode( aarch64_enc_ldrd(dst, mem) );
6807
6808 ins_pipe(pipe_class_memory);
6809 %}
6810
6811
6812 // Load Int Constant
6813 instruct loadConI(iRegINoSp dst, immI src)
6814 %{
6815 match(Set dst src);
6816
6817 ins_cost(INSN_COST);
6818 format %{ "mov $dst, $src\t# int" %}
6819
6820 ins_encode( aarch64_enc_movw_imm(dst, src) );
6821
6822 ins_pipe(ialu_imm);
6823 %}
6824
6825 // Load Long Constant
6826 instruct loadConL(iRegLNoSp dst, immL src)
6827 %{
6828 match(Set dst src);
6829
6830 ins_cost(INSN_COST);
6831 format %{ "mov $dst, $src\t# long" %}
6832
6833 ins_encode( aarch64_enc_mov_imm(dst, src) );
6834
6835 ins_pipe(ialu_imm);
6836 %}
6837
6838 // Load Pointer Constant
6839
6840 instruct loadConP(iRegPNoSp dst, immP con)
6841 %{
6842 match(Set dst con);
6843
6844 ins_cost(INSN_COST * 4);
6845 format %{
6846 "mov $dst, $con\t# ptr"
6847 %}
6848
6849 ins_encode(aarch64_enc_mov_p(dst, con));
6850
6851 ins_pipe(ialu_imm);
6852 %}
6853
6854 // Load Null Pointer Constant
6855
6856 instruct loadConP0(iRegPNoSp dst, immP0 con)
6857 %{
6858 match(Set dst con);
6859
6860 ins_cost(INSN_COST);
6861 format %{ "mov $dst, $con\t# nullptr ptr" %}
6862
6863 ins_encode(aarch64_enc_mov_p0(dst, con));
6864
6865 ins_pipe(ialu_imm);
6866 %}
6867
6868 // Load Pointer Constant One
6869
6870 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6871 %{
6872 match(Set dst con);
6873
6874 ins_cost(INSN_COST);
6875 format %{ "mov $dst, $con\t# nullptr ptr" %}
6876
6877 ins_encode(aarch64_enc_mov_p1(dst, con));
6878
6879 ins_pipe(ialu_imm);
6880 %}
6881
6882 // Load Byte Map Base Constant
6883
6884 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6885 %{
6886 match(Set dst con);
6887
6888 ins_cost(INSN_COST);
6889 format %{ "adr $dst, $con\t# Byte Map Base" %}
6890
6891 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6892
6893 ins_pipe(ialu_imm);
6894 %}
6895
6896 // Load Narrow Pointer Constant
6897
6898 instruct loadConN(iRegNNoSp dst, immN con)
6899 %{
6900 match(Set dst con);
6901
6902 ins_cost(INSN_COST * 4);
6903 format %{ "mov $dst, $con\t# compressed ptr" %}
6904
6905 ins_encode(aarch64_enc_mov_n(dst, con));
6906
6907 ins_pipe(ialu_imm);
6908 %}
6909
6910 // Load Narrow Null Pointer Constant
6911
6912 instruct loadConN0(iRegNNoSp dst, immN0 con)
6913 %{
6914 match(Set dst con);
6915
6916 ins_cost(INSN_COST);
6917 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6918
6919 ins_encode(aarch64_enc_mov_n0(dst, con));
6920
6921 ins_pipe(ialu_imm);
6922 %}
6923
6924 // Load Narrow Klass Constant
6925
6926 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6927 %{
6928 match(Set dst con);
6929
6930 ins_cost(INSN_COST);
6931 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6932
6933 ins_encode(aarch64_enc_mov_nk(dst, con));
6934
6935 ins_pipe(ialu_imm);
6936 %}
6937
6938 // Load Packed Float Constant
6939
6940 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6941 match(Set dst con);
6942 ins_cost(INSN_COST * 4);
6943 format %{ "fmovs $dst, $con"%}
6944 ins_encode %{
6945 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6946 %}
6947
6948 ins_pipe(fp_imm_s);
6949 %}
6950
6951 // Load Float Constant
6952
6953 instruct loadConF(vRegF dst, immF con) %{
6954 match(Set dst con);
6955
6956 ins_cost(INSN_COST * 4);
6957
6958 format %{
6959 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6960 %}
6961
6962 ins_encode %{
6963 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6964 %}
6965
6966 ins_pipe(fp_load_constant_s);
6967 %}
6968
6969 // Load Packed Double Constant
6970
6971 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6972 match(Set dst con);
6973 ins_cost(INSN_COST);
6974 format %{ "fmovd $dst, $con"%}
6975 ins_encode %{
6976 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6977 %}
6978
6979 ins_pipe(fp_imm_d);
6980 %}
6981
6982 // Load Double Constant
6983
6984 instruct loadConD(vRegD dst, immD con) %{
6985 match(Set dst con);
6986
6987 ins_cost(INSN_COST * 5);
6988 format %{
6989 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6990 %}
6991
6992 ins_encode %{
6993 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6994 %}
6995
6996 ins_pipe(fp_load_constant_d);
6997 %}
6998
6999 // Load Half Float Constant
7000 // The "ldr" instruction loads a 32-bit word from the constant pool into a
7001 // 32-bit register but only the bottom half will be populated and the top
7002 // 16 bits are zero.
7003 instruct loadConH(vRegF dst, immH con) %{
7004 match(Set dst con);
7005 format %{
7006 "ldrs $dst, [$constantaddress]\t# load from constant table: half float=$con\n\t"
7007 %}
7008 ins_encode %{
7009 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7010 %}
7011 ins_pipe(fp_load_constant_s);
7012 %}
7013
7014 // Store Instructions
7015
7016 // Store Byte
7017 instruct storeB(iRegIorL2I src, memory1 mem)
7018 %{
7019 match(Set mem (StoreB mem src));
7020 predicate(!needs_releasing_store(n));
7021
7022 ins_cost(INSN_COST);
7023 format %{ "strb $src, $mem\t# byte" %}
7024
7025 ins_encode(aarch64_enc_strb(src, mem));
7026
7027 ins_pipe(istore_reg_mem);
7028 %}
7029
7030
7031 instruct storeimmB0(immI0 zero, memory1 mem)
7032 %{
7033 match(Set mem (StoreB mem zero));
7034 predicate(!needs_releasing_store(n));
7035
7036 ins_cost(INSN_COST);
7037 format %{ "strb rscractch2, $mem\t# byte" %}
7038
7039 ins_encode(aarch64_enc_strb0(mem));
7040
7041 ins_pipe(istore_mem);
7042 %}
7043
7044 // Store Char/Short
7045 instruct storeC(iRegIorL2I src, memory2 mem)
7046 %{
7047 match(Set mem (StoreC mem src));
7048 predicate(!needs_releasing_store(n));
7049
7050 ins_cost(INSN_COST);
7051 format %{ "strh $src, $mem\t# short" %}
7052
7053 ins_encode(aarch64_enc_strh(src, mem));
7054
7055 ins_pipe(istore_reg_mem);
7056 %}
7057
7058 instruct storeimmC0(immI0 zero, memory2 mem)
7059 %{
7060 match(Set mem (StoreC mem zero));
7061 predicate(!needs_releasing_store(n));
7062
7063 ins_cost(INSN_COST);
7064 format %{ "strh zr, $mem\t# short" %}
7065
7066 ins_encode(aarch64_enc_strh0(mem));
7067
7068 ins_pipe(istore_mem);
7069 %}
7070
7071 // Store Integer
7072
7073 instruct storeI(iRegIorL2I src, memory4 mem)
7074 %{
7075 match(Set mem(StoreI mem src));
7076 predicate(!needs_releasing_store(n));
7077
7078 ins_cost(INSN_COST);
7079 format %{ "strw $src, $mem\t# int" %}
7080
7081 ins_encode(aarch64_enc_strw(src, mem));
7082
7083 ins_pipe(istore_reg_mem);
7084 %}
7085
7086 instruct storeimmI0(immI0 zero, memory4 mem)
7087 %{
7088 match(Set mem(StoreI mem zero));
7089 predicate(!needs_releasing_store(n));
7090
7091 ins_cost(INSN_COST);
7092 format %{ "strw zr, $mem\t# int" %}
7093
7094 ins_encode(aarch64_enc_strw0(mem));
7095
7096 ins_pipe(istore_mem);
7097 %}
7098
7099 // Store Long (64 bit signed)
7100 instruct storeL(iRegL src, memory8 mem)
7101 %{
7102 match(Set mem (StoreL mem src));
7103 predicate(!needs_releasing_store(n));
7104
7105 ins_cost(INSN_COST);
7106 format %{ "str $src, $mem\t# int" %}
7107
7108 ins_encode(aarch64_enc_str(src, mem));
7109
7110 ins_pipe(istore_reg_mem);
7111 %}
7112
7113 // Store Long (64 bit signed)
7114 instruct storeimmL0(immL0 zero, memory8 mem)
7115 %{
7116 match(Set mem (StoreL mem zero));
7117 predicate(!needs_releasing_store(n));
7118
7119 ins_cost(INSN_COST);
7120 format %{ "str zr, $mem\t# int" %}
7121
7122 ins_encode(aarch64_enc_str0(mem));
7123
7124 ins_pipe(istore_mem);
7125 %}
7126
7127 // Store Pointer
7128 instruct storeP(iRegP src, memory8 mem)
7129 %{
7130 match(Set mem (StoreP mem src));
7131 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7132
7133 ins_cost(INSN_COST);
7134 format %{ "str $src, $mem\t# ptr" %}
7135
7136 ins_encode(aarch64_enc_str(src, mem));
7137
7138 ins_pipe(istore_reg_mem);
7139 %}
7140
7141 // Store Pointer
7142 instruct storeimmP0(immP0 zero, memory8 mem)
7143 %{
7144 match(Set mem (StoreP mem zero));
7145 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7146
7147 ins_cost(INSN_COST);
7148 format %{ "str zr, $mem\t# ptr" %}
7149
7150 ins_encode(aarch64_enc_str0(mem));
7151
7152 ins_pipe(istore_mem);
7153 %}
7154
7155 // Store Compressed Pointer
7156 instruct storeN(iRegN src, memory4 mem)
7157 %{
7158 match(Set mem (StoreN mem src));
7159 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7160
7161 ins_cost(INSN_COST);
7162 format %{ "strw $src, $mem\t# compressed ptr" %}
7163
7164 ins_encode(aarch64_enc_strw(src, mem));
7165
7166 ins_pipe(istore_reg_mem);
7167 %}
7168
7169 instruct storeImmN0(immN0 zero, memory4 mem)
7170 %{
7171 match(Set mem (StoreN mem zero));
7172 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7173
7174 ins_cost(INSN_COST);
7175 format %{ "strw zr, $mem\t# compressed ptr" %}
7176
7177 ins_encode(aarch64_enc_strw0(mem));
7178
7179 ins_pipe(istore_mem);
7180 %}
7181
7182 // Store Float
7183 instruct storeF(vRegF src, memory4 mem)
7184 %{
7185 match(Set mem (StoreF mem src));
7186 predicate(!needs_releasing_store(n));
7187
7188 ins_cost(INSN_COST);
7189 format %{ "strs $src, $mem\t# float" %}
7190
7191 ins_encode( aarch64_enc_strs(src, mem) );
7192
7193 ins_pipe(pipe_class_memory);
7194 %}
7195
7196 // TODO
7197 // implement storeImmF0 and storeFImmPacked
7198
7199 // Store Double
7200 instruct storeD(vRegD src, memory8 mem)
7201 %{
7202 match(Set mem (StoreD mem src));
7203 predicate(!needs_releasing_store(n));
7204
7205 ins_cost(INSN_COST);
7206 format %{ "strd $src, $mem\t# double" %}
7207
7208 ins_encode( aarch64_enc_strd(src, mem) );
7209
7210 ins_pipe(pipe_class_memory);
7211 %}
7212
7213 // Store Compressed Klass Pointer
7214 instruct storeNKlass(iRegN src, memory4 mem)
7215 %{
7216 predicate(!needs_releasing_store(n));
7217 match(Set mem (StoreNKlass mem src));
7218
7219 ins_cost(INSN_COST);
7220 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7221
7222 ins_encode(aarch64_enc_strw(src, mem));
7223
7224 ins_pipe(istore_reg_mem);
7225 %}
7226
7227 // TODO
7228 // implement storeImmD0 and storeDImmPacked
7229
7230 // prefetch instructions
7231 // Must be safe to execute with invalid address (cannot fault).
7232
7233 instruct prefetchalloc( memory8 mem ) %{
7234 match(PrefetchAllocation mem);
7235
7236 ins_cost(INSN_COST);
7237 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7238
7239 ins_encode( aarch64_enc_prefetchw(mem) );
7240
7241 ins_pipe(iload_prefetch);
7242 %}
7243
7244 // ---------------- volatile loads and stores ----------------
7245
7246 // Load Byte (8 bit signed)
7247 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7248 %{
7249 match(Set dst (LoadB mem));
7250
7251 ins_cost(VOLATILE_REF_COST);
7252 format %{ "ldarsb $dst, $mem\t# byte" %}
7253
7254 ins_encode(aarch64_enc_ldarsb(dst, mem));
7255
7256 ins_pipe(pipe_serial);
7257 %}
7258
7259 // Load Byte (8 bit signed) into long
7260 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7261 %{
7262 match(Set dst (ConvI2L (LoadB mem)));
7263
7264 ins_cost(VOLATILE_REF_COST);
7265 format %{ "ldarsb $dst, $mem\t# byte" %}
7266
7267 ins_encode(aarch64_enc_ldarsb(dst, mem));
7268
7269 ins_pipe(pipe_serial);
7270 %}
7271
7272 // Load Byte (8 bit unsigned)
7273 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7274 %{
7275 match(Set dst (LoadUB mem));
7276
7277 ins_cost(VOLATILE_REF_COST);
7278 format %{ "ldarb $dst, $mem\t# byte" %}
7279
7280 ins_encode(aarch64_enc_ldarb(dst, mem));
7281
7282 ins_pipe(pipe_serial);
7283 %}
7284
7285 // Load Byte (8 bit unsigned) into long
7286 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7287 %{
7288 match(Set dst (ConvI2L (LoadUB mem)));
7289
7290 ins_cost(VOLATILE_REF_COST);
7291 format %{ "ldarb $dst, $mem\t# byte" %}
7292
7293 ins_encode(aarch64_enc_ldarb(dst, mem));
7294
7295 ins_pipe(pipe_serial);
7296 %}
7297
7298 // Load Short (16 bit signed)
7299 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7300 %{
7301 match(Set dst (LoadS mem));
7302
7303 ins_cost(VOLATILE_REF_COST);
7304 format %{ "ldarshw $dst, $mem\t# short" %}
7305
7306 ins_encode(aarch64_enc_ldarshw(dst, mem));
7307
7308 ins_pipe(pipe_serial);
7309 %}
7310
7311 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7312 %{
7313 match(Set dst (LoadUS mem));
7314
7315 ins_cost(VOLATILE_REF_COST);
7316 format %{ "ldarhw $dst, $mem\t# short" %}
7317
7318 ins_encode(aarch64_enc_ldarhw(dst, mem));
7319
7320 ins_pipe(pipe_serial);
7321 %}
7322
7323 // Load Short/Char (16 bit unsigned) into long
7324 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7325 %{
7326 match(Set dst (ConvI2L (LoadUS mem)));
7327
7328 ins_cost(VOLATILE_REF_COST);
7329 format %{ "ldarh $dst, $mem\t# short" %}
7330
7331 ins_encode(aarch64_enc_ldarh(dst, mem));
7332
7333 ins_pipe(pipe_serial);
7334 %}
7335
7336 // Load Short/Char (16 bit signed) into long
7337 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7338 %{
7339 match(Set dst (ConvI2L (LoadS mem)));
7340
7341 ins_cost(VOLATILE_REF_COST);
7342 format %{ "ldarh $dst, $mem\t# short" %}
7343
7344 ins_encode(aarch64_enc_ldarsh(dst, mem));
7345
7346 ins_pipe(pipe_serial);
7347 %}
7348
7349 // Load Integer (32 bit signed)
7350 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7351 %{
7352 match(Set dst (LoadI mem));
7353
7354 ins_cost(VOLATILE_REF_COST);
7355 format %{ "ldarw $dst, $mem\t# int" %}
7356
7357 ins_encode(aarch64_enc_ldarw(dst, mem));
7358
7359 ins_pipe(pipe_serial);
7360 %}
7361
7362 // Load Integer (32 bit unsigned) into long
7363 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7364 %{
7365 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7366
7367 ins_cost(VOLATILE_REF_COST);
7368 format %{ "ldarw $dst, $mem\t# int" %}
7369
7370 ins_encode(aarch64_enc_ldarw(dst, mem));
7371
7372 ins_pipe(pipe_serial);
7373 %}
7374
7375 // Load Long (64 bit signed)
7376 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7377 %{
7378 match(Set dst (LoadL mem));
7379
7380 ins_cost(VOLATILE_REF_COST);
7381 format %{ "ldar $dst, $mem\t# int" %}
7382
7383 ins_encode(aarch64_enc_ldar(dst, mem));
7384
7385 ins_pipe(pipe_serial);
7386 %}
7387
7388 // Load Pointer
7389 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7390 %{
7391 match(Set dst (LoadP mem));
7392 predicate(n->as_Load()->barrier_data() == 0);
7393
7394 ins_cost(VOLATILE_REF_COST);
7395 format %{ "ldar $dst, $mem\t# ptr" %}
7396
7397 ins_encode(aarch64_enc_ldar(dst, mem));
7398
7399 ins_pipe(pipe_serial);
7400 %}
7401
7402 // Load Compressed Pointer
7403 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7404 %{
7405 match(Set dst (LoadN mem));
7406 predicate(n->as_Load()->barrier_data() == 0);
7407
7408 ins_cost(VOLATILE_REF_COST);
7409 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7410
7411 ins_encode(aarch64_enc_ldarw(dst, mem));
7412
7413 ins_pipe(pipe_serial);
7414 %}
7415
7416 // Load Float
7417 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7418 %{
7419 match(Set dst (LoadF mem));
7420
7421 ins_cost(VOLATILE_REF_COST);
7422 format %{ "ldars $dst, $mem\t# float" %}
7423
7424 ins_encode( aarch64_enc_fldars(dst, mem) );
7425
7426 ins_pipe(pipe_serial);
7427 %}
7428
7429 // Load Double
7430 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7431 %{
7432 match(Set dst (LoadD mem));
7433
7434 ins_cost(VOLATILE_REF_COST);
7435 format %{ "ldard $dst, $mem\t# double" %}
7436
7437 ins_encode( aarch64_enc_fldard(dst, mem) );
7438
7439 ins_pipe(pipe_serial);
7440 %}
7441
7442 // Store Byte
7443 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7444 %{
7445 match(Set mem (StoreB mem src));
7446
7447 ins_cost(VOLATILE_REF_COST);
7448 format %{ "stlrb $src, $mem\t# byte" %}
7449
7450 ins_encode(aarch64_enc_stlrb(src, mem));
7451
7452 ins_pipe(pipe_class_memory);
7453 %}
7454
7455 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7456 %{
7457 match(Set mem (StoreB mem zero));
7458
7459 ins_cost(VOLATILE_REF_COST);
7460 format %{ "stlrb zr, $mem\t# byte" %}
7461
7462 ins_encode(aarch64_enc_stlrb0(mem));
7463
7464 ins_pipe(pipe_class_memory);
7465 %}
7466
7467 // Store Char/Short
7468 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7469 %{
7470 match(Set mem (StoreC mem src));
7471
7472 ins_cost(VOLATILE_REF_COST);
7473 format %{ "stlrh $src, $mem\t# short" %}
7474
7475 ins_encode(aarch64_enc_stlrh(src, mem));
7476
7477 ins_pipe(pipe_class_memory);
7478 %}
7479
7480 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7481 %{
7482 match(Set mem (StoreC mem zero));
7483
7484 ins_cost(VOLATILE_REF_COST);
7485 format %{ "stlrh zr, $mem\t# short" %}
7486
7487 ins_encode(aarch64_enc_stlrh0(mem));
7488
7489 ins_pipe(pipe_class_memory);
7490 %}
7491
7492 // Store Integer
7493
7494 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7495 %{
7496 match(Set mem(StoreI mem src));
7497
7498 ins_cost(VOLATILE_REF_COST);
7499 format %{ "stlrw $src, $mem\t# int" %}
7500
7501 ins_encode(aarch64_enc_stlrw(src, mem));
7502
7503 ins_pipe(pipe_class_memory);
7504 %}
7505
7506 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7507 %{
7508 match(Set mem(StoreI mem zero));
7509
7510 ins_cost(VOLATILE_REF_COST);
7511 format %{ "stlrw zr, $mem\t# int" %}
7512
7513 ins_encode(aarch64_enc_stlrw0(mem));
7514
7515 ins_pipe(pipe_class_memory);
7516 %}
7517
7518 // Store Long (64 bit signed)
7519 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7520 %{
7521 match(Set mem (StoreL mem src));
7522
7523 ins_cost(VOLATILE_REF_COST);
7524 format %{ "stlr $src, $mem\t# int" %}
7525
7526 ins_encode(aarch64_enc_stlr(src, mem));
7527
7528 ins_pipe(pipe_class_memory);
7529 %}
7530
7531 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7532 %{
7533 match(Set mem (StoreL mem zero));
7534
7535 ins_cost(VOLATILE_REF_COST);
7536 format %{ "stlr zr, $mem\t# int" %}
7537
7538 ins_encode(aarch64_enc_stlr0(mem));
7539
7540 ins_pipe(pipe_class_memory);
7541 %}
7542
7543 // Store Pointer
7544 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7545 %{
7546 match(Set mem (StoreP mem src));
7547 predicate(n->as_Store()->barrier_data() == 0);
7548
7549 ins_cost(VOLATILE_REF_COST);
7550 format %{ "stlr $src, $mem\t# ptr" %}
7551
7552 ins_encode(aarch64_enc_stlr(src, mem));
7553
7554 ins_pipe(pipe_class_memory);
7555 %}
7556
7557 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7558 %{
7559 match(Set mem (StoreP mem zero));
7560 predicate(n->as_Store()->barrier_data() == 0);
7561
7562 ins_cost(VOLATILE_REF_COST);
7563 format %{ "stlr zr, $mem\t# ptr" %}
7564
7565 ins_encode(aarch64_enc_stlr0(mem));
7566
7567 ins_pipe(pipe_class_memory);
7568 %}
7569
7570 // Store Compressed Pointer
7571 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7572 %{
7573 match(Set mem (StoreN mem src));
7574 predicate(n->as_Store()->barrier_data() == 0);
7575
7576 ins_cost(VOLATILE_REF_COST);
7577 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7578
7579 ins_encode(aarch64_enc_stlrw(src, mem));
7580
7581 ins_pipe(pipe_class_memory);
7582 %}
7583
7584 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7585 %{
7586 match(Set mem (StoreN mem zero));
7587 predicate(n->as_Store()->barrier_data() == 0);
7588
7589 ins_cost(VOLATILE_REF_COST);
7590 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7591
7592 ins_encode(aarch64_enc_stlrw0(mem));
7593
7594 ins_pipe(pipe_class_memory);
7595 %}
7596
7597 // Store Float
7598 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7599 %{
7600 match(Set mem (StoreF mem src));
7601
7602 ins_cost(VOLATILE_REF_COST);
7603 format %{ "stlrs $src, $mem\t# float" %}
7604
7605 ins_encode( aarch64_enc_fstlrs(src, mem) );
7606
7607 ins_pipe(pipe_class_memory);
7608 %}
7609
7610 // TODO
7611 // implement storeImmF0 and storeFImmPacked
7612
7613 // Store Double
7614 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7615 %{
7616 match(Set mem (StoreD mem src));
7617
7618 ins_cost(VOLATILE_REF_COST);
7619 format %{ "stlrd $src, $mem\t# double" %}
7620
7621 ins_encode( aarch64_enc_fstlrd(src, mem) );
7622
7623 ins_pipe(pipe_class_memory);
7624 %}
7625
7626 // ---------------- end of volatile loads and stores ----------------
7627
7628 instruct cacheWB(indirect addr)
7629 %{
7630 predicate(VM_Version::supports_data_cache_line_flush());
7631 match(CacheWB addr);
7632
7633 ins_cost(100);
7634 format %{"cache wb $addr" %}
7635 ins_encode %{
7636 assert($addr->index_position() < 0, "should be");
7637 assert($addr$$disp == 0, "should be");
7638 __ cache_wb(Address($addr$$base$$Register, 0));
7639 %}
7640 ins_pipe(pipe_slow); // XXX
7641 %}
7642
7643 instruct cacheWBPreSync()
7644 %{
7645 predicate(VM_Version::supports_data_cache_line_flush());
7646 match(CacheWBPreSync);
7647
7648 ins_cost(100);
7649 format %{"cache wb presync" %}
7650 ins_encode %{
7651 __ cache_wbsync(true);
7652 %}
7653 ins_pipe(pipe_slow); // XXX
7654 %}
7655
7656 instruct cacheWBPostSync()
7657 %{
7658 predicate(VM_Version::supports_data_cache_line_flush());
7659 match(CacheWBPostSync);
7660
7661 ins_cost(100);
7662 format %{"cache wb postsync" %}
7663 ins_encode %{
7664 __ cache_wbsync(false);
7665 %}
7666 ins_pipe(pipe_slow); // XXX
7667 %}
7668
7669 // ============================================================================
7670 // BSWAP Instructions
7671
7672 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7673 match(Set dst (ReverseBytesI src));
7674
7675 ins_cost(INSN_COST);
7676 format %{ "revw $dst, $src" %}
7677
7678 ins_encode %{
7679 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7680 %}
7681
7682 ins_pipe(ialu_reg);
7683 %}
7684
7685 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7686 match(Set dst (ReverseBytesL src));
7687
7688 ins_cost(INSN_COST);
7689 format %{ "rev $dst, $src" %}
7690
7691 ins_encode %{
7692 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7693 %}
7694
7695 ins_pipe(ialu_reg);
7696 %}
7697
7698 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7699 match(Set dst (ReverseBytesUS src));
7700
7701 ins_cost(INSN_COST);
7702 format %{ "rev16w $dst, $src" %}
7703
7704 ins_encode %{
7705 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7706 %}
7707
7708 ins_pipe(ialu_reg);
7709 %}
7710
7711 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7712 match(Set dst (ReverseBytesS src));
7713
7714 ins_cost(INSN_COST);
7715 format %{ "rev16w $dst, $src\n\t"
7716 "sbfmw $dst, $dst, #0, #15" %}
7717
7718 ins_encode %{
7719 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7720 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7721 %}
7722
7723 ins_pipe(ialu_reg);
7724 %}
7725
7726 // ============================================================================
7727 // Zero Count Instructions
7728
7729 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7730 match(Set dst (CountLeadingZerosI src));
7731
7732 ins_cost(INSN_COST);
7733 format %{ "clzw $dst, $src" %}
7734 ins_encode %{
7735 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7736 %}
7737
7738 ins_pipe(ialu_reg);
7739 %}
7740
7741 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7742 match(Set dst (CountLeadingZerosL src));
7743
7744 ins_cost(INSN_COST);
7745 format %{ "clz $dst, $src" %}
7746 ins_encode %{
7747 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7748 %}
7749
7750 ins_pipe(ialu_reg);
7751 %}
7752
7753 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7754 match(Set dst (CountTrailingZerosI src));
7755
7756 ins_cost(INSN_COST * 2);
7757 format %{ "rbitw $dst, $src\n\t"
7758 "clzw $dst, $dst" %}
7759 ins_encode %{
7760 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7761 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7762 %}
7763
7764 ins_pipe(ialu_reg);
7765 %}
7766
7767 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7768 match(Set dst (CountTrailingZerosL src));
7769
7770 ins_cost(INSN_COST * 2);
7771 format %{ "rbit $dst, $src\n\t"
7772 "clz $dst, $dst" %}
7773 ins_encode %{
7774 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7775 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7776 %}
7777
7778 ins_pipe(ialu_reg);
7779 %}
7780
7781 //---------- Population Count Instructions -------------------------------------
7782 //
7783
7784 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7785 match(Set dst (PopCountI src));
7786 effect(TEMP tmp);
7787 ins_cost(INSN_COST * 13);
7788
7789 format %{ "movw $src, $src\n\t"
7790 "mov $tmp, $src\t# vector (1D)\n\t"
7791 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7792 "addv $tmp, $tmp\t# vector (8B)\n\t"
7793 "mov $dst, $tmp\t# vector (1D)" %}
7794 ins_encode %{
7795 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7796 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7797 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7798 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7799 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7800 %}
7801
7802 ins_pipe(pipe_class_default);
7803 %}
7804
7805 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7806 match(Set dst (PopCountI (LoadI mem)));
7807 effect(TEMP tmp);
7808 ins_cost(INSN_COST * 13);
7809
7810 format %{ "ldrs $tmp, $mem\n\t"
7811 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7812 "addv $tmp, $tmp\t# vector (8B)\n\t"
7813 "mov $dst, $tmp\t# vector (1D)" %}
7814 ins_encode %{
7815 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7816 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7817 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7818 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7819 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7820 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7821 %}
7822
7823 ins_pipe(pipe_class_default);
7824 %}
7825
7826 // Note: Long.bitCount(long) returns an int.
7827 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7828 match(Set dst (PopCountL src));
7829 effect(TEMP tmp);
7830 ins_cost(INSN_COST * 13);
7831
7832 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7833 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7834 "addv $tmp, $tmp\t# vector (8B)\n\t"
7835 "mov $dst, $tmp\t# vector (1D)" %}
7836 ins_encode %{
7837 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7838 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7839 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7840 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7841 %}
7842
7843 ins_pipe(pipe_class_default);
7844 %}
7845
7846 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7847 match(Set dst (PopCountL (LoadL mem)));
7848 effect(TEMP tmp);
7849 ins_cost(INSN_COST * 13);
7850
7851 format %{ "ldrd $tmp, $mem\n\t"
7852 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7853 "addv $tmp, $tmp\t# vector (8B)\n\t"
7854 "mov $dst, $tmp\t# vector (1D)" %}
7855 ins_encode %{
7856 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7857 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7858 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7859 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7860 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7861 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7862 %}
7863
7864 ins_pipe(pipe_class_default);
7865 %}
7866
7867 // ============================================================================
7868 // VerifyVectorAlignment Instruction
7869
7870 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7871 match(Set addr (VerifyVectorAlignment addr mask));
7872 effect(KILL cr);
7873 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7874 ins_encode %{
7875 Label Lskip;
7876 // check if masked bits of addr are zero
7877 __ tst($addr$$Register, $mask$$constant);
7878 __ br(Assembler::EQ, Lskip);
7879 __ stop("verify_vector_alignment found a misaligned vector memory access");
7880 __ bind(Lskip);
7881 %}
7882 ins_pipe(pipe_slow);
7883 %}
7884
7885 // ============================================================================
7886 // MemBar Instruction
7887
7888 instruct load_fence() %{
7889 match(LoadFence);
7890 ins_cost(VOLATILE_REF_COST);
7891
7892 format %{ "load_fence" %}
7893
7894 ins_encode %{
7895 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7896 %}
7897 ins_pipe(pipe_serial);
7898 %}
7899
7900 instruct unnecessary_membar_acquire() %{
7901 predicate(unnecessary_acquire(n));
7902 match(MemBarAcquire);
7903 ins_cost(0);
7904
7905 format %{ "membar_acquire (elided)" %}
7906
7907 ins_encode %{
7908 __ block_comment("membar_acquire (elided)");
7909 %}
7910
7911 ins_pipe(pipe_class_empty);
7912 %}
7913
7914 instruct membar_acquire() %{
7915 match(MemBarAcquire);
7916 ins_cost(VOLATILE_REF_COST);
7917
7918 format %{ "membar_acquire\n\t"
7919 "dmb ishld" %}
7920
7921 ins_encode %{
7922 __ block_comment("membar_acquire");
7923 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7924 %}
7925
7926 ins_pipe(pipe_serial);
7927 %}
7928
7929
7930 instruct membar_acquire_lock() %{
7931 match(MemBarAcquireLock);
7932 ins_cost(VOLATILE_REF_COST);
7933
7934 format %{ "membar_acquire_lock (elided)" %}
7935
7936 ins_encode %{
7937 __ block_comment("membar_acquire_lock (elided)");
7938 %}
7939
7940 ins_pipe(pipe_serial);
7941 %}
7942
7943 instruct store_fence() %{
7944 match(StoreFence);
7945 ins_cost(VOLATILE_REF_COST);
7946
7947 format %{ "store_fence" %}
7948
7949 ins_encode %{
7950 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7951 %}
7952 ins_pipe(pipe_serial);
7953 %}
7954
7955 instruct unnecessary_membar_release() %{
7956 predicate(unnecessary_release(n));
7957 match(MemBarRelease);
7958 ins_cost(0);
7959
7960 format %{ "membar_release (elided)" %}
7961
7962 ins_encode %{
7963 __ block_comment("membar_release (elided)");
7964 %}
7965 ins_pipe(pipe_serial);
7966 %}
7967
7968 instruct membar_release() %{
7969 match(MemBarRelease);
7970 ins_cost(VOLATILE_REF_COST);
7971
7972 format %{ "membar_release\n\t"
7973 "dmb ishst\n\tdmb ishld" %}
7974
7975 ins_encode %{
7976 __ block_comment("membar_release");
7977 // These will be merged if AlwaysMergeDMB is enabled.
7978 __ membar(Assembler::StoreStore);
7979 __ membar(Assembler::LoadStore);
7980 %}
7981 ins_pipe(pipe_serial);
7982 %}
7983
7984 instruct membar_storestore() %{
7985 match(MemBarStoreStore);
7986 match(StoreStoreFence);
7987 ins_cost(VOLATILE_REF_COST);
7988
7989 format %{ "MEMBAR-store-store" %}
7990
7991 ins_encode %{
7992 __ membar(Assembler::StoreStore);
7993 %}
7994 ins_pipe(pipe_serial);
7995 %}
7996
7997 instruct membar_release_lock() %{
7998 match(MemBarReleaseLock);
7999 ins_cost(VOLATILE_REF_COST);
8000
8001 format %{ "membar_release_lock (elided)" %}
8002
8003 ins_encode %{
8004 __ block_comment("membar_release_lock (elided)");
8005 %}
8006
8007 ins_pipe(pipe_serial);
8008 %}
8009
8010 instruct unnecessary_membar_volatile() %{
8011 predicate(unnecessary_volatile(n));
8012 match(MemBarVolatile);
8013 ins_cost(0);
8014
8015 format %{ "membar_volatile (elided)" %}
8016
8017 ins_encode %{
8018 __ block_comment("membar_volatile (elided)");
8019 %}
8020
8021 ins_pipe(pipe_serial);
8022 %}
8023
8024 instruct membar_volatile() %{
8025 match(MemBarVolatile);
8026 ins_cost(VOLATILE_REF_COST*100);
8027
8028 format %{ "membar_volatile\n\t"
8029 "dmb ish"%}
8030
8031 ins_encode %{
8032 __ block_comment("membar_volatile");
8033 __ membar(Assembler::StoreLoad);
8034 %}
8035
8036 ins_pipe(pipe_serial);
8037 %}
8038
8039 // ============================================================================
8040 // Cast/Convert Instructions
8041
8042 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8043 match(Set dst (CastX2P src));
8044
8045 ins_cost(INSN_COST);
8046 format %{ "mov $dst, $src\t# long -> ptr" %}
8047
8048 ins_encode %{
8049 if ($dst$$reg != $src$$reg) {
8050 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8051 }
8052 %}
8053
8054 ins_pipe(ialu_reg);
8055 %}
8056
8057 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8058 match(Set dst (CastI2N src));
8059
8060 ins_cost(INSN_COST);
8061 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8062
8063 ins_encode %{
8064 if ($dst$$reg != $src$$reg) {
8065 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8066 }
8067 %}
8068
8069 ins_pipe(ialu_reg);
8070 %}
8071
8072 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8073 match(Set dst (CastP2X src));
8074
8075 ins_cost(INSN_COST);
8076 format %{ "mov $dst, $src\t# ptr -> long" %}
8077
8078 ins_encode %{
8079 if ($dst$$reg != $src$$reg) {
8080 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8081 }
8082 %}
8083
8084 ins_pipe(ialu_reg);
8085 %}
8086
8087 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8088 match(Set dst (CastP2X src));
8089
8090 ins_cost(INSN_COST);
8091 format %{ "mov $dst, $src\t# ptr -> long" %}
8092
8093 ins_encode %{
8094 if ($dst$$reg != $src$$reg) {
8095 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8096 }
8097 %}
8098
8099 ins_pipe(ialu_reg);
8100 %}
8101
8102 // Convert oop into int for vectors alignment masking
8103 instruct convP2I(iRegINoSp dst, iRegP src) %{
8104 match(Set dst (ConvL2I (CastP2X src)));
8105
8106 ins_cost(INSN_COST);
8107 format %{ "movw $dst, $src\t# ptr -> int" %}
8108 ins_encode %{
8109 __ movw($dst$$Register, $src$$Register);
8110 %}
8111
8112 ins_pipe(ialu_reg);
8113 %}
8114
8115 // Convert compressed oop into int for vectors alignment masking
8116 // in case of 32bit oops (heap < 4Gb).
8117 instruct convN2I(iRegINoSp dst, iRegN src)
8118 %{
8119 predicate(CompressedOops::shift() == 0);
8120 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8121
8122 ins_cost(INSN_COST);
8123 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8124 ins_encode %{
8125 __ movw($dst$$Register, $src$$Register);
8126 %}
8127
8128 ins_pipe(ialu_reg);
8129 %}
8130
8131
8132 // Convert oop pointer into compressed form
8133 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8134 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8135 match(Set dst (EncodeP src));
8136 effect(KILL cr);
8137 ins_cost(INSN_COST * 3);
8138 format %{ "encode_heap_oop $dst, $src" %}
8139 ins_encode %{
8140 Register s = $src$$Register;
8141 Register d = $dst$$Register;
8142 __ encode_heap_oop(d, s);
8143 %}
8144 ins_pipe(ialu_reg);
8145 %}
8146
8147 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8148 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8149 match(Set dst (EncodeP src));
8150 ins_cost(INSN_COST * 3);
8151 format %{ "encode_heap_oop_not_null $dst, $src" %}
8152 ins_encode %{
8153 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8154 %}
8155 ins_pipe(ialu_reg);
8156 %}
8157
8158 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8159 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8160 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8161 match(Set dst (DecodeN src));
8162 ins_cost(INSN_COST * 3);
8163 format %{ "decode_heap_oop $dst, $src" %}
8164 ins_encode %{
8165 Register s = $src$$Register;
8166 Register d = $dst$$Register;
8167 __ decode_heap_oop(d, s);
8168 %}
8169 ins_pipe(ialu_reg);
8170 %}
8171
8172 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8173 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8174 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8175 match(Set dst (DecodeN src));
8176 ins_cost(INSN_COST * 3);
8177 format %{ "decode_heap_oop_not_null $dst, $src" %}
8178 ins_encode %{
8179 Register s = $src$$Register;
8180 Register d = $dst$$Register;
8181 __ decode_heap_oop_not_null(d, s);
8182 %}
8183 ins_pipe(ialu_reg);
8184 %}
8185
8186 // n.b. AArch64 implementations of encode_klass_not_null and
8187 // decode_klass_not_null do not modify the flags register so, unlike
8188 // Intel, we don't kill CR as a side effect here
8189
8190 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8191 match(Set dst (EncodePKlass src));
8192
8193 ins_cost(INSN_COST * 3);
8194 format %{ "encode_klass_not_null $dst,$src" %}
8195
8196 ins_encode %{
8197 Register src_reg = as_Register($src$$reg);
8198 Register dst_reg = as_Register($dst$$reg);
8199 __ encode_klass_not_null(dst_reg, src_reg);
8200 %}
8201
8202 ins_pipe(ialu_reg);
8203 %}
8204
8205 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8206 match(Set dst (DecodeNKlass src));
8207
8208 ins_cost(INSN_COST * 3);
8209 format %{ "decode_klass_not_null $dst,$src" %}
8210
8211 ins_encode %{
8212 Register src_reg = as_Register($src$$reg);
8213 Register dst_reg = as_Register($dst$$reg);
8214 if (dst_reg != src_reg) {
8215 __ decode_klass_not_null(dst_reg, src_reg);
8216 } else {
8217 __ decode_klass_not_null(dst_reg);
8218 }
8219 %}
8220
8221 ins_pipe(ialu_reg);
8222 %}
8223
8224 instruct checkCastPP(iRegPNoSp dst)
8225 %{
8226 match(Set dst (CheckCastPP dst));
8227
8228 size(0);
8229 format %{ "# checkcastPP of $dst" %}
8230 ins_encode(/* empty encoding */);
8231 ins_pipe(pipe_class_empty);
8232 %}
8233
8234 instruct castPP(iRegPNoSp dst)
8235 %{
8236 match(Set dst (CastPP dst));
8237
8238 size(0);
8239 format %{ "# castPP of $dst" %}
8240 ins_encode(/* empty encoding */);
8241 ins_pipe(pipe_class_empty);
8242 %}
8243
8244 instruct castII(iRegI dst)
8245 %{
8246 predicate(VerifyConstraintCasts == 0);
8247 match(Set dst (CastII dst));
8248
8249 size(0);
8250 format %{ "# castII of $dst" %}
8251 ins_encode(/* empty encoding */);
8252 ins_cost(0);
8253 ins_pipe(pipe_class_empty);
8254 %}
8255
8256 instruct castII_checked(iRegI dst, rFlagsReg cr)
8257 %{
8258 predicate(VerifyConstraintCasts > 0);
8259 match(Set dst (CastII dst));
8260 effect(KILL cr);
8261
8262 format %{ "# castII_checked of $dst" %}
8263 ins_encode %{
8264 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8265 %}
8266 ins_pipe(pipe_slow);
8267 %}
8268
8269 instruct castLL(iRegL dst)
8270 %{
8271 predicate(VerifyConstraintCasts == 0);
8272 match(Set dst (CastLL dst));
8273
8274 size(0);
8275 format %{ "# castLL of $dst" %}
8276 ins_encode(/* empty encoding */);
8277 ins_cost(0);
8278 ins_pipe(pipe_class_empty);
8279 %}
8280
8281 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8282 %{
8283 predicate(VerifyConstraintCasts > 0);
8284 match(Set dst (CastLL dst));
8285 effect(KILL cr);
8286
8287 format %{ "# castLL_checked of $dst" %}
8288 ins_encode %{
8289 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8290 %}
8291 ins_pipe(pipe_slow);
8292 %}
8293
8294 instruct castHH(vRegF dst)
8295 %{
8296 match(Set dst (CastHH dst));
8297 size(0);
8298 format %{ "# castHH of $dst" %}
8299 ins_encode(/* empty encoding */);
8300 ins_cost(0);
8301 ins_pipe(pipe_class_empty);
8302 %}
8303
8304 instruct castFF(vRegF dst)
8305 %{
8306 match(Set dst (CastFF dst));
8307
8308 size(0);
8309 format %{ "# castFF of $dst" %}
8310 ins_encode(/* empty encoding */);
8311 ins_cost(0);
8312 ins_pipe(pipe_class_empty);
8313 %}
8314
8315 instruct castDD(vRegD dst)
8316 %{
8317 match(Set dst (CastDD dst));
8318
8319 size(0);
8320 format %{ "# castDD of $dst" %}
8321 ins_encode(/* empty encoding */);
8322 ins_cost(0);
8323 ins_pipe(pipe_class_empty);
8324 %}
8325
8326 instruct castVV(vReg dst)
8327 %{
8328 match(Set dst (CastVV dst));
8329
8330 size(0);
8331 format %{ "# castVV of $dst" %}
8332 ins_encode(/* empty encoding */);
8333 ins_cost(0);
8334 ins_pipe(pipe_class_empty);
8335 %}
8336
8337 instruct castVVMask(pRegGov dst)
8338 %{
8339 match(Set dst (CastVV dst));
8340
8341 size(0);
8342 format %{ "# castVV of $dst" %}
8343 ins_encode(/* empty encoding */);
8344 ins_cost(0);
8345 ins_pipe(pipe_class_empty);
8346 %}
8347
8348 // ============================================================================
8349 // Atomic operation instructions
8350 //
8351
8352 // standard CompareAndSwapX when we are using barriers
8353 // these have higher priority than the rules selected by a predicate
8354
8355 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8356 // can't match them
8357
8358 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8359
8360 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8361 ins_cost(2 * VOLATILE_REF_COST);
8362
8363 effect(KILL cr);
8364
8365 format %{
8366 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8367 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8368 %}
8369
8370 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8371 aarch64_enc_cset_eq(res));
8372
8373 ins_pipe(pipe_slow);
8374 %}
8375
8376 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8377
8378 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8379 ins_cost(2 * VOLATILE_REF_COST);
8380
8381 effect(KILL cr);
8382
8383 format %{
8384 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8385 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8386 %}
8387
8388 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8389 aarch64_enc_cset_eq(res));
8390
8391 ins_pipe(pipe_slow);
8392 %}
8393
8394 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8395
8396 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8397 ins_cost(2 * VOLATILE_REF_COST);
8398
8399 effect(KILL cr);
8400
8401 format %{
8402 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8403 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8404 %}
8405
8406 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8407 aarch64_enc_cset_eq(res));
8408
8409 ins_pipe(pipe_slow);
8410 %}
8411
8412 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8413
8414 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8415 ins_cost(2 * VOLATILE_REF_COST);
8416
8417 effect(KILL cr);
8418
8419 format %{
8420 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8421 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8422 %}
8423
8424 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8425 aarch64_enc_cset_eq(res));
8426
8427 ins_pipe(pipe_slow);
8428 %}
8429
8430 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8431
8432 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8433 predicate(n->as_LoadStore()->barrier_data() == 0);
8434 ins_cost(2 * VOLATILE_REF_COST);
8435
8436 effect(KILL cr);
8437
8438 format %{
8439 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8440 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8441 %}
8442
8443 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8444 aarch64_enc_cset_eq(res));
8445
8446 ins_pipe(pipe_slow);
8447 %}
8448
8449 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8450
8451 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8452 predicate(n->as_LoadStore()->barrier_data() == 0);
8453 ins_cost(2 * VOLATILE_REF_COST);
8454
8455 effect(KILL cr);
8456
8457 format %{
8458 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8459 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8460 %}
8461
8462 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8463 aarch64_enc_cset_eq(res));
8464
8465 ins_pipe(pipe_slow);
8466 %}
8467
8468 // alternative CompareAndSwapX when we are eliding barriers
8469
8470 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8471
8472 predicate(needs_acquiring_load_exclusive(n));
8473 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8474 ins_cost(VOLATILE_REF_COST);
8475
8476 effect(KILL cr);
8477
8478 format %{
8479 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8480 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8481 %}
8482
8483 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8484 aarch64_enc_cset_eq(res));
8485
8486 ins_pipe(pipe_slow);
8487 %}
8488
8489 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8490
8491 predicate(needs_acquiring_load_exclusive(n));
8492 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8493 ins_cost(VOLATILE_REF_COST);
8494
8495 effect(KILL cr);
8496
8497 format %{
8498 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8499 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8500 %}
8501
8502 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8503 aarch64_enc_cset_eq(res));
8504
8505 ins_pipe(pipe_slow);
8506 %}
8507
8508 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8509
8510 predicate(needs_acquiring_load_exclusive(n));
8511 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8512 ins_cost(VOLATILE_REF_COST);
8513
8514 effect(KILL cr);
8515
8516 format %{
8517 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8518 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8519 %}
8520
8521 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8522 aarch64_enc_cset_eq(res));
8523
8524 ins_pipe(pipe_slow);
8525 %}
8526
8527 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8528
8529 predicate(needs_acquiring_load_exclusive(n));
8530 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8531 ins_cost(VOLATILE_REF_COST);
8532
8533 effect(KILL cr);
8534
8535 format %{
8536 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8537 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8538 %}
8539
8540 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8541 aarch64_enc_cset_eq(res));
8542
8543 ins_pipe(pipe_slow);
8544 %}
8545
8546 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8547
8548 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8549 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8550 ins_cost(VOLATILE_REF_COST);
8551
8552 effect(KILL cr);
8553
8554 format %{
8555 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8556 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8557 %}
8558
8559 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8560 aarch64_enc_cset_eq(res));
8561
8562 ins_pipe(pipe_slow);
8563 %}
8564
8565 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8566
8567 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8568 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8569 ins_cost(VOLATILE_REF_COST);
8570
8571 effect(KILL cr);
8572
8573 format %{
8574 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8575 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8576 %}
8577
8578 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8579 aarch64_enc_cset_eq(res));
8580
8581 ins_pipe(pipe_slow);
8582 %}
8583
8584
8585 // ---------------------------------------------------------------------
8586
8587 // BEGIN This section of the file is automatically generated. Do not edit --------------
8588
8589 // Sundry CAS operations. Note that release is always true,
8590 // regardless of the memory ordering of the CAS. This is because we
8591 // need the volatile case to be sequentially consistent but there is
8592 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8593 // can't check the type of memory ordering here, so we always emit a
8594 // STLXR.
8595
8596 // This section is generated from cas.m4
8597
8598
8599 // This pattern is generated automatically from cas.m4.
8600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8601 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8602 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8603 ins_cost(2 * VOLATILE_REF_COST);
8604 effect(TEMP_DEF res, KILL cr);
8605 format %{
8606 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8607 %}
8608 ins_encode %{
8609 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8610 Assembler::byte, /*acquire*/ false, /*release*/ true,
8611 /*weak*/ false, $res$$Register);
8612 __ sxtbw($res$$Register, $res$$Register);
8613 %}
8614 ins_pipe(pipe_slow);
8615 %}
8616
8617 // This pattern is generated automatically from cas.m4.
8618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8619 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8620 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8621 ins_cost(2 * VOLATILE_REF_COST);
8622 effect(TEMP_DEF res, KILL cr);
8623 format %{
8624 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8625 %}
8626 ins_encode %{
8627 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8628 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8629 /*weak*/ false, $res$$Register);
8630 __ sxthw($res$$Register, $res$$Register);
8631 %}
8632 ins_pipe(pipe_slow);
8633 %}
8634
8635 // This pattern is generated automatically from cas.m4.
8636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8637 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8638 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8639 ins_cost(2 * VOLATILE_REF_COST);
8640 effect(TEMP_DEF res, KILL cr);
8641 format %{
8642 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8643 %}
8644 ins_encode %{
8645 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8646 Assembler::word, /*acquire*/ false, /*release*/ true,
8647 /*weak*/ false, $res$$Register);
8648 %}
8649 ins_pipe(pipe_slow);
8650 %}
8651
8652 // This pattern is generated automatically from cas.m4.
8653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8654 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8655 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8656 ins_cost(2 * VOLATILE_REF_COST);
8657 effect(TEMP_DEF res, KILL cr);
8658 format %{
8659 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8660 %}
8661 ins_encode %{
8662 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8663 Assembler::xword, /*acquire*/ false, /*release*/ true,
8664 /*weak*/ false, $res$$Register);
8665 %}
8666 ins_pipe(pipe_slow);
8667 %}
8668
8669 // This pattern is generated automatically from cas.m4.
8670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8671 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8672 predicate(n->as_LoadStore()->barrier_data() == 0);
8673 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8674 ins_cost(2 * VOLATILE_REF_COST);
8675 effect(TEMP_DEF res, KILL cr);
8676 format %{
8677 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8678 %}
8679 ins_encode %{
8680 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8681 Assembler::word, /*acquire*/ false, /*release*/ true,
8682 /*weak*/ false, $res$$Register);
8683 %}
8684 ins_pipe(pipe_slow);
8685 %}
8686
8687 // This pattern is generated automatically from cas.m4.
8688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8689 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8690 predicate(n->as_LoadStore()->barrier_data() == 0);
8691 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8692 ins_cost(2 * VOLATILE_REF_COST);
8693 effect(TEMP_DEF res, KILL cr);
8694 format %{
8695 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8696 %}
8697 ins_encode %{
8698 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8699 Assembler::xword, /*acquire*/ false, /*release*/ true,
8700 /*weak*/ false, $res$$Register);
8701 %}
8702 ins_pipe(pipe_slow);
8703 %}
8704
8705 // This pattern is generated automatically from cas.m4.
8706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8707 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8708 predicate(needs_acquiring_load_exclusive(n));
8709 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8710 ins_cost(VOLATILE_REF_COST);
8711 effect(TEMP_DEF res, KILL cr);
8712 format %{
8713 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8714 %}
8715 ins_encode %{
8716 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8717 Assembler::byte, /*acquire*/ true, /*release*/ true,
8718 /*weak*/ false, $res$$Register);
8719 __ sxtbw($res$$Register, $res$$Register);
8720 %}
8721 ins_pipe(pipe_slow);
8722 %}
8723
8724 // This pattern is generated automatically from cas.m4.
8725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8726 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8727 predicate(needs_acquiring_load_exclusive(n));
8728 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8729 ins_cost(VOLATILE_REF_COST);
8730 effect(TEMP_DEF res, KILL cr);
8731 format %{
8732 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8733 %}
8734 ins_encode %{
8735 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8736 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8737 /*weak*/ false, $res$$Register);
8738 __ sxthw($res$$Register, $res$$Register);
8739 %}
8740 ins_pipe(pipe_slow);
8741 %}
8742
8743 // This pattern is generated automatically from cas.m4.
8744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8745 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8746 predicate(needs_acquiring_load_exclusive(n));
8747 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8748 ins_cost(VOLATILE_REF_COST);
8749 effect(TEMP_DEF res, KILL cr);
8750 format %{
8751 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8752 %}
8753 ins_encode %{
8754 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8755 Assembler::word, /*acquire*/ true, /*release*/ true,
8756 /*weak*/ false, $res$$Register);
8757 %}
8758 ins_pipe(pipe_slow);
8759 %}
8760
8761 // This pattern is generated automatically from cas.m4.
8762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8763 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8764 predicate(needs_acquiring_load_exclusive(n));
8765 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8766 ins_cost(VOLATILE_REF_COST);
8767 effect(TEMP_DEF res, KILL cr);
8768 format %{
8769 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8770 %}
8771 ins_encode %{
8772 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8773 Assembler::xword, /*acquire*/ true, /*release*/ true,
8774 /*weak*/ false, $res$$Register);
8775 %}
8776 ins_pipe(pipe_slow);
8777 %}
8778
8779 // This pattern is generated automatically from cas.m4.
8780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8781 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8782 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8783 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8784 ins_cost(VOLATILE_REF_COST);
8785 effect(TEMP_DEF res, KILL cr);
8786 format %{
8787 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8788 %}
8789 ins_encode %{
8790 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8791 Assembler::word, /*acquire*/ true, /*release*/ true,
8792 /*weak*/ false, $res$$Register);
8793 %}
8794 ins_pipe(pipe_slow);
8795 %}
8796
8797 // This pattern is generated automatically from cas.m4.
8798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8799 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8800 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8801 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8802 ins_cost(VOLATILE_REF_COST);
8803 effect(TEMP_DEF res, KILL cr);
8804 format %{
8805 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8806 %}
8807 ins_encode %{
8808 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8809 Assembler::xword, /*acquire*/ true, /*release*/ true,
8810 /*weak*/ false, $res$$Register);
8811 %}
8812 ins_pipe(pipe_slow);
8813 %}
8814
8815 // This pattern is generated automatically from cas.m4.
8816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8817 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8818 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8819 ins_cost(2 * VOLATILE_REF_COST);
8820 effect(KILL cr);
8821 format %{
8822 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8823 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8824 %}
8825 ins_encode %{
8826 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8827 Assembler::byte, /*acquire*/ false, /*release*/ true,
8828 /*weak*/ true, noreg);
8829 __ csetw($res$$Register, Assembler::EQ);
8830 %}
8831 ins_pipe(pipe_slow);
8832 %}
8833
8834 // This pattern is generated automatically from cas.m4.
8835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8836 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8837 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8838 ins_cost(2 * VOLATILE_REF_COST);
8839 effect(KILL cr);
8840 format %{
8841 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8842 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8843 %}
8844 ins_encode %{
8845 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8846 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8847 /*weak*/ true, noreg);
8848 __ csetw($res$$Register, Assembler::EQ);
8849 %}
8850 ins_pipe(pipe_slow);
8851 %}
8852
8853 // This pattern is generated automatically from cas.m4.
8854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8855 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8856 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8857 ins_cost(2 * VOLATILE_REF_COST);
8858 effect(KILL cr);
8859 format %{
8860 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8861 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8862 %}
8863 ins_encode %{
8864 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8865 Assembler::word, /*acquire*/ false, /*release*/ true,
8866 /*weak*/ true, noreg);
8867 __ csetw($res$$Register, Assembler::EQ);
8868 %}
8869 ins_pipe(pipe_slow);
8870 %}
8871
8872 // This pattern is generated automatically from cas.m4.
8873 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8874 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8875 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8876 ins_cost(2 * VOLATILE_REF_COST);
8877 effect(KILL cr);
8878 format %{
8879 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8880 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8881 %}
8882 ins_encode %{
8883 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8884 Assembler::xword, /*acquire*/ false, /*release*/ true,
8885 /*weak*/ true, noreg);
8886 __ csetw($res$$Register, Assembler::EQ);
8887 %}
8888 ins_pipe(pipe_slow);
8889 %}
8890
8891 // This pattern is generated automatically from cas.m4.
8892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8893 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8894 predicate(n->as_LoadStore()->barrier_data() == 0);
8895 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8896 ins_cost(2 * VOLATILE_REF_COST);
8897 effect(KILL cr);
8898 format %{
8899 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8900 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8901 %}
8902 ins_encode %{
8903 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8904 Assembler::word, /*acquire*/ false, /*release*/ true,
8905 /*weak*/ true, noreg);
8906 __ csetw($res$$Register, Assembler::EQ);
8907 %}
8908 ins_pipe(pipe_slow);
8909 %}
8910
8911 // This pattern is generated automatically from cas.m4.
8912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8913 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8914 predicate(n->as_LoadStore()->barrier_data() == 0);
8915 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8916 ins_cost(2 * VOLATILE_REF_COST);
8917 effect(KILL cr);
8918 format %{
8919 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8920 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8921 %}
8922 ins_encode %{
8923 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8924 Assembler::xword, /*acquire*/ false, /*release*/ true,
8925 /*weak*/ true, noreg);
8926 __ csetw($res$$Register, Assembler::EQ);
8927 %}
8928 ins_pipe(pipe_slow);
8929 %}
8930
8931 // This pattern is generated automatically from cas.m4.
8932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8933 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8934 predicate(needs_acquiring_load_exclusive(n));
8935 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8936 ins_cost(VOLATILE_REF_COST);
8937 effect(KILL cr);
8938 format %{
8939 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8940 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8941 %}
8942 ins_encode %{
8943 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8944 Assembler::byte, /*acquire*/ true, /*release*/ true,
8945 /*weak*/ true, noreg);
8946 __ csetw($res$$Register, Assembler::EQ);
8947 %}
8948 ins_pipe(pipe_slow);
8949 %}
8950
8951 // This pattern is generated automatically from cas.m4.
8952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8953 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8954 predicate(needs_acquiring_load_exclusive(n));
8955 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8956 ins_cost(VOLATILE_REF_COST);
8957 effect(KILL cr);
8958 format %{
8959 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8960 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8961 %}
8962 ins_encode %{
8963 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8964 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8965 /*weak*/ true, noreg);
8966 __ csetw($res$$Register, Assembler::EQ);
8967 %}
8968 ins_pipe(pipe_slow);
8969 %}
8970
8971 // This pattern is generated automatically from cas.m4.
8972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8973 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8974 predicate(needs_acquiring_load_exclusive(n));
8975 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8976 ins_cost(VOLATILE_REF_COST);
8977 effect(KILL cr);
8978 format %{
8979 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8980 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8981 %}
8982 ins_encode %{
8983 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8984 Assembler::word, /*acquire*/ true, /*release*/ true,
8985 /*weak*/ true, noreg);
8986 __ csetw($res$$Register, Assembler::EQ);
8987 %}
8988 ins_pipe(pipe_slow);
8989 %}
8990
8991 // This pattern is generated automatically from cas.m4.
8992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8993 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8994 predicate(needs_acquiring_load_exclusive(n));
8995 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8996 ins_cost(VOLATILE_REF_COST);
8997 effect(KILL cr);
8998 format %{
8999 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9000 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9001 %}
9002 ins_encode %{
9003 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9004 Assembler::xword, /*acquire*/ true, /*release*/ true,
9005 /*weak*/ true, noreg);
9006 __ csetw($res$$Register, Assembler::EQ);
9007 %}
9008 ins_pipe(pipe_slow);
9009 %}
9010
9011 // This pattern is generated automatically from cas.m4.
9012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9013 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9014 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9015 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9016 ins_cost(VOLATILE_REF_COST);
9017 effect(KILL cr);
9018 format %{
9019 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9020 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9021 %}
9022 ins_encode %{
9023 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9024 Assembler::word, /*acquire*/ true, /*release*/ true,
9025 /*weak*/ true, noreg);
9026 __ csetw($res$$Register, Assembler::EQ);
9027 %}
9028 ins_pipe(pipe_slow);
9029 %}
9030
9031 // This pattern is generated automatically from cas.m4.
9032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9033 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9034 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9035 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9036 ins_cost(VOLATILE_REF_COST);
9037 effect(KILL cr);
9038 format %{
9039 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9040 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9041 %}
9042 ins_encode %{
9043 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9044 Assembler::xword, /*acquire*/ true, /*release*/ true,
9045 /*weak*/ true, noreg);
9046 __ csetw($res$$Register, Assembler::EQ);
9047 %}
9048 ins_pipe(pipe_slow);
9049 %}
9050
9051 // END This section of the file is automatically generated. Do not edit --------------
9052 // ---------------------------------------------------------------------
9053
9054 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9055 match(Set prev (GetAndSetI mem newv));
9056 ins_cost(2 * VOLATILE_REF_COST);
9057 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9058 ins_encode %{
9059 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9060 %}
9061 ins_pipe(pipe_serial);
9062 %}
9063
9064 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9065 match(Set prev (GetAndSetL mem newv));
9066 ins_cost(2 * VOLATILE_REF_COST);
9067 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9068 ins_encode %{
9069 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9070 %}
9071 ins_pipe(pipe_serial);
9072 %}
9073
9074 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9075 predicate(n->as_LoadStore()->barrier_data() == 0);
9076 match(Set prev (GetAndSetN mem newv));
9077 ins_cost(2 * VOLATILE_REF_COST);
9078 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9079 ins_encode %{
9080 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9081 %}
9082 ins_pipe(pipe_serial);
9083 %}
9084
9085 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9086 predicate(n->as_LoadStore()->barrier_data() == 0);
9087 match(Set prev (GetAndSetP mem newv));
9088 ins_cost(2 * VOLATILE_REF_COST);
9089 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9090 ins_encode %{
9091 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9092 %}
9093 ins_pipe(pipe_serial);
9094 %}
9095
9096 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9097 predicate(needs_acquiring_load_exclusive(n));
9098 match(Set prev (GetAndSetI mem newv));
9099 ins_cost(VOLATILE_REF_COST);
9100 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9101 ins_encode %{
9102 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9103 %}
9104 ins_pipe(pipe_serial);
9105 %}
9106
9107 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9108 predicate(needs_acquiring_load_exclusive(n));
9109 match(Set prev (GetAndSetL mem newv));
9110 ins_cost(VOLATILE_REF_COST);
9111 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9112 ins_encode %{
9113 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9114 %}
9115 ins_pipe(pipe_serial);
9116 %}
9117
9118 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9119 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9120 match(Set prev (GetAndSetN mem newv));
9121 ins_cost(VOLATILE_REF_COST);
9122 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9123 ins_encode %{
9124 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9125 %}
9126 ins_pipe(pipe_serial);
9127 %}
9128
9129 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9130 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9131 match(Set prev (GetAndSetP mem newv));
9132 ins_cost(VOLATILE_REF_COST);
9133 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9134 ins_encode %{
9135 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9136 %}
9137 ins_pipe(pipe_serial);
9138 %}
9139
9140
9141 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9142 match(Set newval (GetAndAddL mem incr));
9143 ins_cost(2 * VOLATILE_REF_COST + 1);
9144 format %{ "get_and_addL $newval, [$mem], $incr" %}
9145 ins_encode %{
9146 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9147 %}
9148 ins_pipe(pipe_serial);
9149 %}
9150
9151 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9152 predicate(n->as_LoadStore()->result_not_used());
9153 match(Set dummy (GetAndAddL mem incr));
9154 ins_cost(2 * VOLATILE_REF_COST);
9155 format %{ "get_and_addL [$mem], $incr" %}
9156 ins_encode %{
9157 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9158 %}
9159 ins_pipe(pipe_serial);
9160 %}
9161
9162 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9163 match(Set newval (GetAndAddL mem incr));
9164 ins_cost(2 * VOLATILE_REF_COST + 1);
9165 format %{ "get_and_addL $newval, [$mem], $incr" %}
9166 ins_encode %{
9167 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9168 %}
9169 ins_pipe(pipe_serial);
9170 %}
9171
9172 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9173 predicate(n->as_LoadStore()->result_not_used());
9174 match(Set dummy (GetAndAddL mem incr));
9175 ins_cost(2 * VOLATILE_REF_COST);
9176 format %{ "get_and_addL [$mem], $incr" %}
9177 ins_encode %{
9178 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9179 %}
9180 ins_pipe(pipe_serial);
9181 %}
9182
9183 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9184 match(Set newval (GetAndAddI mem incr));
9185 ins_cost(2 * VOLATILE_REF_COST + 1);
9186 format %{ "get_and_addI $newval, [$mem], $incr" %}
9187 ins_encode %{
9188 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9189 %}
9190 ins_pipe(pipe_serial);
9191 %}
9192
9193 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9194 predicate(n->as_LoadStore()->result_not_used());
9195 match(Set dummy (GetAndAddI mem incr));
9196 ins_cost(2 * VOLATILE_REF_COST);
9197 format %{ "get_and_addI [$mem], $incr" %}
9198 ins_encode %{
9199 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9200 %}
9201 ins_pipe(pipe_serial);
9202 %}
9203
9204 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9205 match(Set newval (GetAndAddI mem incr));
9206 ins_cost(2 * VOLATILE_REF_COST + 1);
9207 format %{ "get_and_addI $newval, [$mem], $incr" %}
9208 ins_encode %{
9209 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9210 %}
9211 ins_pipe(pipe_serial);
9212 %}
9213
9214 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9215 predicate(n->as_LoadStore()->result_not_used());
9216 match(Set dummy (GetAndAddI mem incr));
9217 ins_cost(2 * VOLATILE_REF_COST);
9218 format %{ "get_and_addI [$mem], $incr" %}
9219 ins_encode %{
9220 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9221 %}
9222 ins_pipe(pipe_serial);
9223 %}
9224
9225 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9226 predicate(needs_acquiring_load_exclusive(n));
9227 match(Set newval (GetAndAddL mem incr));
9228 ins_cost(VOLATILE_REF_COST + 1);
9229 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9230 ins_encode %{
9231 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9232 %}
9233 ins_pipe(pipe_serial);
9234 %}
9235
9236 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9237 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9238 match(Set dummy (GetAndAddL mem incr));
9239 ins_cost(VOLATILE_REF_COST);
9240 format %{ "get_and_addL_acq [$mem], $incr" %}
9241 ins_encode %{
9242 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9243 %}
9244 ins_pipe(pipe_serial);
9245 %}
9246
9247 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9248 predicate(needs_acquiring_load_exclusive(n));
9249 match(Set newval (GetAndAddL mem incr));
9250 ins_cost(VOLATILE_REF_COST + 1);
9251 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9252 ins_encode %{
9253 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9254 %}
9255 ins_pipe(pipe_serial);
9256 %}
9257
9258 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9259 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9260 match(Set dummy (GetAndAddL mem incr));
9261 ins_cost(VOLATILE_REF_COST);
9262 format %{ "get_and_addL_acq [$mem], $incr" %}
9263 ins_encode %{
9264 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9265 %}
9266 ins_pipe(pipe_serial);
9267 %}
9268
9269 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9270 predicate(needs_acquiring_load_exclusive(n));
9271 match(Set newval (GetAndAddI mem incr));
9272 ins_cost(VOLATILE_REF_COST + 1);
9273 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9274 ins_encode %{
9275 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9276 %}
9277 ins_pipe(pipe_serial);
9278 %}
9279
9280 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9281 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9282 match(Set dummy (GetAndAddI mem incr));
9283 ins_cost(VOLATILE_REF_COST);
9284 format %{ "get_and_addI_acq [$mem], $incr" %}
9285 ins_encode %{
9286 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9287 %}
9288 ins_pipe(pipe_serial);
9289 %}
9290
9291 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9292 predicate(needs_acquiring_load_exclusive(n));
9293 match(Set newval (GetAndAddI mem incr));
9294 ins_cost(VOLATILE_REF_COST + 1);
9295 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9296 ins_encode %{
9297 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9298 %}
9299 ins_pipe(pipe_serial);
9300 %}
9301
9302 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9303 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9304 match(Set dummy (GetAndAddI mem incr));
9305 ins_cost(VOLATILE_REF_COST);
9306 format %{ "get_and_addI_acq [$mem], $incr" %}
9307 ins_encode %{
9308 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9309 %}
9310 ins_pipe(pipe_serial);
9311 %}
9312
9313 // Manifest a CmpU result in an integer register.
9314 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9315 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9316 %{
9317 match(Set dst (CmpU3 src1 src2));
9318 effect(KILL flags);
9319
9320 ins_cost(INSN_COST * 3);
9321 format %{
9322 "cmpw $src1, $src2\n\t"
9323 "csetw $dst, ne\n\t"
9324 "cnegw $dst, lo\t# CmpU3(reg)"
9325 %}
9326 ins_encode %{
9327 __ cmpw($src1$$Register, $src2$$Register);
9328 __ csetw($dst$$Register, Assembler::NE);
9329 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9330 %}
9331
9332 ins_pipe(pipe_class_default);
9333 %}
9334
9335 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9336 %{
9337 match(Set dst (CmpU3 src1 src2));
9338 effect(KILL flags);
9339
9340 ins_cost(INSN_COST * 3);
9341 format %{
9342 "subsw zr, $src1, $src2\n\t"
9343 "csetw $dst, ne\n\t"
9344 "cnegw $dst, lo\t# CmpU3(imm)"
9345 %}
9346 ins_encode %{
9347 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9348 __ csetw($dst$$Register, Assembler::NE);
9349 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9350 %}
9351
9352 ins_pipe(pipe_class_default);
9353 %}
9354
9355 // Manifest a CmpUL result in an integer register.
9356 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9357 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9358 %{
9359 match(Set dst (CmpUL3 src1 src2));
9360 effect(KILL flags);
9361
9362 ins_cost(INSN_COST * 3);
9363 format %{
9364 "cmp $src1, $src2\n\t"
9365 "csetw $dst, ne\n\t"
9366 "cnegw $dst, lo\t# CmpUL3(reg)"
9367 %}
9368 ins_encode %{
9369 __ cmp($src1$$Register, $src2$$Register);
9370 __ csetw($dst$$Register, Assembler::NE);
9371 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9372 %}
9373
9374 ins_pipe(pipe_class_default);
9375 %}
9376
9377 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9378 %{
9379 match(Set dst (CmpUL3 src1 src2));
9380 effect(KILL flags);
9381
9382 ins_cost(INSN_COST * 3);
9383 format %{
9384 "subs zr, $src1, $src2\n\t"
9385 "csetw $dst, ne\n\t"
9386 "cnegw $dst, lo\t# CmpUL3(imm)"
9387 %}
9388 ins_encode %{
9389 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9390 __ csetw($dst$$Register, Assembler::NE);
9391 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9392 %}
9393
9394 ins_pipe(pipe_class_default);
9395 %}
9396
9397 // Manifest a CmpL result in an integer register.
9398 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9399 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9400 %{
9401 match(Set dst (CmpL3 src1 src2));
9402 effect(KILL flags);
9403
9404 ins_cost(INSN_COST * 3);
9405 format %{
9406 "cmp $src1, $src2\n\t"
9407 "csetw $dst, ne\n\t"
9408 "cnegw $dst, lt\t# CmpL3(reg)"
9409 %}
9410 ins_encode %{
9411 __ cmp($src1$$Register, $src2$$Register);
9412 __ csetw($dst$$Register, Assembler::NE);
9413 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9414 %}
9415
9416 ins_pipe(pipe_class_default);
9417 %}
9418
9419 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9420 %{
9421 match(Set dst (CmpL3 src1 src2));
9422 effect(KILL flags);
9423
9424 ins_cost(INSN_COST * 3);
9425 format %{
9426 "subs zr, $src1, $src2\n\t"
9427 "csetw $dst, ne\n\t"
9428 "cnegw $dst, lt\t# CmpL3(imm)"
9429 %}
9430 ins_encode %{
9431 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9432 __ csetw($dst$$Register, Assembler::NE);
9433 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9434 %}
9435
9436 ins_pipe(pipe_class_default);
9437 %}
9438
9439 // ============================================================================
9440 // Conditional Move Instructions
9441
9442 // n.b. we have identical rules for both a signed compare op (cmpOp)
9443 // and an unsigned compare op (cmpOpU). it would be nice if we could
9444 // define an op class which merged both inputs and use it to type the
9445 // argument to a single rule. unfortunatelyt his fails because the
9446 // opclass does not live up to the COND_INTER interface of its
9447 // component operands. When the generic code tries to negate the
9448 // operand it ends up running the generci Machoper::negate method
9449 // which throws a ShouldNotHappen. So, we have to provide two flavours
9450 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9451
9452 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9453 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9454
9455 ins_cost(INSN_COST * 2);
9456 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9457
9458 ins_encode %{
9459 __ cselw(as_Register($dst$$reg),
9460 as_Register($src2$$reg),
9461 as_Register($src1$$reg),
9462 (Assembler::Condition)$cmp$$cmpcode);
9463 %}
9464
9465 ins_pipe(icond_reg_reg);
9466 %}
9467
9468 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9469 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9470
9471 ins_cost(INSN_COST * 2);
9472 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9473
9474 ins_encode %{
9475 __ cselw(as_Register($dst$$reg),
9476 as_Register($src2$$reg),
9477 as_Register($src1$$reg),
9478 (Assembler::Condition)$cmp$$cmpcode);
9479 %}
9480
9481 ins_pipe(icond_reg_reg);
9482 %}
9483
9484 // special cases where one arg is zero
9485
9486 // n.b. this is selected in preference to the rule above because it
9487 // avoids loading constant 0 into a source register
9488
9489 // TODO
9490 // we ought only to be able to cull one of these variants as the ideal
9491 // transforms ought always to order the zero consistently (to left/right?)
9492
9493 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9494 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9495
9496 ins_cost(INSN_COST * 2);
9497 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9498
9499 ins_encode %{
9500 __ cselw(as_Register($dst$$reg),
9501 as_Register($src$$reg),
9502 zr,
9503 (Assembler::Condition)$cmp$$cmpcode);
9504 %}
9505
9506 ins_pipe(icond_reg);
9507 %}
9508
9509 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9510 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9511
9512 ins_cost(INSN_COST * 2);
9513 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9514
9515 ins_encode %{
9516 __ cselw(as_Register($dst$$reg),
9517 as_Register($src$$reg),
9518 zr,
9519 (Assembler::Condition)$cmp$$cmpcode);
9520 %}
9521
9522 ins_pipe(icond_reg);
9523 %}
9524
9525 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9526 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9527
9528 ins_cost(INSN_COST * 2);
9529 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9530
9531 ins_encode %{
9532 __ cselw(as_Register($dst$$reg),
9533 zr,
9534 as_Register($src$$reg),
9535 (Assembler::Condition)$cmp$$cmpcode);
9536 %}
9537
9538 ins_pipe(icond_reg);
9539 %}
9540
9541 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9542 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9543
9544 ins_cost(INSN_COST * 2);
9545 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9546
9547 ins_encode %{
9548 __ cselw(as_Register($dst$$reg),
9549 zr,
9550 as_Register($src$$reg),
9551 (Assembler::Condition)$cmp$$cmpcode);
9552 %}
9553
9554 ins_pipe(icond_reg);
9555 %}
9556
9557 // special case for creating a boolean 0 or 1
9558
9559 // n.b. this is selected in preference to the rule above because it
9560 // avoids loading constants 0 and 1 into a source register
9561
9562 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9563 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9564
9565 ins_cost(INSN_COST * 2);
9566 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9567
9568 ins_encode %{
9569 // equivalently
9570 // cset(as_Register($dst$$reg),
9571 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9572 __ csincw(as_Register($dst$$reg),
9573 zr,
9574 zr,
9575 (Assembler::Condition)$cmp$$cmpcode);
9576 %}
9577
9578 ins_pipe(icond_none);
9579 %}
9580
9581 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9582 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9583
9584 ins_cost(INSN_COST * 2);
9585 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9586
9587 ins_encode %{
9588 // equivalently
9589 // cset(as_Register($dst$$reg),
9590 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9591 __ csincw(as_Register($dst$$reg),
9592 zr,
9593 zr,
9594 (Assembler::Condition)$cmp$$cmpcode);
9595 %}
9596
9597 ins_pipe(icond_none);
9598 %}
9599
9600 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9601 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9602
9603 ins_cost(INSN_COST * 2);
9604 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9605
9606 ins_encode %{
9607 __ csel(as_Register($dst$$reg),
9608 as_Register($src2$$reg),
9609 as_Register($src1$$reg),
9610 (Assembler::Condition)$cmp$$cmpcode);
9611 %}
9612
9613 ins_pipe(icond_reg_reg);
9614 %}
9615
9616 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9617 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9618
9619 ins_cost(INSN_COST * 2);
9620 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9621
9622 ins_encode %{
9623 __ csel(as_Register($dst$$reg),
9624 as_Register($src2$$reg),
9625 as_Register($src1$$reg),
9626 (Assembler::Condition)$cmp$$cmpcode);
9627 %}
9628
9629 ins_pipe(icond_reg_reg);
9630 %}
9631
9632 // special cases where one arg is zero
9633
9634 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9635 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9636
9637 ins_cost(INSN_COST * 2);
9638 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9639
9640 ins_encode %{
9641 __ csel(as_Register($dst$$reg),
9642 zr,
9643 as_Register($src$$reg),
9644 (Assembler::Condition)$cmp$$cmpcode);
9645 %}
9646
9647 ins_pipe(icond_reg);
9648 %}
9649
9650 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9651 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9652
9653 ins_cost(INSN_COST * 2);
9654 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9655
9656 ins_encode %{
9657 __ csel(as_Register($dst$$reg),
9658 zr,
9659 as_Register($src$$reg),
9660 (Assembler::Condition)$cmp$$cmpcode);
9661 %}
9662
9663 ins_pipe(icond_reg);
9664 %}
9665
9666 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9667 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9668
9669 ins_cost(INSN_COST * 2);
9670 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9671
9672 ins_encode %{
9673 __ csel(as_Register($dst$$reg),
9674 as_Register($src$$reg),
9675 zr,
9676 (Assembler::Condition)$cmp$$cmpcode);
9677 %}
9678
9679 ins_pipe(icond_reg);
9680 %}
9681
9682 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9683 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9684
9685 ins_cost(INSN_COST * 2);
9686 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9687
9688 ins_encode %{
9689 __ csel(as_Register($dst$$reg),
9690 as_Register($src$$reg),
9691 zr,
9692 (Assembler::Condition)$cmp$$cmpcode);
9693 %}
9694
9695 ins_pipe(icond_reg);
9696 %}
9697
9698 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9699 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9700
9701 ins_cost(INSN_COST * 2);
9702 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9703
9704 ins_encode %{
9705 __ csel(as_Register($dst$$reg),
9706 as_Register($src2$$reg),
9707 as_Register($src1$$reg),
9708 (Assembler::Condition)$cmp$$cmpcode);
9709 %}
9710
9711 ins_pipe(icond_reg_reg);
9712 %}
9713
9714 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9715 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9716
9717 ins_cost(INSN_COST * 2);
9718 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9719
9720 ins_encode %{
9721 __ csel(as_Register($dst$$reg),
9722 as_Register($src2$$reg),
9723 as_Register($src1$$reg),
9724 (Assembler::Condition)$cmp$$cmpcode);
9725 %}
9726
9727 ins_pipe(icond_reg_reg);
9728 %}
9729
9730 // special cases where one arg is zero
9731
9732 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9733 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9734
9735 ins_cost(INSN_COST * 2);
9736 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9737
9738 ins_encode %{
9739 __ csel(as_Register($dst$$reg),
9740 zr,
9741 as_Register($src$$reg),
9742 (Assembler::Condition)$cmp$$cmpcode);
9743 %}
9744
9745 ins_pipe(icond_reg);
9746 %}
9747
9748 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9749 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9750
9751 ins_cost(INSN_COST * 2);
9752 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9753
9754 ins_encode %{
9755 __ csel(as_Register($dst$$reg),
9756 zr,
9757 as_Register($src$$reg),
9758 (Assembler::Condition)$cmp$$cmpcode);
9759 %}
9760
9761 ins_pipe(icond_reg);
9762 %}
9763
9764 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9765 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9766
9767 ins_cost(INSN_COST * 2);
9768 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9769
9770 ins_encode %{
9771 __ csel(as_Register($dst$$reg),
9772 as_Register($src$$reg),
9773 zr,
9774 (Assembler::Condition)$cmp$$cmpcode);
9775 %}
9776
9777 ins_pipe(icond_reg);
9778 %}
9779
9780 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9781 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9782
9783 ins_cost(INSN_COST * 2);
9784 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9785
9786 ins_encode %{
9787 __ csel(as_Register($dst$$reg),
9788 as_Register($src$$reg),
9789 zr,
9790 (Assembler::Condition)$cmp$$cmpcode);
9791 %}
9792
9793 ins_pipe(icond_reg);
9794 %}
9795
9796 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9797 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9798
9799 ins_cost(INSN_COST * 2);
9800 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9801
9802 ins_encode %{
9803 __ cselw(as_Register($dst$$reg),
9804 as_Register($src2$$reg),
9805 as_Register($src1$$reg),
9806 (Assembler::Condition)$cmp$$cmpcode);
9807 %}
9808
9809 ins_pipe(icond_reg_reg);
9810 %}
9811
9812 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9813 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9814
9815 ins_cost(INSN_COST * 2);
9816 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9817
9818 ins_encode %{
9819 __ cselw(as_Register($dst$$reg),
9820 as_Register($src2$$reg),
9821 as_Register($src1$$reg),
9822 (Assembler::Condition)$cmp$$cmpcode);
9823 %}
9824
9825 ins_pipe(icond_reg_reg);
9826 %}
9827
9828 // special cases where one arg is zero
9829
9830 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9831 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9832
9833 ins_cost(INSN_COST * 2);
9834 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9835
9836 ins_encode %{
9837 __ cselw(as_Register($dst$$reg),
9838 zr,
9839 as_Register($src$$reg),
9840 (Assembler::Condition)$cmp$$cmpcode);
9841 %}
9842
9843 ins_pipe(icond_reg);
9844 %}
9845
9846 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9847 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9848
9849 ins_cost(INSN_COST * 2);
9850 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9851
9852 ins_encode %{
9853 __ cselw(as_Register($dst$$reg),
9854 zr,
9855 as_Register($src$$reg),
9856 (Assembler::Condition)$cmp$$cmpcode);
9857 %}
9858
9859 ins_pipe(icond_reg);
9860 %}
9861
9862 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9863 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9864
9865 ins_cost(INSN_COST * 2);
9866 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9867
9868 ins_encode %{
9869 __ cselw(as_Register($dst$$reg),
9870 as_Register($src$$reg),
9871 zr,
9872 (Assembler::Condition)$cmp$$cmpcode);
9873 %}
9874
9875 ins_pipe(icond_reg);
9876 %}
9877
9878 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9879 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9880
9881 ins_cost(INSN_COST * 2);
9882 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9883
9884 ins_encode %{
9885 __ cselw(as_Register($dst$$reg),
9886 as_Register($src$$reg),
9887 zr,
9888 (Assembler::Condition)$cmp$$cmpcode);
9889 %}
9890
9891 ins_pipe(icond_reg);
9892 %}
9893
9894 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9895 %{
9896 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9897
9898 ins_cost(INSN_COST * 3);
9899
9900 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9901 ins_encode %{
9902 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9903 __ fcsels(as_FloatRegister($dst$$reg),
9904 as_FloatRegister($src2$$reg),
9905 as_FloatRegister($src1$$reg),
9906 cond);
9907 %}
9908
9909 ins_pipe(fp_cond_reg_reg_s);
9910 %}
9911
9912 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9913 %{
9914 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9915
9916 ins_cost(INSN_COST * 3);
9917
9918 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9919 ins_encode %{
9920 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9921 __ fcsels(as_FloatRegister($dst$$reg),
9922 as_FloatRegister($src2$$reg),
9923 as_FloatRegister($src1$$reg),
9924 cond);
9925 %}
9926
9927 ins_pipe(fp_cond_reg_reg_s);
9928 %}
9929
9930 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9931 %{
9932 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9933
9934 ins_cost(INSN_COST * 3);
9935
9936 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9937 ins_encode %{
9938 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9939 __ fcseld(as_FloatRegister($dst$$reg),
9940 as_FloatRegister($src2$$reg),
9941 as_FloatRegister($src1$$reg),
9942 cond);
9943 %}
9944
9945 ins_pipe(fp_cond_reg_reg_d);
9946 %}
9947
9948 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9949 %{
9950 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9951
9952 ins_cost(INSN_COST * 3);
9953
9954 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9955 ins_encode %{
9956 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9957 __ fcseld(as_FloatRegister($dst$$reg),
9958 as_FloatRegister($src2$$reg),
9959 as_FloatRegister($src1$$reg),
9960 cond);
9961 %}
9962
9963 ins_pipe(fp_cond_reg_reg_d);
9964 %}
9965
9966 // ============================================================================
9967 // Arithmetic Instructions
9968 //
9969
9970 // Integer Addition
9971
9972 // TODO
9973 // these currently employ operations which do not set CR and hence are
9974 // not flagged as killing CR but we would like to isolate the cases
9975 // where we want to set flags from those where we don't. need to work
9976 // out how to do that.
9977
9978 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9979 match(Set dst (AddI src1 src2));
9980
9981 ins_cost(INSN_COST);
9982 format %{ "addw $dst, $src1, $src2" %}
9983
9984 ins_encode %{
9985 __ addw(as_Register($dst$$reg),
9986 as_Register($src1$$reg),
9987 as_Register($src2$$reg));
9988 %}
9989
9990 ins_pipe(ialu_reg_reg);
9991 %}
9992
9993 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9994 match(Set dst (AddI src1 src2));
9995
9996 ins_cost(INSN_COST);
9997 format %{ "addw $dst, $src1, $src2" %}
9998
9999 // use opcode to indicate that this is an add not a sub
10000 opcode(0x0);
10001
10002 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10003
10004 ins_pipe(ialu_reg_imm);
10005 %}
10006
10007 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10008 match(Set dst (AddI (ConvL2I src1) src2));
10009
10010 ins_cost(INSN_COST);
10011 format %{ "addw $dst, $src1, $src2" %}
10012
10013 // use opcode to indicate that this is an add not a sub
10014 opcode(0x0);
10015
10016 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10017
10018 ins_pipe(ialu_reg_imm);
10019 %}
10020
10021 // Pointer Addition
10022 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
10023 match(Set dst (AddP src1 src2));
10024
10025 ins_cost(INSN_COST);
10026 format %{ "add $dst, $src1, $src2\t# ptr" %}
10027
10028 ins_encode %{
10029 __ add(as_Register($dst$$reg),
10030 as_Register($src1$$reg),
10031 as_Register($src2$$reg));
10032 %}
10033
10034 ins_pipe(ialu_reg_reg);
10035 %}
10036
10037 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
10038 match(Set dst (AddP src1 (ConvI2L src2)));
10039
10040 ins_cost(1.9 * INSN_COST);
10041 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10042
10043 ins_encode %{
10044 __ add(as_Register($dst$$reg),
10045 as_Register($src1$$reg),
10046 as_Register($src2$$reg), ext::sxtw);
10047 %}
10048
10049 ins_pipe(ialu_reg_reg);
10050 %}
10051
10052 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10053 match(Set dst (AddP src1 (LShiftL src2 scale)));
10054
10055 ins_cost(1.9 * INSN_COST);
10056 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10057
10058 ins_encode %{
10059 __ lea(as_Register($dst$$reg),
10060 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10061 Address::lsl($scale$$constant)));
10062 %}
10063
10064 ins_pipe(ialu_reg_reg_shift);
10065 %}
10066
10067 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10068 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10069
10070 ins_cost(1.9 * INSN_COST);
10071 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10072
10073 ins_encode %{
10074 __ lea(as_Register($dst$$reg),
10075 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10076 Address::sxtw($scale$$constant)));
10077 %}
10078
10079 ins_pipe(ialu_reg_reg_shift);
10080 %}
10081
10082 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10083 match(Set dst (LShiftL (ConvI2L src) scale));
10084
10085 ins_cost(INSN_COST);
10086 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10087
10088 ins_encode %{
10089 __ sbfiz(as_Register($dst$$reg),
10090 as_Register($src$$reg),
10091 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10092 %}
10093
10094 ins_pipe(ialu_reg_shift);
10095 %}
10096
10097 // Pointer Immediate Addition
10098 // n.b. this needs to be more expensive than using an indirect memory
10099 // operand
10100 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10101 match(Set dst (AddP src1 src2));
10102
10103 ins_cost(INSN_COST);
10104 format %{ "add $dst, $src1, $src2\t# ptr" %}
10105
10106 // use opcode to indicate that this is an add not a sub
10107 opcode(0x0);
10108
10109 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10110
10111 ins_pipe(ialu_reg_imm);
10112 %}
10113
10114 // Long Addition
10115 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10116
10117 match(Set dst (AddL src1 src2));
10118
10119 ins_cost(INSN_COST);
10120 format %{ "add $dst, $src1, $src2" %}
10121
10122 ins_encode %{
10123 __ add(as_Register($dst$$reg),
10124 as_Register($src1$$reg),
10125 as_Register($src2$$reg));
10126 %}
10127
10128 ins_pipe(ialu_reg_reg);
10129 %}
10130
10131 // No constant pool entries requiredLong Immediate Addition.
10132 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10133 match(Set dst (AddL src1 src2));
10134
10135 ins_cost(INSN_COST);
10136 format %{ "add $dst, $src1, $src2" %}
10137
10138 // use opcode to indicate that this is an add not a sub
10139 opcode(0x0);
10140
10141 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10142
10143 ins_pipe(ialu_reg_imm);
10144 %}
10145
10146 // Integer Subtraction
10147 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10148 match(Set dst (SubI src1 src2));
10149
10150 ins_cost(INSN_COST);
10151 format %{ "subw $dst, $src1, $src2" %}
10152
10153 ins_encode %{
10154 __ subw(as_Register($dst$$reg),
10155 as_Register($src1$$reg),
10156 as_Register($src2$$reg));
10157 %}
10158
10159 ins_pipe(ialu_reg_reg);
10160 %}
10161
10162 // Immediate Subtraction
10163 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10164 match(Set dst (SubI src1 src2));
10165
10166 ins_cost(INSN_COST);
10167 format %{ "subw $dst, $src1, $src2" %}
10168
10169 // use opcode to indicate that this is a sub not an add
10170 opcode(0x1);
10171
10172 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10173
10174 ins_pipe(ialu_reg_imm);
10175 %}
10176
10177 // Long Subtraction
10178 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10179
10180 match(Set dst (SubL src1 src2));
10181
10182 ins_cost(INSN_COST);
10183 format %{ "sub $dst, $src1, $src2" %}
10184
10185 ins_encode %{
10186 __ sub(as_Register($dst$$reg),
10187 as_Register($src1$$reg),
10188 as_Register($src2$$reg));
10189 %}
10190
10191 ins_pipe(ialu_reg_reg);
10192 %}
10193
10194 // No constant pool entries requiredLong Immediate Subtraction.
10195 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10196 match(Set dst (SubL src1 src2));
10197
10198 ins_cost(INSN_COST);
10199 format %{ "sub$dst, $src1, $src2" %}
10200
10201 // use opcode to indicate that this is a sub not an add
10202 opcode(0x1);
10203
10204 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10205
10206 ins_pipe(ialu_reg_imm);
10207 %}
10208
10209 // Integer Negation (special case for sub)
10210
10211 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10212 match(Set dst (SubI zero src));
10213
10214 ins_cost(INSN_COST);
10215 format %{ "negw $dst, $src\t# int" %}
10216
10217 ins_encode %{
10218 __ negw(as_Register($dst$$reg),
10219 as_Register($src$$reg));
10220 %}
10221
10222 ins_pipe(ialu_reg);
10223 %}
10224
10225 // Long Negation
10226
10227 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10228 match(Set dst (SubL zero src));
10229
10230 ins_cost(INSN_COST);
10231 format %{ "neg $dst, $src\t# long" %}
10232
10233 ins_encode %{
10234 __ neg(as_Register($dst$$reg),
10235 as_Register($src$$reg));
10236 %}
10237
10238 ins_pipe(ialu_reg);
10239 %}
10240
10241 // Integer Multiply
10242
10243 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10244 match(Set dst (MulI src1 src2));
10245
10246 ins_cost(INSN_COST * 3);
10247 format %{ "mulw $dst, $src1, $src2" %}
10248
10249 ins_encode %{
10250 __ mulw(as_Register($dst$$reg),
10251 as_Register($src1$$reg),
10252 as_Register($src2$$reg));
10253 %}
10254
10255 ins_pipe(imul_reg_reg);
10256 %}
10257
10258 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10259 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10260
10261 ins_cost(INSN_COST * 3);
10262 format %{ "smull $dst, $src1, $src2" %}
10263
10264 ins_encode %{
10265 __ smull(as_Register($dst$$reg),
10266 as_Register($src1$$reg),
10267 as_Register($src2$$reg));
10268 %}
10269
10270 ins_pipe(imul_reg_reg);
10271 %}
10272
10273 // Long Multiply
10274
10275 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10276 match(Set dst (MulL src1 src2));
10277
10278 ins_cost(INSN_COST * 5);
10279 format %{ "mul $dst, $src1, $src2" %}
10280
10281 ins_encode %{
10282 __ mul(as_Register($dst$$reg),
10283 as_Register($src1$$reg),
10284 as_Register($src2$$reg));
10285 %}
10286
10287 ins_pipe(lmul_reg_reg);
10288 %}
10289
10290 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10291 %{
10292 match(Set dst (MulHiL src1 src2));
10293
10294 ins_cost(INSN_COST * 7);
10295 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10296
10297 ins_encode %{
10298 __ smulh(as_Register($dst$$reg),
10299 as_Register($src1$$reg),
10300 as_Register($src2$$reg));
10301 %}
10302
10303 ins_pipe(lmul_reg_reg);
10304 %}
10305
10306 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10307 %{
10308 match(Set dst (UMulHiL src1 src2));
10309
10310 ins_cost(INSN_COST * 7);
10311 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10312
10313 ins_encode %{
10314 __ umulh(as_Register($dst$$reg),
10315 as_Register($src1$$reg),
10316 as_Register($src2$$reg));
10317 %}
10318
10319 ins_pipe(lmul_reg_reg);
10320 %}
10321
10322 // Combined Integer Multiply & Add/Sub
10323
10324 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10325 match(Set dst (AddI src3 (MulI src1 src2)));
10326
10327 ins_cost(INSN_COST * 3);
10328 format %{ "madd $dst, $src1, $src2, $src3" %}
10329
10330 ins_encode %{
10331 __ maddw(as_Register($dst$$reg),
10332 as_Register($src1$$reg),
10333 as_Register($src2$$reg),
10334 as_Register($src3$$reg));
10335 %}
10336
10337 ins_pipe(imac_reg_reg);
10338 %}
10339
10340 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10341 match(Set dst (SubI src3 (MulI src1 src2)));
10342
10343 ins_cost(INSN_COST * 3);
10344 format %{ "msub $dst, $src1, $src2, $src3" %}
10345
10346 ins_encode %{
10347 __ msubw(as_Register($dst$$reg),
10348 as_Register($src1$$reg),
10349 as_Register($src2$$reg),
10350 as_Register($src3$$reg));
10351 %}
10352
10353 ins_pipe(imac_reg_reg);
10354 %}
10355
10356 // Combined Integer Multiply & Neg
10357
10358 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10359 match(Set dst (MulI (SubI zero src1) src2));
10360
10361 ins_cost(INSN_COST * 3);
10362 format %{ "mneg $dst, $src1, $src2" %}
10363
10364 ins_encode %{
10365 __ mnegw(as_Register($dst$$reg),
10366 as_Register($src1$$reg),
10367 as_Register($src2$$reg));
10368 %}
10369
10370 ins_pipe(imac_reg_reg);
10371 %}
10372
10373 // Combined Long Multiply & Add/Sub
10374
10375 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10376 match(Set dst (AddL src3 (MulL src1 src2)));
10377
10378 ins_cost(INSN_COST * 5);
10379 format %{ "madd $dst, $src1, $src2, $src3" %}
10380
10381 ins_encode %{
10382 __ madd(as_Register($dst$$reg),
10383 as_Register($src1$$reg),
10384 as_Register($src2$$reg),
10385 as_Register($src3$$reg));
10386 %}
10387
10388 ins_pipe(lmac_reg_reg);
10389 %}
10390
10391 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10392 match(Set dst (SubL src3 (MulL src1 src2)));
10393
10394 ins_cost(INSN_COST * 5);
10395 format %{ "msub $dst, $src1, $src2, $src3" %}
10396
10397 ins_encode %{
10398 __ msub(as_Register($dst$$reg),
10399 as_Register($src1$$reg),
10400 as_Register($src2$$reg),
10401 as_Register($src3$$reg));
10402 %}
10403
10404 ins_pipe(lmac_reg_reg);
10405 %}
10406
10407 // Combined Long Multiply & Neg
10408
10409 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10410 match(Set dst (MulL (SubL zero src1) src2));
10411
10412 ins_cost(INSN_COST * 5);
10413 format %{ "mneg $dst, $src1, $src2" %}
10414
10415 ins_encode %{
10416 __ mneg(as_Register($dst$$reg),
10417 as_Register($src1$$reg),
10418 as_Register($src2$$reg));
10419 %}
10420
10421 ins_pipe(lmac_reg_reg);
10422 %}
10423
10424 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10425
10426 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10427 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10428
10429 ins_cost(INSN_COST * 3);
10430 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10431
10432 ins_encode %{
10433 __ smaddl(as_Register($dst$$reg),
10434 as_Register($src1$$reg),
10435 as_Register($src2$$reg),
10436 as_Register($src3$$reg));
10437 %}
10438
10439 ins_pipe(imac_reg_reg);
10440 %}
10441
10442 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10443 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10444
10445 ins_cost(INSN_COST * 3);
10446 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10447
10448 ins_encode %{
10449 __ smsubl(as_Register($dst$$reg),
10450 as_Register($src1$$reg),
10451 as_Register($src2$$reg),
10452 as_Register($src3$$reg));
10453 %}
10454
10455 ins_pipe(imac_reg_reg);
10456 %}
10457
10458 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10459 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10460
10461 ins_cost(INSN_COST * 3);
10462 format %{ "smnegl $dst, $src1, $src2" %}
10463
10464 ins_encode %{
10465 __ smnegl(as_Register($dst$$reg),
10466 as_Register($src1$$reg),
10467 as_Register($src2$$reg));
10468 %}
10469
10470 ins_pipe(imac_reg_reg);
10471 %}
10472
10473 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10474
10475 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10476 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10477
10478 ins_cost(INSN_COST * 5);
10479 format %{ "mulw rscratch1, $src1, $src2\n\t"
10480 "maddw $dst, $src3, $src4, rscratch1" %}
10481
10482 ins_encode %{
10483 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10484 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10485
10486 ins_pipe(imac_reg_reg);
10487 %}
10488
10489 // Integer Divide
10490
10491 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10492 match(Set dst (DivI src1 src2));
10493
10494 ins_cost(INSN_COST * 19);
10495 format %{ "sdivw $dst, $src1, $src2" %}
10496
10497 ins_encode(aarch64_enc_divw(dst, src1, src2));
10498 ins_pipe(idiv_reg_reg);
10499 %}
10500
10501 // Long Divide
10502
10503 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10504 match(Set dst (DivL src1 src2));
10505
10506 ins_cost(INSN_COST * 35);
10507 format %{ "sdiv $dst, $src1, $src2" %}
10508
10509 ins_encode(aarch64_enc_div(dst, src1, src2));
10510 ins_pipe(ldiv_reg_reg);
10511 %}
10512
10513 // Integer Remainder
10514
10515 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10516 match(Set dst (ModI src1 src2));
10517
10518 ins_cost(INSN_COST * 22);
10519 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10520 "msubw $dst, rscratch1, $src2, $src1" %}
10521
10522 ins_encode(aarch64_enc_modw(dst, src1, src2));
10523 ins_pipe(idiv_reg_reg);
10524 %}
10525
10526 // Long Remainder
10527
10528 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10529 match(Set dst (ModL src1 src2));
10530
10531 ins_cost(INSN_COST * 38);
10532 format %{ "sdiv rscratch1, $src1, $src2\n"
10533 "msub $dst, rscratch1, $src2, $src1" %}
10534
10535 ins_encode(aarch64_enc_mod(dst, src1, src2));
10536 ins_pipe(ldiv_reg_reg);
10537 %}
10538
10539 // Unsigned Integer Divide
10540
10541 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10542 match(Set dst (UDivI src1 src2));
10543
10544 ins_cost(INSN_COST * 19);
10545 format %{ "udivw $dst, $src1, $src2" %}
10546
10547 ins_encode %{
10548 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10549 %}
10550
10551 ins_pipe(idiv_reg_reg);
10552 %}
10553
10554 // Unsigned Long Divide
10555
10556 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10557 match(Set dst (UDivL src1 src2));
10558
10559 ins_cost(INSN_COST * 35);
10560 format %{ "udiv $dst, $src1, $src2" %}
10561
10562 ins_encode %{
10563 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10564 %}
10565
10566 ins_pipe(ldiv_reg_reg);
10567 %}
10568
10569 // Unsigned Integer Remainder
10570
10571 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10572 match(Set dst (UModI src1 src2));
10573
10574 ins_cost(INSN_COST * 22);
10575 format %{ "udivw rscratch1, $src1, $src2\n\t"
10576 "msubw $dst, rscratch1, $src2, $src1" %}
10577
10578 ins_encode %{
10579 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10580 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10581 %}
10582
10583 ins_pipe(idiv_reg_reg);
10584 %}
10585
10586 // Unsigned Long Remainder
10587
10588 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10589 match(Set dst (UModL src1 src2));
10590
10591 ins_cost(INSN_COST * 38);
10592 format %{ "udiv rscratch1, $src1, $src2\n"
10593 "msub $dst, rscratch1, $src2, $src1" %}
10594
10595 ins_encode %{
10596 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10597 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10598 %}
10599
10600 ins_pipe(ldiv_reg_reg);
10601 %}
10602
10603 // Integer Shifts
10604
10605 // Shift Left Register
10606 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10607 match(Set dst (LShiftI src1 src2));
10608
10609 ins_cost(INSN_COST * 2);
10610 format %{ "lslvw $dst, $src1, $src2" %}
10611
10612 ins_encode %{
10613 __ lslvw(as_Register($dst$$reg),
10614 as_Register($src1$$reg),
10615 as_Register($src2$$reg));
10616 %}
10617
10618 ins_pipe(ialu_reg_reg_vshift);
10619 %}
10620
10621 // Shift Left Immediate
10622 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10623 match(Set dst (LShiftI src1 src2));
10624
10625 ins_cost(INSN_COST);
10626 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10627
10628 ins_encode %{
10629 __ lslw(as_Register($dst$$reg),
10630 as_Register($src1$$reg),
10631 $src2$$constant & 0x1f);
10632 %}
10633
10634 ins_pipe(ialu_reg_shift);
10635 %}
10636
10637 // Shift Right Logical Register
10638 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10639 match(Set dst (URShiftI src1 src2));
10640
10641 ins_cost(INSN_COST * 2);
10642 format %{ "lsrvw $dst, $src1, $src2" %}
10643
10644 ins_encode %{
10645 __ lsrvw(as_Register($dst$$reg),
10646 as_Register($src1$$reg),
10647 as_Register($src2$$reg));
10648 %}
10649
10650 ins_pipe(ialu_reg_reg_vshift);
10651 %}
10652
10653 // Shift Right Logical Immediate
10654 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10655 match(Set dst (URShiftI src1 src2));
10656
10657 ins_cost(INSN_COST);
10658 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10659
10660 ins_encode %{
10661 __ lsrw(as_Register($dst$$reg),
10662 as_Register($src1$$reg),
10663 $src2$$constant & 0x1f);
10664 %}
10665
10666 ins_pipe(ialu_reg_shift);
10667 %}
10668
10669 // Shift Right Arithmetic Register
10670 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10671 match(Set dst (RShiftI src1 src2));
10672
10673 ins_cost(INSN_COST * 2);
10674 format %{ "asrvw $dst, $src1, $src2" %}
10675
10676 ins_encode %{
10677 __ asrvw(as_Register($dst$$reg),
10678 as_Register($src1$$reg),
10679 as_Register($src2$$reg));
10680 %}
10681
10682 ins_pipe(ialu_reg_reg_vshift);
10683 %}
10684
10685 // Shift Right Arithmetic Immediate
10686 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10687 match(Set dst (RShiftI src1 src2));
10688
10689 ins_cost(INSN_COST);
10690 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10691
10692 ins_encode %{
10693 __ asrw(as_Register($dst$$reg),
10694 as_Register($src1$$reg),
10695 $src2$$constant & 0x1f);
10696 %}
10697
10698 ins_pipe(ialu_reg_shift);
10699 %}
10700
10701 // Combined Int Mask and Right Shift (using UBFM)
10702 // TODO
10703
10704 // Long Shifts
10705
10706 // Shift Left Register
10707 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10708 match(Set dst (LShiftL src1 src2));
10709
10710 ins_cost(INSN_COST * 2);
10711 format %{ "lslv $dst, $src1, $src2" %}
10712
10713 ins_encode %{
10714 __ lslv(as_Register($dst$$reg),
10715 as_Register($src1$$reg),
10716 as_Register($src2$$reg));
10717 %}
10718
10719 ins_pipe(ialu_reg_reg_vshift);
10720 %}
10721
10722 // Shift Left Immediate
10723 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10724 match(Set dst (LShiftL src1 src2));
10725
10726 ins_cost(INSN_COST);
10727 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10728
10729 ins_encode %{
10730 __ lsl(as_Register($dst$$reg),
10731 as_Register($src1$$reg),
10732 $src2$$constant & 0x3f);
10733 %}
10734
10735 ins_pipe(ialu_reg_shift);
10736 %}
10737
10738 // Shift Right Logical Register
10739 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10740 match(Set dst (URShiftL src1 src2));
10741
10742 ins_cost(INSN_COST * 2);
10743 format %{ "lsrv $dst, $src1, $src2" %}
10744
10745 ins_encode %{
10746 __ lsrv(as_Register($dst$$reg),
10747 as_Register($src1$$reg),
10748 as_Register($src2$$reg));
10749 %}
10750
10751 ins_pipe(ialu_reg_reg_vshift);
10752 %}
10753
10754 // Shift Right Logical Immediate
10755 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10756 match(Set dst (URShiftL src1 src2));
10757
10758 ins_cost(INSN_COST);
10759 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10760
10761 ins_encode %{
10762 __ lsr(as_Register($dst$$reg),
10763 as_Register($src1$$reg),
10764 $src2$$constant & 0x3f);
10765 %}
10766
10767 ins_pipe(ialu_reg_shift);
10768 %}
10769
10770 // A special-case pattern for card table stores.
10771 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10772 match(Set dst (URShiftL (CastP2X src1) src2));
10773
10774 ins_cost(INSN_COST);
10775 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10776
10777 ins_encode %{
10778 __ lsr(as_Register($dst$$reg),
10779 as_Register($src1$$reg),
10780 $src2$$constant & 0x3f);
10781 %}
10782
10783 ins_pipe(ialu_reg_shift);
10784 %}
10785
10786 // Shift Right Arithmetic Register
10787 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10788 match(Set dst (RShiftL src1 src2));
10789
10790 ins_cost(INSN_COST * 2);
10791 format %{ "asrv $dst, $src1, $src2" %}
10792
10793 ins_encode %{
10794 __ asrv(as_Register($dst$$reg),
10795 as_Register($src1$$reg),
10796 as_Register($src2$$reg));
10797 %}
10798
10799 ins_pipe(ialu_reg_reg_vshift);
10800 %}
10801
10802 // Shift Right Arithmetic Immediate
10803 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10804 match(Set dst (RShiftL src1 src2));
10805
10806 ins_cost(INSN_COST);
10807 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10808
10809 ins_encode %{
10810 __ asr(as_Register($dst$$reg),
10811 as_Register($src1$$reg),
10812 $src2$$constant & 0x3f);
10813 %}
10814
10815 ins_pipe(ialu_reg_shift);
10816 %}
10817
10818 // BEGIN This section of the file is automatically generated. Do not edit --------------
10819 // This section is generated from aarch64_ad.m4
10820
10821 // This pattern is automatically generated from aarch64_ad.m4
10822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10823 instruct regL_not_reg(iRegLNoSp dst,
10824 iRegL src1, immL_M1 m1,
10825 rFlagsReg cr) %{
10826 match(Set dst (XorL src1 m1));
10827 ins_cost(INSN_COST);
10828 format %{ "eon $dst, $src1, zr" %}
10829
10830 ins_encode %{
10831 __ eon(as_Register($dst$$reg),
10832 as_Register($src1$$reg),
10833 zr,
10834 Assembler::LSL, 0);
10835 %}
10836
10837 ins_pipe(ialu_reg);
10838 %}
10839
10840 // This pattern is automatically generated from aarch64_ad.m4
10841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10842 instruct regI_not_reg(iRegINoSp dst,
10843 iRegIorL2I src1, immI_M1 m1,
10844 rFlagsReg cr) %{
10845 match(Set dst (XorI src1 m1));
10846 ins_cost(INSN_COST);
10847 format %{ "eonw $dst, $src1, zr" %}
10848
10849 ins_encode %{
10850 __ eonw(as_Register($dst$$reg),
10851 as_Register($src1$$reg),
10852 zr,
10853 Assembler::LSL, 0);
10854 %}
10855
10856 ins_pipe(ialu_reg);
10857 %}
10858
10859 // This pattern is automatically generated from aarch64_ad.m4
10860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10861 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10862 immI0 zero, iRegIorL2I src1, immI src2) %{
10863 match(Set dst (SubI zero (URShiftI src1 src2)));
10864
10865 ins_cost(1.9 * INSN_COST);
10866 format %{ "negw $dst, $src1, LSR $src2" %}
10867
10868 ins_encode %{
10869 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10870 Assembler::LSR, $src2$$constant & 0x1f);
10871 %}
10872
10873 ins_pipe(ialu_reg_shift);
10874 %}
10875
10876 // This pattern is automatically generated from aarch64_ad.m4
10877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10878 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10879 immI0 zero, iRegIorL2I src1, immI src2) %{
10880 match(Set dst (SubI zero (RShiftI src1 src2)));
10881
10882 ins_cost(1.9 * INSN_COST);
10883 format %{ "negw $dst, $src1, ASR $src2" %}
10884
10885 ins_encode %{
10886 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10887 Assembler::ASR, $src2$$constant & 0x1f);
10888 %}
10889
10890 ins_pipe(ialu_reg_shift);
10891 %}
10892
10893 // This pattern is automatically generated from aarch64_ad.m4
10894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10895 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10896 immI0 zero, iRegIorL2I src1, immI src2) %{
10897 match(Set dst (SubI zero (LShiftI src1 src2)));
10898
10899 ins_cost(1.9 * INSN_COST);
10900 format %{ "negw $dst, $src1, LSL $src2" %}
10901
10902 ins_encode %{
10903 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10904 Assembler::LSL, $src2$$constant & 0x1f);
10905 %}
10906
10907 ins_pipe(ialu_reg_shift);
10908 %}
10909
10910 // This pattern is automatically generated from aarch64_ad.m4
10911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10912 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10913 immL0 zero, iRegL src1, immI src2) %{
10914 match(Set dst (SubL zero (URShiftL src1 src2)));
10915
10916 ins_cost(1.9 * INSN_COST);
10917 format %{ "neg $dst, $src1, LSR $src2" %}
10918
10919 ins_encode %{
10920 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10921 Assembler::LSR, $src2$$constant & 0x3f);
10922 %}
10923
10924 ins_pipe(ialu_reg_shift);
10925 %}
10926
10927 // This pattern is automatically generated from aarch64_ad.m4
10928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10929 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10930 immL0 zero, iRegL src1, immI src2) %{
10931 match(Set dst (SubL zero (RShiftL src1 src2)));
10932
10933 ins_cost(1.9 * INSN_COST);
10934 format %{ "neg $dst, $src1, ASR $src2" %}
10935
10936 ins_encode %{
10937 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10938 Assembler::ASR, $src2$$constant & 0x3f);
10939 %}
10940
10941 ins_pipe(ialu_reg_shift);
10942 %}
10943
10944 // This pattern is automatically generated from aarch64_ad.m4
10945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10946 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10947 immL0 zero, iRegL src1, immI src2) %{
10948 match(Set dst (SubL zero (LShiftL src1 src2)));
10949
10950 ins_cost(1.9 * INSN_COST);
10951 format %{ "neg $dst, $src1, LSL $src2" %}
10952
10953 ins_encode %{
10954 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10955 Assembler::LSL, $src2$$constant & 0x3f);
10956 %}
10957
10958 ins_pipe(ialu_reg_shift);
10959 %}
10960
10961 // This pattern is automatically generated from aarch64_ad.m4
10962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10963 instruct AndI_reg_not_reg(iRegINoSp dst,
10964 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10965 match(Set dst (AndI src1 (XorI src2 m1)));
10966 ins_cost(INSN_COST);
10967 format %{ "bicw $dst, $src1, $src2" %}
10968
10969 ins_encode %{
10970 __ bicw(as_Register($dst$$reg),
10971 as_Register($src1$$reg),
10972 as_Register($src2$$reg),
10973 Assembler::LSL, 0);
10974 %}
10975
10976 ins_pipe(ialu_reg_reg);
10977 %}
10978
10979 // This pattern is automatically generated from aarch64_ad.m4
10980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10981 instruct AndL_reg_not_reg(iRegLNoSp dst,
10982 iRegL src1, iRegL src2, immL_M1 m1) %{
10983 match(Set dst (AndL src1 (XorL src2 m1)));
10984 ins_cost(INSN_COST);
10985 format %{ "bic $dst, $src1, $src2" %}
10986
10987 ins_encode %{
10988 __ bic(as_Register($dst$$reg),
10989 as_Register($src1$$reg),
10990 as_Register($src2$$reg),
10991 Assembler::LSL, 0);
10992 %}
10993
10994 ins_pipe(ialu_reg_reg);
10995 %}
10996
10997 // This pattern is automatically generated from aarch64_ad.m4
10998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10999 instruct OrI_reg_not_reg(iRegINoSp dst,
11000 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11001 match(Set dst (OrI src1 (XorI src2 m1)));
11002 ins_cost(INSN_COST);
11003 format %{ "ornw $dst, $src1, $src2" %}
11004
11005 ins_encode %{
11006 __ ornw(as_Register($dst$$reg),
11007 as_Register($src1$$reg),
11008 as_Register($src2$$reg),
11009 Assembler::LSL, 0);
11010 %}
11011
11012 ins_pipe(ialu_reg_reg);
11013 %}
11014
11015 // This pattern is automatically generated from aarch64_ad.m4
11016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11017 instruct OrL_reg_not_reg(iRegLNoSp dst,
11018 iRegL src1, iRegL src2, immL_M1 m1) %{
11019 match(Set dst (OrL src1 (XorL src2 m1)));
11020 ins_cost(INSN_COST);
11021 format %{ "orn $dst, $src1, $src2" %}
11022
11023 ins_encode %{
11024 __ orn(as_Register($dst$$reg),
11025 as_Register($src1$$reg),
11026 as_Register($src2$$reg),
11027 Assembler::LSL, 0);
11028 %}
11029
11030 ins_pipe(ialu_reg_reg);
11031 %}
11032
11033 // This pattern is automatically generated from aarch64_ad.m4
11034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11035 instruct XorI_reg_not_reg(iRegINoSp dst,
11036 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11037 match(Set dst (XorI m1 (XorI src2 src1)));
11038 ins_cost(INSN_COST);
11039 format %{ "eonw $dst, $src1, $src2" %}
11040
11041 ins_encode %{
11042 __ eonw(as_Register($dst$$reg),
11043 as_Register($src1$$reg),
11044 as_Register($src2$$reg),
11045 Assembler::LSL, 0);
11046 %}
11047
11048 ins_pipe(ialu_reg_reg);
11049 %}
11050
11051 // This pattern is automatically generated from aarch64_ad.m4
11052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11053 instruct XorL_reg_not_reg(iRegLNoSp dst,
11054 iRegL src1, iRegL src2, immL_M1 m1) %{
11055 match(Set dst (XorL m1 (XorL src2 src1)));
11056 ins_cost(INSN_COST);
11057 format %{ "eon $dst, $src1, $src2" %}
11058
11059 ins_encode %{
11060 __ eon(as_Register($dst$$reg),
11061 as_Register($src1$$reg),
11062 as_Register($src2$$reg),
11063 Assembler::LSL, 0);
11064 %}
11065
11066 ins_pipe(ialu_reg_reg);
11067 %}
11068
11069 // This pattern is automatically generated from aarch64_ad.m4
11070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11071 // val & (-1 ^ (val >>> shift)) ==> bicw
11072 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11073 iRegIorL2I src1, iRegIorL2I src2,
11074 immI src3, immI_M1 src4) %{
11075 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11076 ins_cost(1.9 * INSN_COST);
11077 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11078
11079 ins_encode %{
11080 __ bicw(as_Register($dst$$reg),
11081 as_Register($src1$$reg),
11082 as_Register($src2$$reg),
11083 Assembler::LSR,
11084 $src3$$constant & 0x1f);
11085 %}
11086
11087 ins_pipe(ialu_reg_reg_shift);
11088 %}
11089
11090 // This pattern is automatically generated from aarch64_ad.m4
11091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11092 // val & (-1 ^ (val >>> shift)) ==> bic
11093 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11094 iRegL src1, iRegL src2,
11095 immI src3, immL_M1 src4) %{
11096 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11097 ins_cost(1.9 * INSN_COST);
11098 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11099
11100 ins_encode %{
11101 __ bic(as_Register($dst$$reg),
11102 as_Register($src1$$reg),
11103 as_Register($src2$$reg),
11104 Assembler::LSR,
11105 $src3$$constant & 0x3f);
11106 %}
11107
11108 ins_pipe(ialu_reg_reg_shift);
11109 %}
11110
11111 // This pattern is automatically generated from aarch64_ad.m4
11112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11113 // val & (-1 ^ (val >> shift)) ==> bicw
11114 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11115 iRegIorL2I src1, iRegIorL2I src2,
11116 immI src3, immI_M1 src4) %{
11117 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11118 ins_cost(1.9 * INSN_COST);
11119 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11120
11121 ins_encode %{
11122 __ bicw(as_Register($dst$$reg),
11123 as_Register($src1$$reg),
11124 as_Register($src2$$reg),
11125 Assembler::ASR,
11126 $src3$$constant & 0x1f);
11127 %}
11128
11129 ins_pipe(ialu_reg_reg_shift);
11130 %}
11131
11132 // This pattern is automatically generated from aarch64_ad.m4
11133 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11134 // val & (-1 ^ (val >> shift)) ==> bic
11135 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11136 iRegL src1, iRegL src2,
11137 immI src3, immL_M1 src4) %{
11138 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11139 ins_cost(1.9 * INSN_COST);
11140 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11141
11142 ins_encode %{
11143 __ bic(as_Register($dst$$reg),
11144 as_Register($src1$$reg),
11145 as_Register($src2$$reg),
11146 Assembler::ASR,
11147 $src3$$constant & 0x3f);
11148 %}
11149
11150 ins_pipe(ialu_reg_reg_shift);
11151 %}
11152
11153 // This pattern is automatically generated from aarch64_ad.m4
11154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11155 // val & (-1 ^ (val ror shift)) ==> bicw
11156 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11157 iRegIorL2I src1, iRegIorL2I src2,
11158 immI src3, immI_M1 src4) %{
11159 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11160 ins_cost(1.9 * INSN_COST);
11161 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11162
11163 ins_encode %{
11164 __ bicw(as_Register($dst$$reg),
11165 as_Register($src1$$reg),
11166 as_Register($src2$$reg),
11167 Assembler::ROR,
11168 $src3$$constant & 0x1f);
11169 %}
11170
11171 ins_pipe(ialu_reg_reg_shift);
11172 %}
11173
11174 // This pattern is automatically generated from aarch64_ad.m4
11175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11176 // val & (-1 ^ (val ror shift)) ==> bic
11177 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11178 iRegL src1, iRegL src2,
11179 immI src3, immL_M1 src4) %{
11180 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11181 ins_cost(1.9 * INSN_COST);
11182 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11183
11184 ins_encode %{
11185 __ bic(as_Register($dst$$reg),
11186 as_Register($src1$$reg),
11187 as_Register($src2$$reg),
11188 Assembler::ROR,
11189 $src3$$constant & 0x3f);
11190 %}
11191
11192 ins_pipe(ialu_reg_reg_shift);
11193 %}
11194
11195 // This pattern is automatically generated from aarch64_ad.m4
11196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11197 // val & (-1 ^ (val << shift)) ==> bicw
11198 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11199 iRegIorL2I src1, iRegIorL2I src2,
11200 immI src3, immI_M1 src4) %{
11201 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11202 ins_cost(1.9 * INSN_COST);
11203 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11204
11205 ins_encode %{
11206 __ bicw(as_Register($dst$$reg),
11207 as_Register($src1$$reg),
11208 as_Register($src2$$reg),
11209 Assembler::LSL,
11210 $src3$$constant & 0x1f);
11211 %}
11212
11213 ins_pipe(ialu_reg_reg_shift);
11214 %}
11215
11216 // This pattern is automatically generated from aarch64_ad.m4
11217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11218 // val & (-1 ^ (val << shift)) ==> bic
11219 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11220 iRegL src1, iRegL src2,
11221 immI src3, immL_M1 src4) %{
11222 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11223 ins_cost(1.9 * INSN_COST);
11224 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11225
11226 ins_encode %{
11227 __ bic(as_Register($dst$$reg),
11228 as_Register($src1$$reg),
11229 as_Register($src2$$reg),
11230 Assembler::LSL,
11231 $src3$$constant & 0x3f);
11232 %}
11233
11234 ins_pipe(ialu_reg_reg_shift);
11235 %}
11236
11237 // This pattern is automatically generated from aarch64_ad.m4
11238 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11239 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11240 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11241 iRegIorL2I src1, iRegIorL2I src2,
11242 immI src3, immI_M1 src4) %{
11243 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11244 ins_cost(1.9 * INSN_COST);
11245 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11246
11247 ins_encode %{
11248 __ eonw(as_Register($dst$$reg),
11249 as_Register($src1$$reg),
11250 as_Register($src2$$reg),
11251 Assembler::LSR,
11252 $src3$$constant & 0x1f);
11253 %}
11254
11255 ins_pipe(ialu_reg_reg_shift);
11256 %}
11257
11258 // This pattern is automatically generated from aarch64_ad.m4
11259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11260 // val ^ (-1 ^ (val >>> shift)) ==> eon
11261 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11262 iRegL src1, iRegL src2,
11263 immI src3, immL_M1 src4) %{
11264 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11265 ins_cost(1.9 * INSN_COST);
11266 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11267
11268 ins_encode %{
11269 __ eon(as_Register($dst$$reg),
11270 as_Register($src1$$reg),
11271 as_Register($src2$$reg),
11272 Assembler::LSR,
11273 $src3$$constant & 0x3f);
11274 %}
11275
11276 ins_pipe(ialu_reg_reg_shift);
11277 %}
11278
11279 // This pattern is automatically generated from aarch64_ad.m4
11280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11281 // val ^ (-1 ^ (val >> shift)) ==> eonw
11282 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11283 iRegIorL2I src1, iRegIorL2I src2,
11284 immI src3, immI_M1 src4) %{
11285 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11286 ins_cost(1.9 * INSN_COST);
11287 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11288
11289 ins_encode %{
11290 __ eonw(as_Register($dst$$reg),
11291 as_Register($src1$$reg),
11292 as_Register($src2$$reg),
11293 Assembler::ASR,
11294 $src3$$constant & 0x1f);
11295 %}
11296
11297 ins_pipe(ialu_reg_reg_shift);
11298 %}
11299
11300 // This pattern is automatically generated from aarch64_ad.m4
11301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11302 // val ^ (-1 ^ (val >> shift)) ==> eon
11303 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11304 iRegL src1, iRegL src2,
11305 immI src3, immL_M1 src4) %{
11306 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11307 ins_cost(1.9 * INSN_COST);
11308 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11309
11310 ins_encode %{
11311 __ eon(as_Register($dst$$reg),
11312 as_Register($src1$$reg),
11313 as_Register($src2$$reg),
11314 Assembler::ASR,
11315 $src3$$constant & 0x3f);
11316 %}
11317
11318 ins_pipe(ialu_reg_reg_shift);
11319 %}
11320
11321 // This pattern is automatically generated from aarch64_ad.m4
11322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11323 // val ^ (-1 ^ (val ror shift)) ==> eonw
11324 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11325 iRegIorL2I src1, iRegIorL2I src2,
11326 immI src3, immI_M1 src4) %{
11327 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11328 ins_cost(1.9 * INSN_COST);
11329 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11330
11331 ins_encode %{
11332 __ eonw(as_Register($dst$$reg),
11333 as_Register($src1$$reg),
11334 as_Register($src2$$reg),
11335 Assembler::ROR,
11336 $src3$$constant & 0x1f);
11337 %}
11338
11339 ins_pipe(ialu_reg_reg_shift);
11340 %}
11341
11342 // This pattern is automatically generated from aarch64_ad.m4
11343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11344 // val ^ (-1 ^ (val ror shift)) ==> eon
11345 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11346 iRegL src1, iRegL src2,
11347 immI src3, immL_M1 src4) %{
11348 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11349 ins_cost(1.9 * INSN_COST);
11350 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11351
11352 ins_encode %{
11353 __ eon(as_Register($dst$$reg),
11354 as_Register($src1$$reg),
11355 as_Register($src2$$reg),
11356 Assembler::ROR,
11357 $src3$$constant & 0x3f);
11358 %}
11359
11360 ins_pipe(ialu_reg_reg_shift);
11361 %}
11362
11363 // This pattern is automatically generated from aarch64_ad.m4
11364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11365 // val ^ (-1 ^ (val << shift)) ==> eonw
11366 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11367 iRegIorL2I src1, iRegIorL2I src2,
11368 immI src3, immI_M1 src4) %{
11369 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11370 ins_cost(1.9 * INSN_COST);
11371 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11372
11373 ins_encode %{
11374 __ eonw(as_Register($dst$$reg),
11375 as_Register($src1$$reg),
11376 as_Register($src2$$reg),
11377 Assembler::LSL,
11378 $src3$$constant & 0x1f);
11379 %}
11380
11381 ins_pipe(ialu_reg_reg_shift);
11382 %}
11383
11384 // This pattern is automatically generated from aarch64_ad.m4
11385 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11386 // val ^ (-1 ^ (val << shift)) ==> eon
11387 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11388 iRegL src1, iRegL src2,
11389 immI src3, immL_M1 src4) %{
11390 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11391 ins_cost(1.9 * INSN_COST);
11392 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11393
11394 ins_encode %{
11395 __ eon(as_Register($dst$$reg),
11396 as_Register($src1$$reg),
11397 as_Register($src2$$reg),
11398 Assembler::LSL,
11399 $src3$$constant & 0x3f);
11400 %}
11401
11402 ins_pipe(ialu_reg_reg_shift);
11403 %}
11404
11405 // This pattern is automatically generated from aarch64_ad.m4
11406 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11407 // val | (-1 ^ (val >>> shift)) ==> ornw
11408 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11409 iRegIorL2I src1, iRegIorL2I src2,
11410 immI src3, immI_M1 src4) %{
11411 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11412 ins_cost(1.9 * INSN_COST);
11413 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11414
11415 ins_encode %{
11416 __ ornw(as_Register($dst$$reg),
11417 as_Register($src1$$reg),
11418 as_Register($src2$$reg),
11419 Assembler::LSR,
11420 $src3$$constant & 0x1f);
11421 %}
11422
11423 ins_pipe(ialu_reg_reg_shift);
11424 %}
11425
11426 // This pattern is automatically generated from aarch64_ad.m4
11427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11428 // val | (-1 ^ (val >>> shift)) ==> orn
11429 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11430 iRegL src1, iRegL src2,
11431 immI src3, immL_M1 src4) %{
11432 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11433 ins_cost(1.9 * INSN_COST);
11434 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11435
11436 ins_encode %{
11437 __ orn(as_Register($dst$$reg),
11438 as_Register($src1$$reg),
11439 as_Register($src2$$reg),
11440 Assembler::LSR,
11441 $src3$$constant & 0x3f);
11442 %}
11443
11444 ins_pipe(ialu_reg_reg_shift);
11445 %}
11446
11447 // This pattern is automatically generated from aarch64_ad.m4
11448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11449 // val | (-1 ^ (val >> shift)) ==> ornw
11450 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11451 iRegIorL2I src1, iRegIorL2I src2,
11452 immI src3, immI_M1 src4) %{
11453 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11454 ins_cost(1.9 * INSN_COST);
11455 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11456
11457 ins_encode %{
11458 __ ornw(as_Register($dst$$reg),
11459 as_Register($src1$$reg),
11460 as_Register($src2$$reg),
11461 Assembler::ASR,
11462 $src3$$constant & 0x1f);
11463 %}
11464
11465 ins_pipe(ialu_reg_reg_shift);
11466 %}
11467
11468 // This pattern is automatically generated from aarch64_ad.m4
11469 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11470 // val | (-1 ^ (val >> shift)) ==> orn
11471 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11472 iRegL src1, iRegL src2,
11473 immI src3, immL_M1 src4) %{
11474 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11475 ins_cost(1.9 * INSN_COST);
11476 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11477
11478 ins_encode %{
11479 __ orn(as_Register($dst$$reg),
11480 as_Register($src1$$reg),
11481 as_Register($src2$$reg),
11482 Assembler::ASR,
11483 $src3$$constant & 0x3f);
11484 %}
11485
11486 ins_pipe(ialu_reg_reg_shift);
11487 %}
11488
11489 // This pattern is automatically generated from aarch64_ad.m4
11490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11491 // val | (-1 ^ (val ror shift)) ==> ornw
11492 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11493 iRegIorL2I src1, iRegIorL2I src2,
11494 immI src3, immI_M1 src4) %{
11495 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11496 ins_cost(1.9 * INSN_COST);
11497 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11498
11499 ins_encode %{
11500 __ ornw(as_Register($dst$$reg),
11501 as_Register($src1$$reg),
11502 as_Register($src2$$reg),
11503 Assembler::ROR,
11504 $src3$$constant & 0x1f);
11505 %}
11506
11507 ins_pipe(ialu_reg_reg_shift);
11508 %}
11509
11510 // This pattern is automatically generated from aarch64_ad.m4
11511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11512 // val | (-1 ^ (val ror shift)) ==> orn
11513 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11514 iRegL src1, iRegL src2,
11515 immI src3, immL_M1 src4) %{
11516 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11517 ins_cost(1.9 * INSN_COST);
11518 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11519
11520 ins_encode %{
11521 __ orn(as_Register($dst$$reg),
11522 as_Register($src1$$reg),
11523 as_Register($src2$$reg),
11524 Assembler::ROR,
11525 $src3$$constant & 0x3f);
11526 %}
11527
11528 ins_pipe(ialu_reg_reg_shift);
11529 %}
11530
11531 // This pattern is automatically generated from aarch64_ad.m4
11532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11533 // val | (-1 ^ (val << shift)) ==> ornw
11534 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11535 iRegIorL2I src1, iRegIorL2I src2,
11536 immI src3, immI_M1 src4) %{
11537 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11538 ins_cost(1.9 * INSN_COST);
11539 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11540
11541 ins_encode %{
11542 __ ornw(as_Register($dst$$reg),
11543 as_Register($src1$$reg),
11544 as_Register($src2$$reg),
11545 Assembler::LSL,
11546 $src3$$constant & 0x1f);
11547 %}
11548
11549 ins_pipe(ialu_reg_reg_shift);
11550 %}
11551
11552 // This pattern is automatically generated from aarch64_ad.m4
11553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11554 // val | (-1 ^ (val << shift)) ==> orn
11555 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11556 iRegL src1, iRegL src2,
11557 immI src3, immL_M1 src4) %{
11558 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11559 ins_cost(1.9 * INSN_COST);
11560 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11561
11562 ins_encode %{
11563 __ orn(as_Register($dst$$reg),
11564 as_Register($src1$$reg),
11565 as_Register($src2$$reg),
11566 Assembler::LSL,
11567 $src3$$constant & 0x3f);
11568 %}
11569
11570 ins_pipe(ialu_reg_reg_shift);
11571 %}
11572
11573 // This pattern is automatically generated from aarch64_ad.m4
11574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11575 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11576 iRegIorL2I src1, iRegIorL2I src2,
11577 immI src3) %{
11578 match(Set dst (AndI src1 (URShiftI src2 src3)));
11579
11580 ins_cost(1.9 * INSN_COST);
11581 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11582
11583 ins_encode %{
11584 __ andw(as_Register($dst$$reg),
11585 as_Register($src1$$reg),
11586 as_Register($src2$$reg),
11587 Assembler::LSR,
11588 $src3$$constant & 0x1f);
11589 %}
11590
11591 ins_pipe(ialu_reg_reg_shift);
11592 %}
11593
11594 // This pattern is automatically generated from aarch64_ad.m4
11595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11596 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11597 iRegL src1, iRegL src2,
11598 immI src3) %{
11599 match(Set dst (AndL src1 (URShiftL src2 src3)));
11600
11601 ins_cost(1.9 * INSN_COST);
11602 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11603
11604 ins_encode %{
11605 __ andr(as_Register($dst$$reg),
11606 as_Register($src1$$reg),
11607 as_Register($src2$$reg),
11608 Assembler::LSR,
11609 $src3$$constant & 0x3f);
11610 %}
11611
11612 ins_pipe(ialu_reg_reg_shift);
11613 %}
11614
11615 // This pattern is automatically generated from aarch64_ad.m4
11616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11617 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11618 iRegIorL2I src1, iRegIorL2I src2,
11619 immI src3) %{
11620 match(Set dst (AndI src1 (RShiftI src2 src3)));
11621
11622 ins_cost(1.9 * INSN_COST);
11623 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11624
11625 ins_encode %{
11626 __ andw(as_Register($dst$$reg),
11627 as_Register($src1$$reg),
11628 as_Register($src2$$reg),
11629 Assembler::ASR,
11630 $src3$$constant & 0x1f);
11631 %}
11632
11633 ins_pipe(ialu_reg_reg_shift);
11634 %}
11635
11636 // This pattern is automatically generated from aarch64_ad.m4
11637 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11638 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11639 iRegL src1, iRegL src2,
11640 immI src3) %{
11641 match(Set dst (AndL src1 (RShiftL src2 src3)));
11642
11643 ins_cost(1.9 * INSN_COST);
11644 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11645
11646 ins_encode %{
11647 __ andr(as_Register($dst$$reg),
11648 as_Register($src1$$reg),
11649 as_Register($src2$$reg),
11650 Assembler::ASR,
11651 $src3$$constant & 0x3f);
11652 %}
11653
11654 ins_pipe(ialu_reg_reg_shift);
11655 %}
11656
11657 // This pattern is automatically generated from aarch64_ad.m4
11658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11659 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11660 iRegIorL2I src1, iRegIorL2I src2,
11661 immI src3) %{
11662 match(Set dst (AndI src1 (LShiftI src2 src3)));
11663
11664 ins_cost(1.9 * INSN_COST);
11665 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11666
11667 ins_encode %{
11668 __ andw(as_Register($dst$$reg),
11669 as_Register($src1$$reg),
11670 as_Register($src2$$reg),
11671 Assembler::LSL,
11672 $src3$$constant & 0x1f);
11673 %}
11674
11675 ins_pipe(ialu_reg_reg_shift);
11676 %}
11677
11678 // This pattern is automatically generated from aarch64_ad.m4
11679 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11680 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11681 iRegL src1, iRegL src2,
11682 immI src3) %{
11683 match(Set dst (AndL src1 (LShiftL src2 src3)));
11684
11685 ins_cost(1.9 * INSN_COST);
11686 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11687
11688 ins_encode %{
11689 __ andr(as_Register($dst$$reg),
11690 as_Register($src1$$reg),
11691 as_Register($src2$$reg),
11692 Assembler::LSL,
11693 $src3$$constant & 0x3f);
11694 %}
11695
11696 ins_pipe(ialu_reg_reg_shift);
11697 %}
11698
11699 // This pattern is automatically generated from aarch64_ad.m4
11700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11701 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11702 iRegIorL2I src1, iRegIorL2I src2,
11703 immI src3) %{
11704 match(Set dst (AndI src1 (RotateRight src2 src3)));
11705
11706 ins_cost(1.9 * INSN_COST);
11707 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11708
11709 ins_encode %{
11710 __ andw(as_Register($dst$$reg),
11711 as_Register($src1$$reg),
11712 as_Register($src2$$reg),
11713 Assembler::ROR,
11714 $src3$$constant & 0x1f);
11715 %}
11716
11717 ins_pipe(ialu_reg_reg_shift);
11718 %}
11719
11720 // This pattern is automatically generated from aarch64_ad.m4
11721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11722 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11723 iRegL src1, iRegL src2,
11724 immI src3) %{
11725 match(Set dst (AndL src1 (RotateRight src2 src3)));
11726
11727 ins_cost(1.9 * INSN_COST);
11728 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11729
11730 ins_encode %{
11731 __ andr(as_Register($dst$$reg),
11732 as_Register($src1$$reg),
11733 as_Register($src2$$reg),
11734 Assembler::ROR,
11735 $src3$$constant & 0x3f);
11736 %}
11737
11738 ins_pipe(ialu_reg_reg_shift);
11739 %}
11740
11741 // This pattern is automatically generated from aarch64_ad.m4
11742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11743 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11744 iRegIorL2I src1, iRegIorL2I src2,
11745 immI src3) %{
11746 match(Set dst (XorI src1 (URShiftI src2 src3)));
11747
11748 ins_cost(1.9 * INSN_COST);
11749 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11750
11751 ins_encode %{
11752 __ eorw(as_Register($dst$$reg),
11753 as_Register($src1$$reg),
11754 as_Register($src2$$reg),
11755 Assembler::LSR,
11756 $src3$$constant & 0x1f);
11757 %}
11758
11759 ins_pipe(ialu_reg_reg_shift);
11760 %}
11761
11762 // This pattern is automatically generated from aarch64_ad.m4
11763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11764 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11765 iRegL src1, iRegL src2,
11766 immI src3) %{
11767 match(Set dst (XorL src1 (URShiftL src2 src3)));
11768
11769 ins_cost(1.9 * INSN_COST);
11770 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11771
11772 ins_encode %{
11773 __ eor(as_Register($dst$$reg),
11774 as_Register($src1$$reg),
11775 as_Register($src2$$reg),
11776 Assembler::LSR,
11777 $src3$$constant & 0x3f);
11778 %}
11779
11780 ins_pipe(ialu_reg_reg_shift);
11781 %}
11782
11783 // This pattern is automatically generated from aarch64_ad.m4
11784 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11785 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11786 iRegIorL2I src1, iRegIorL2I src2,
11787 immI src3) %{
11788 match(Set dst (XorI src1 (RShiftI src2 src3)));
11789
11790 ins_cost(1.9 * INSN_COST);
11791 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11792
11793 ins_encode %{
11794 __ eorw(as_Register($dst$$reg),
11795 as_Register($src1$$reg),
11796 as_Register($src2$$reg),
11797 Assembler::ASR,
11798 $src3$$constant & 0x1f);
11799 %}
11800
11801 ins_pipe(ialu_reg_reg_shift);
11802 %}
11803
11804 // This pattern is automatically generated from aarch64_ad.m4
11805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11806 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11807 iRegL src1, iRegL src2,
11808 immI src3) %{
11809 match(Set dst (XorL src1 (RShiftL src2 src3)));
11810
11811 ins_cost(1.9 * INSN_COST);
11812 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11813
11814 ins_encode %{
11815 __ eor(as_Register($dst$$reg),
11816 as_Register($src1$$reg),
11817 as_Register($src2$$reg),
11818 Assembler::ASR,
11819 $src3$$constant & 0x3f);
11820 %}
11821
11822 ins_pipe(ialu_reg_reg_shift);
11823 %}
11824
11825 // This pattern is automatically generated from aarch64_ad.m4
11826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11827 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11828 iRegIorL2I src1, iRegIorL2I src2,
11829 immI src3) %{
11830 match(Set dst (XorI src1 (LShiftI src2 src3)));
11831
11832 ins_cost(1.9 * INSN_COST);
11833 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11834
11835 ins_encode %{
11836 __ eorw(as_Register($dst$$reg),
11837 as_Register($src1$$reg),
11838 as_Register($src2$$reg),
11839 Assembler::LSL,
11840 $src3$$constant & 0x1f);
11841 %}
11842
11843 ins_pipe(ialu_reg_reg_shift);
11844 %}
11845
11846 // This pattern is automatically generated from aarch64_ad.m4
11847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11848 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11849 iRegL src1, iRegL src2,
11850 immI src3) %{
11851 match(Set dst (XorL src1 (LShiftL src2 src3)));
11852
11853 ins_cost(1.9 * INSN_COST);
11854 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11855
11856 ins_encode %{
11857 __ eor(as_Register($dst$$reg),
11858 as_Register($src1$$reg),
11859 as_Register($src2$$reg),
11860 Assembler::LSL,
11861 $src3$$constant & 0x3f);
11862 %}
11863
11864 ins_pipe(ialu_reg_reg_shift);
11865 %}
11866
11867 // This pattern is automatically generated from aarch64_ad.m4
11868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11869 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11870 iRegIorL2I src1, iRegIorL2I src2,
11871 immI src3) %{
11872 match(Set dst (XorI src1 (RotateRight src2 src3)));
11873
11874 ins_cost(1.9 * INSN_COST);
11875 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11876
11877 ins_encode %{
11878 __ eorw(as_Register($dst$$reg),
11879 as_Register($src1$$reg),
11880 as_Register($src2$$reg),
11881 Assembler::ROR,
11882 $src3$$constant & 0x1f);
11883 %}
11884
11885 ins_pipe(ialu_reg_reg_shift);
11886 %}
11887
11888 // This pattern is automatically generated from aarch64_ad.m4
11889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11890 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11891 iRegL src1, iRegL src2,
11892 immI src3) %{
11893 match(Set dst (XorL src1 (RotateRight src2 src3)));
11894
11895 ins_cost(1.9 * INSN_COST);
11896 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11897
11898 ins_encode %{
11899 __ eor(as_Register($dst$$reg),
11900 as_Register($src1$$reg),
11901 as_Register($src2$$reg),
11902 Assembler::ROR,
11903 $src3$$constant & 0x3f);
11904 %}
11905
11906 ins_pipe(ialu_reg_reg_shift);
11907 %}
11908
11909 // This pattern is automatically generated from aarch64_ad.m4
11910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11911 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11912 iRegIorL2I src1, iRegIorL2I src2,
11913 immI src3) %{
11914 match(Set dst (OrI src1 (URShiftI src2 src3)));
11915
11916 ins_cost(1.9 * INSN_COST);
11917 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11918
11919 ins_encode %{
11920 __ orrw(as_Register($dst$$reg),
11921 as_Register($src1$$reg),
11922 as_Register($src2$$reg),
11923 Assembler::LSR,
11924 $src3$$constant & 0x1f);
11925 %}
11926
11927 ins_pipe(ialu_reg_reg_shift);
11928 %}
11929
11930 // This pattern is automatically generated from aarch64_ad.m4
11931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11932 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11933 iRegL src1, iRegL src2,
11934 immI src3) %{
11935 match(Set dst (OrL src1 (URShiftL src2 src3)));
11936
11937 ins_cost(1.9 * INSN_COST);
11938 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11939
11940 ins_encode %{
11941 __ orr(as_Register($dst$$reg),
11942 as_Register($src1$$reg),
11943 as_Register($src2$$reg),
11944 Assembler::LSR,
11945 $src3$$constant & 0x3f);
11946 %}
11947
11948 ins_pipe(ialu_reg_reg_shift);
11949 %}
11950
11951 // This pattern is automatically generated from aarch64_ad.m4
11952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11953 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11954 iRegIorL2I src1, iRegIorL2I src2,
11955 immI src3) %{
11956 match(Set dst (OrI src1 (RShiftI src2 src3)));
11957
11958 ins_cost(1.9 * INSN_COST);
11959 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11960
11961 ins_encode %{
11962 __ orrw(as_Register($dst$$reg),
11963 as_Register($src1$$reg),
11964 as_Register($src2$$reg),
11965 Assembler::ASR,
11966 $src3$$constant & 0x1f);
11967 %}
11968
11969 ins_pipe(ialu_reg_reg_shift);
11970 %}
11971
11972 // This pattern is automatically generated from aarch64_ad.m4
11973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11974 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11975 iRegL src1, iRegL src2,
11976 immI src3) %{
11977 match(Set dst (OrL src1 (RShiftL src2 src3)));
11978
11979 ins_cost(1.9 * INSN_COST);
11980 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11981
11982 ins_encode %{
11983 __ orr(as_Register($dst$$reg),
11984 as_Register($src1$$reg),
11985 as_Register($src2$$reg),
11986 Assembler::ASR,
11987 $src3$$constant & 0x3f);
11988 %}
11989
11990 ins_pipe(ialu_reg_reg_shift);
11991 %}
11992
11993 // This pattern is automatically generated from aarch64_ad.m4
11994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11995 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11996 iRegIorL2I src1, iRegIorL2I src2,
11997 immI src3) %{
11998 match(Set dst (OrI src1 (LShiftI src2 src3)));
11999
12000 ins_cost(1.9 * INSN_COST);
12001 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12002
12003 ins_encode %{
12004 __ orrw(as_Register($dst$$reg),
12005 as_Register($src1$$reg),
12006 as_Register($src2$$reg),
12007 Assembler::LSL,
12008 $src3$$constant & 0x1f);
12009 %}
12010
12011 ins_pipe(ialu_reg_reg_shift);
12012 %}
12013
12014 // This pattern is automatically generated from aarch64_ad.m4
12015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12016 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12017 iRegL src1, iRegL src2,
12018 immI src3) %{
12019 match(Set dst (OrL src1 (LShiftL src2 src3)));
12020
12021 ins_cost(1.9 * INSN_COST);
12022 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12023
12024 ins_encode %{
12025 __ orr(as_Register($dst$$reg),
12026 as_Register($src1$$reg),
12027 as_Register($src2$$reg),
12028 Assembler::LSL,
12029 $src3$$constant & 0x3f);
12030 %}
12031
12032 ins_pipe(ialu_reg_reg_shift);
12033 %}
12034
12035 // This pattern is automatically generated from aarch64_ad.m4
12036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12037 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12038 iRegIorL2I src1, iRegIorL2I src2,
12039 immI src3) %{
12040 match(Set dst (OrI src1 (RotateRight src2 src3)));
12041
12042 ins_cost(1.9 * INSN_COST);
12043 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12044
12045 ins_encode %{
12046 __ orrw(as_Register($dst$$reg),
12047 as_Register($src1$$reg),
12048 as_Register($src2$$reg),
12049 Assembler::ROR,
12050 $src3$$constant & 0x1f);
12051 %}
12052
12053 ins_pipe(ialu_reg_reg_shift);
12054 %}
12055
12056 // This pattern is automatically generated from aarch64_ad.m4
12057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12058 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12059 iRegL src1, iRegL src2,
12060 immI src3) %{
12061 match(Set dst (OrL src1 (RotateRight src2 src3)));
12062
12063 ins_cost(1.9 * INSN_COST);
12064 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12065
12066 ins_encode %{
12067 __ orr(as_Register($dst$$reg),
12068 as_Register($src1$$reg),
12069 as_Register($src2$$reg),
12070 Assembler::ROR,
12071 $src3$$constant & 0x3f);
12072 %}
12073
12074 ins_pipe(ialu_reg_reg_shift);
12075 %}
12076
12077 // This pattern is automatically generated from aarch64_ad.m4
12078 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12079 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12080 iRegIorL2I src1, iRegIorL2I src2,
12081 immI src3) %{
12082 match(Set dst (AddI src1 (URShiftI src2 src3)));
12083
12084 ins_cost(1.9 * INSN_COST);
12085 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12086
12087 ins_encode %{
12088 __ addw(as_Register($dst$$reg),
12089 as_Register($src1$$reg),
12090 as_Register($src2$$reg),
12091 Assembler::LSR,
12092 $src3$$constant & 0x1f);
12093 %}
12094
12095 ins_pipe(ialu_reg_reg_shift);
12096 %}
12097
12098 // This pattern is automatically generated from aarch64_ad.m4
12099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12100 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12101 iRegL src1, iRegL src2,
12102 immI src3) %{
12103 match(Set dst (AddL src1 (URShiftL src2 src3)));
12104
12105 ins_cost(1.9 * INSN_COST);
12106 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12107
12108 ins_encode %{
12109 __ add(as_Register($dst$$reg),
12110 as_Register($src1$$reg),
12111 as_Register($src2$$reg),
12112 Assembler::LSR,
12113 $src3$$constant & 0x3f);
12114 %}
12115
12116 ins_pipe(ialu_reg_reg_shift);
12117 %}
12118
12119 // This pattern is automatically generated from aarch64_ad.m4
12120 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12121 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12122 iRegIorL2I src1, iRegIorL2I src2,
12123 immI src3) %{
12124 match(Set dst (AddI src1 (RShiftI src2 src3)));
12125
12126 ins_cost(1.9 * INSN_COST);
12127 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12128
12129 ins_encode %{
12130 __ addw(as_Register($dst$$reg),
12131 as_Register($src1$$reg),
12132 as_Register($src2$$reg),
12133 Assembler::ASR,
12134 $src3$$constant & 0x1f);
12135 %}
12136
12137 ins_pipe(ialu_reg_reg_shift);
12138 %}
12139
12140 // This pattern is automatically generated from aarch64_ad.m4
12141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12142 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12143 iRegL src1, iRegL src2,
12144 immI src3) %{
12145 match(Set dst (AddL src1 (RShiftL src2 src3)));
12146
12147 ins_cost(1.9 * INSN_COST);
12148 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12149
12150 ins_encode %{
12151 __ add(as_Register($dst$$reg),
12152 as_Register($src1$$reg),
12153 as_Register($src2$$reg),
12154 Assembler::ASR,
12155 $src3$$constant & 0x3f);
12156 %}
12157
12158 ins_pipe(ialu_reg_reg_shift);
12159 %}
12160
12161 // This pattern is automatically generated from aarch64_ad.m4
12162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12163 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12164 iRegIorL2I src1, iRegIorL2I src2,
12165 immI src3) %{
12166 match(Set dst (AddI src1 (LShiftI src2 src3)));
12167
12168 ins_cost(1.9 * INSN_COST);
12169 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12170
12171 ins_encode %{
12172 __ addw(as_Register($dst$$reg),
12173 as_Register($src1$$reg),
12174 as_Register($src2$$reg),
12175 Assembler::LSL,
12176 $src3$$constant & 0x1f);
12177 %}
12178
12179 ins_pipe(ialu_reg_reg_shift);
12180 %}
12181
12182 // This pattern is automatically generated from aarch64_ad.m4
12183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12184 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12185 iRegL src1, iRegL src2,
12186 immI src3) %{
12187 match(Set dst (AddL src1 (LShiftL src2 src3)));
12188
12189 ins_cost(1.9 * INSN_COST);
12190 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12191
12192 ins_encode %{
12193 __ add(as_Register($dst$$reg),
12194 as_Register($src1$$reg),
12195 as_Register($src2$$reg),
12196 Assembler::LSL,
12197 $src3$$constant & 0x3f);
12198 %}
12199
12200 ins_pipe(ialu_reg_reg_shift);
12201 %}
12202
12203 // This pattern is automatically generated from aarch64_ad.m4
12204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12205 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12206 iRegIorL2I src1, iRegIorL2I src2,
12207 immI src3) %{
12208 match(Set dst (SubI src1 (URShiftI src2 src3)));
12209
12210 ins_cost(1.9 * INSN_COST);
12211 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12212
12213 ins_encode %{
12214 __ subw(as_Register($dst$$reg),
12215 as_Register($src1$$reg),
12216 as_Register($src2$$reg),
12217 Assembler::LSR,
12218 $src3$$constant & 0x1f);
12219 %}
12220
12221 ins_pipe(ialu_reg_reg_shift);
12222 %}
12223
12224 // This pattern is automatically generated from aarch64_ad.m4
12225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12226 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12227 iRegL src1, iRegL src2,
12228 immI src3) %{
12229 match(Set dst (SubL src1 (URShiftL src2 src3)));
12230
12231 ins_cost(1.9 * INSN_COST);
12232 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12233
12234 ins_encode %{
12235 __ sub(as_Register($dst$$reg),
12236 as_Register($src1$$reg),
12237 as_Register($src2$$reg),
12238 Assembler::LSR,
12239 $src3$$constant & 0x3f);
12240 %}
12241
12242 ins_pipe(ialu_reg_reg_shift);
12243 %}
12244
12245 // This pattern is automatically generated from aarch64_ad.m4
12246 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12247 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12248 iRegIorL2I src1, iRegIorL2I src2,
12249 immI src3) %{
12250 match(Set dst (SubI src1 (RShiftI src2 src3)));
12251
12252 ins_cost(1.9 * INSN_COST);
12253 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12254
12255 ins_encode %{
12256 __ subw(as_Register($dst$$reg),
12257 as_Register($src1$$reg),
12258 as_Register($src2$$reg),
12259 Assembler::ASR,
12260 $src3$$constant & 0x1f);
12261 %}
12262
12263 ins_pipe(ialu_reg_reg_shift);
12264 %}
12265
12266 // This pattern is automatically generated from aarch64_ad.m4
12267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12268 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12269 iRegL src1, iRegL src2,
12270 immI src3) %{
12271 match(Set dst (SubL src1 (RShiftL src2 src3)));
12272
12273 ins_cost(1.9 * INSN_COST);
12274 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12275
12276 ins_encode %{
12277 __ sub(as_Register($dst$$reg),
12278 as_Register($src1$$reg),
12279 as_Register($src2$$reg),
12280 Assembler::ASR,
12281 $src3$$constant & 0x3f);
12282 %}
12283
12284 ins_pipe(ialu_reg_reg_shift);
12285 %}
12286
12287 // This pattern is automatically generated from aarch64_ad.m4
12288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12289 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12290 iRegIorL2I src1, iRegIorL2I src2,
12291 immI src3) %{
12292 match(Set dst (SubI src1 (LShiftI src2 src3)));
12293
12294 ins_cost(1.9 * INSN_COST);
12295 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12296
12297 ins_encode %{
12298 __ subw(as_Register($dst$$reg),
12299 as_Register($src1$$reg),
12300 as_Register($src2$$reg),
12301 Assembler::LSL,
12302 $src3$$constant & 0x1f);
12303 %}
12304
12305 ins_pipe(ialu_reg_reg_shift);
12306 %}
12307
12308 // This pattern is automatically generated from aarch64_ad.m4
12309 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12310 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12311 iRegL src1, iRegL src2,
12312 immI src3) %{
12313 match(Set dst (SubL src1 (LShiftL src2 src3)));
12314
12315 ins_cost(1.9 * INSN_COST);
12316 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12317
12318 ins_encode %{
12319 __ sub(as_Register($dst$$reg),
12320 as_Register($src1$$reg),
12321 as_Register($src2$$reg),
12322 Assembler::LSL,
12323 $src3$$constant & 0x3f);
12324 %}
12325
12326 ins_pipe(ialu_reg_reg_shift);
12327 %}
12328
12329 // This pattern is automatically generated from aarch64_ad.m4
12330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12331
12332 // Shift Left followed by Shift Right.
12333 // This idiom is used by the compiler for the i2b bytecode etc.
12334 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12335 %{
12336 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12337 ins_cost(INSN_COST * 2);
12338 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12339 ins_encode %{
12340 int lshift = $lshift_count$$constant & 63;
12341 int rshift = $rshift_count$$constant & 63;
12342 int s = 63 - lshift;
12343 int r = (rshift - lshift) & 63;
12344 __ sbfm(as_Register($dst$$reg),
12345 as_Register($src$$reg),
12346 r, s);
12347 %}
12348
12349 ins_pipe(ialu_reg_shift);
12350 %}
12351
12352 // This pattern is automatically generated from aarch64_ad.m4
12353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12354
12355 // Shift Left followed by Shift Right.
12356 // This idiom is used by the compiler for the i2b bytecode etc.
12357 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12358 %{
12359 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12360 ins_cost(INSN_COST * 2);
12361 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12362 ins_encode %{
12363 int lshift = $lshift_count$$constant & 31;
12364 int rshift = $rshift_count$$constant & 31;
12365 int s = 31 - lshift;
12366 int r = (rshift - lshift) & 31;
12367 __ sbfmw(as_Register($dst$$reg),
12368 as_Register($src$$reg),
12369 r, s);
12370 %}
12371
12372 ins_pipe(ialu_reg_shift);
12373 %}
12374
12375 // This pattern is automatically generated from aarch64_ad.m4
12376 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12377
12378 // Shift Left followed by Shift Right.
12379 // This idiom is used by the compiler for the i2b bytecode etc.
12380 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12381 %{
12382 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12383 ins_cost(INSN_COST * 2);
12384 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12385 ins_encode %{
12386 int lshift = $lshift_count$$constant & 63;
12387 int rshift = $rshift_count$$constant & 63;
12388 int s = 63 - lshift;
12389 int r = (rshift - lshift) & 63;
12390 __ ubfm(as_Register($dst$$reg),
12391 as_Register($src$$reg),
12392 r, s);
12393 %}
12394
12395 ins_pipe(ialu_reg_shift);
12396 %}
12397
12398 // This pattern is automatically generated from aarch64_ad.m4
12399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12400
12401 // Shift Left followed by Shift Right.
12402 // This idiom is used by the compiler for the i2b bytecode etc.
12403 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12404 %{
12405 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12406 ins_cost(INSN_COST * 2);
12407 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12408 ins_encode %{
12409 int lshift = $lshift_count$$constant & 31;
12410 int rshift = $rshift_count$$constant & 31;
12411 int s = 31 - lshift;
12412 int r = (rshift - lshift) & 31;
12413 __ ubfmw(as_Register($dst$$reg),
12414 as_Register($src$$reg),
12415 r, s);
12416 %}
12417
12418 ins_pipe(ialu_reg_shift);
12419 %}
12420
12421 // Bitfield extract with shift & mask
12422
12423 // This pattern is automatically generated from aarch64_ad.m4
12424 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12425 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12426 %{
12427 match(Set dst (AndI (URShiftI src rshift) mask));
12428 // Make sure we are not going to exceed what ubfxw can do.
12429 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12430
12431 ins_cost(INSN_COST);
12432 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12433 ins_encode %{
12434 int rshift = $rshift$$constant & 31;
12435 intptr_t mask = $mask$$constant;
12436 int width = exact_log2(mask+1);
12437 __ ubfxw(as_Register($dst$$reg),
12438 as_Register($src$$reg), rshift, width);
12439 %}
12440 ins_pipe(ialu_reg_shift);
12441 %}
12442
12443 // This pattern is automatically generated from aarch64_ad.m4
12444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12445 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12446 %{
12447 match(Set dst (AndL (URShiftL src rshift) mask));
12448 // Make sure we are not going to exceed what ubfx can do.
12449 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12450
12451 ins_cost(INSN_COST);
12452 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12453 ins_encode %{
12454 int rshift = $rshift$$constant & 63;
12455 intptr_t mask = $mask$$constant;
12456 int width = exact_log2_long(mask+1);
12457 __ ubfx(as_Register($dst$$reg),
12458 as_Register($src$$reg), rshift, width);
12459 %}
12460 ins_pipe(ialu_reg_shift);
12461 %}
12462
12463
12464 // This pattern is automatically generated from aarch64_ad.m4
12465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12466
12467 // We can use ubfx when extending an And with a mask when we know mask
12468 // is positive. We know that because immI_bitmask guarantees it.
12469 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12470 %{
12471 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12472 // Make sure we are not going to exceed what ubfxw can do.
12473 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12474
12475 ins_cost(INSN_COST * 2);
12476 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12477 ins_encode %{
12478 int rshift = $rshift$$constant & 31;
12479 intptr_t mask = $mask$$constant;
12480 int width = exact_log2(mask+1);
12481 __ ubfx(as_Register($dst$$reg),
12482 as_Register($src$$reg), rshift, width);
12483 %}
12484 ins_pipe(ialu_reg_shift);
12485 %}
12486
12487
12488 // This pattern is automatically generated from aarch64_ad.m4
12489 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12490
12491 // We can use ubfiz when masking by a positive number and then left shifting the result.
12492 // We know that the mask is positive because immI_bitmask guarantees it.
12493 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12494 %{
12495 match(Set dst (LShiftI (AndI src mask) lshift));
12496 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12497
12498 ins_cost(INSN_COST);
12499 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12500 ins_encode %{
12501 int lshift = $lshift$$constant & 31;
12502 intptr_t mask = $mask$$constant;
12503 int width = exact_log2(mask+1);
12504 __ ubfizw(as_Register($dst$$reg),
12505 as_Register($src$$reg), lshift, width);
12506 %}
12507 ins_pipe(ialu_reg_shift);
12508 %}
12509
12510 // This pattern is automatically generated from aarch64_ad.m4
12511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12512
12513 // We can use ubfiz when masking by a positive number and then left shifting the result.
12514 // We know that the mask is positive because immL_bitmask guarantees it.
12515 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12516 %{
12517 match(Set dst (LShiftL (AndL src mask) lshift));
12518 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12519
12520 ins_cost(INSN_COST);
12521 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12522 ins_encode %{
12523 int lshift = $lshift$$constant & 63;
12524 intptr_t mask = $mask$$constant;
12525 int width = exact_log2_long(mask+1);
12526 __ ubfiz(as_Register($dst$$reg),
12527 as_Register($src$$reg), lshift, width);
12528 %}
12529 ins_pipe(ialu_reg_shift);
12530 %}
12531
12532 // This pattern is automatically generated from aarch64_ad.m4
12533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12534
12535 // We can use ubfiz when masking by a positive number and then left shifting the result.
12536 // We know that the mask is positive because immI_bitmask guarantees it.
12537 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12538 %{
12539 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12540 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12541
12542 ins_cost(INSN_COST);
12543 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12544 ins_encode %{
12545 int lshift = $lshift$$constant & 31;
12546 intptr_t mask = $mask$$constant;
12547 int width = exact_log2(mask+1);
12548 __ ubfizw(as_Register($dst$$reg),
12549 as_Register($src$$reg), lshift, width);
12550 %}
12551 ins_pipe(ialu_reg_shift);
12552 %}
12553
12554 // This pattern is automatically generated from aarch64_ad.m4
12555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12556
12557 // We can use ubfiz when masking by a positive number and then left shifting the result.
12558 // We know that the mask is positive because immL_bitmask guarantees it.
12559 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12560 %{
12561 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12562 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12563
12564 ins_cost(INSN_COST);
12565 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12566 ins_encode %{
12567 int lshift = $lshift$$constant & 63;
12568 intptr_t mask = $mask$$constant;
12569 int width = exact_log2_long(mask+1);
12570 __ ubfiz(as_Register($dst$$reg),
12571 as_Register($src$$reg), lshift, width);
12572 %}
12573 ins_pipe(ialu_reg_shift);
12574 %}
12575
12576
12577 // This pattern is automatically generated from aarch64_ad.m4
12578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12579
12580 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12581 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12582 %{
12583 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12584 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12585
12586 ins_cost(INSN_COST);
12587 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12588 ins_encode %{
12589 int lshift = $lshift$$constant & 63;
12590 intptr_t mask = $mask$$constant;
12591 int width = exact_log2(mask+1);
12592 __ ubfiz(as_Register($dst$$reg),
12593 as_Register($src$$reg), lshift, width);
12594 %}
12595 ins_pipe(ialu_reg_shift);
12596 %}
12597
12598 // This pattern is automatically generated from aarch64_ad.m4
12599 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12600
12601 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12602 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12603 %{
12604 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12605 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12606
12607 ins_cost(INSN_COST);
12608 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12609 ins_encode %{
12610 int lshift = $lshift$$constant & 31;
12611 intptr_t mask = $mask$$constant;
12612 int width = exact_log2(mask+1);
12613 __ ubfiz(as_Register($dst$$reg),
12614 as_Register($src$$reg), lshift, width);
12615 %}
12616 ins_pipe(ialu_reg_shift);
12617 %}
12618
12619 // This pattern is automatically generated from aarch64_ad.m4
12620 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12621
12622 // Can skip int2long conversions after AND with small bitmask
12623 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12624 %{
12625 match(Set dst (ConvI2L (AndI src msk)));
12626 ins_cost(INSN_COST);
12627 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12628 ins_encode %{
12629 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12630 %}
12631 ins_pipe(ialu_reg_shift);
12632 %}
12633
12634
12635 // Rotations
12636
12637 // This pattern is automatically generated from aarch64_ad.m4
12638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12639 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12640 %{
12641 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12642 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12643
12644 ins_cost(INSN_COST);
12645 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12646
12647 ins_encode %{
12648 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12649 $rshift$$constant & 63);
12650 %}
12651 ins_pipe(ialu_reg_reg_extr);
12652 %}
12653
12654
12655 // This pattern is automatically generated from aarch64_ad.m4
12656 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12657 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12658 %{
12659 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12660 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12661
12662 ins_cost(INSN_COST);
12663 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12664
12665 ins_encode %{
12666 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12667 $rshift$$constant & 31);
12668 %}
12669 ins_pipe(ialu_reg_reg_extr);
12670 %}
12671
12672
12673 // This pattern is automatically generated from aarch64_ad.m4
12674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12675 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12676 %{
12677 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12678 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12679
12680 ins_cost(INSN_COST);
12681 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12682
12683 ins_encode %{
12684 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12685 $rshift$$constant & 63);
12686 %}
12687 ins_pipe(ialu_reg_reg_extr);
12688 %}
12689
12690
12691 // This pattern is automatically generated from aarch64_ad.m4
12692 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12693 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12694 %{
12695 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12696 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12697
12698 ins_cost(INSN_COST);
12699 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12700
12701 ins_encode %{
12702 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12703 $rshift$$constant & 31);
12704 %}
12705 ins_pipe(ialu_reg_reg_extr);
12706 %}
12707
12708 // This pattern is automatically generated from aarch64_ad.m4
12709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12710 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12711 %{
12712 match(Set dst (RotateRight src shift));
12713
12714 ins_cost(INSN_COST);
12715 format %{ "ror $dst, $src, $shift" %}
12716
12717 ins_encode %{
12718 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12719 $shift$$constant & 0x1f);
12720 %}
12721 ins_pipe(ialu_reg_reg_vshift);
12722 %}
12723
12724 // This pattern is automatically generated from aarch64_ad.m4
12725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12726 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12727 %{
12728 match(Set dst (RotateRight src shift));
12729
12730 ins_cost(INSN_COST);
12731 format %{ "ror $dst, $src, $shift" %}
12732
12733 ins_encode %{
12734 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12735 $shift$$constant & 0x3f);
12736 %}
12737 ins_pipe(ialu_reg_reg_vshift);
12738 %}
12739
12740 // This pattern is automatically generated from aarch64_ad.m4
12741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12742 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12743 %{
12744 match(Set dst (RotateRight src shift));
12745
12746 ins_cost(INSN_COST);
12747 format %{ "ror $dst, $src, $shift" %}
12748
12749 ins_encode %{
12750 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12751 %}
12752 ins_pipe(ialu_reg_reg_vshift);
12753 %}
12754
12755 // This pattern is automatically generated from aarch64_ad.m4
12756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12757 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12758 %{
12759 match(Set dst (RotateRight src shift));
12760
12761 ins_cost(INSN_COST);
12762 format %{ "ror $dst, $src, $shift" %}
12763
12764 ins_encode %{
12765 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12766 %}
12767 ins_pipe(ialu_reg_reg_vshift);
12768 %}
12769
12770 // This pattern is automatically generated from aarch64_ad.m4
12771 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12772 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12773 %{
12774 match(Set dst (RotateLeft src shift));
12775
12776 ins_cost(INSN_COST);
12777 format %{ "rol $dst, $src, $shift" %}
12778
12779 ins_encode %{
12780 __ subw(rscratch1, zr, as_Register($shift$$reg));
12781 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12782 %}
12783 ins_pipe(ialu_reg_reg_vshift);
12784 %}
12785
12786 // This pattern is automatically generated from aarch64_ad.m4
12787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12788 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12789 %{
12790 match(Set dst (RotateLeft src shift));
12791
12792 ins_cost(INSN_COST);
12793 format %{ "rol $dst, $src, $shift" %}
12794
12795 ins_encode %{
12796 __ subw(rscratch1, zr, as_Register($shift$$reg));
12797 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12798 %}
12799 ins_pipe(ialu_reg_reg_vshift);
12800 %}
12801
12802
12803 // Add/subtract (extended)
12804
12805 // This pattern is automatically generated from aarch64_ad.m4
12806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12807 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12808 %{
12809 match(Set dst (AddL src1 (ConvI2L src2)));
12810 ins_cost(INSN_COST);
12811 format %{ "add $dst, $src1, $src2, sxtw" %}
12812
12813 ins_encode %{
12814 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12815 as_Register($src2$$reg), ext::sxtw);
12816 %}
12817 ins_pipe(ialu_reg_reg);
12818 %}
12819
12820 // This pattern is automatically generated from aarch64_ad.m4
12821 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12822 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12823 %{
12824 match(Set dst (SubL src1 (ConvI2L src2)));
12825 ins_cost(INSN_COST);
12826 format %{ "sub $dst, $src1, $src2, sxtw" %}
12827
12828 ins_encode %{
12829 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12830 as_Register($src2$$reg), ext::sxtw);
12831 %}
12832 ins_pipe(ialu_reg_reg);
12833 %}
12834
12835 // This pattern is automatically generated from aarch64_ad.m4
12836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12837 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12838 %{
12839 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12840 ins_cost(INSN_COST);
12841 format %{ "add $dst, $src1, $src2, sxth" %}
12842
12843 ins_encode %{
12844 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12845 as_Register($src2$$reg), ext::sxth);
12846 %}
12847 ins_pipe(ialu_reg_reg);
12848 %}
12849
12850 // This pattern is automatically generated from aarch64_ad.m4
12851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12852 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12853 %{
12854 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12855 ins_cost(INSN_COST);
12856 format %{ "add $dst, $src1, $src2, sxtb" %}
12857
12858 ins_encode %{
12859 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12860 as_Register($src2$$reg), ext::sxtb);
12861 %}
12862 ins_pipe(ialu_reg_reg);
12863 %}
12864
12865 // This pattern is automatically generated from aarch64_ad.m4
12866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12867 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12868 %{
12869 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12870 ins_cost(INSN_COST);
12871 format %{ "add $dst, $src1, $src2, uxtb" %}
12872
12873 ins_encode %{
12874 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12875 as_Register($src2$$reg), ext::uxtb);
12876 %}
12877 ins_pipe(ialu_reg_reg);
12878 %}
12879
12880 // This pattern is automatically generated from aarch64_ad.m4
12881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12882 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12883 %{
12884 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12885 ins_cost(INSN_COST);
12886 format %{ "add $dst, $src1, $src2, sxth" %}
12887
12888 ins_encode %{
12889 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12890 as_Register($src2$$reg), ext::sxth);
12891 %}
12892 ins_pipe(ialu_reg_reg);
12893 %}
12894
12895 // This pattern is automatically generated from aarch64_ad.m4
12896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12897 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12898 %{
12899 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12900 ins_cost(INSN_COST);
12901 format %{ "add $dst, $src1, $src2, sxtw" %}
12902
12903 ins_encode %{
12904 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12905 as_Register($src2$$reg), ext::sxtw);
12906 %}
12907 ins_pipe(ialu_reg_reg);
12908 %}
12909
12910 // This pattern is automatically generated from aarch64_ad.m4
12911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12912 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12913 %{
12914 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12915 ins_cost(INSN_COST);
12916 format %{ "add $dst, $src1, $src2, sxtb" %}
12917
12918 ins_encode %{
12919 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12920 as_Register($src2$$reg), ext::sxtb);
12921 %}
12922 ins_pipe(ialu_reg_reg);
12923 %}
12924
12925 // This pattern is automatically generated from aarch64_ad.m4
12926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12927 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12928 %{
12929 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12930 ins_cost(INSN_COST);
12931 format %{ "add $dst, $src1, $src2, uxtb" %}
12932
12933 ins_encode %{
12934 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12935 as_Register($src2$$reg), ext::uxtb);
12936 %}
12937 ins_pipe(ialu_reg_reg);
12938 %}
12939
12940 // This pattern is automatically generated from aarch64_ad.m4
12941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12942 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12943 %{
12944 match(Set dst (AddI src1 (AndI src2 mask)));
12945 ins_cost(INSN_COST);
12946 format %{ "addw $dst, $src1, $src2, uxtb" %}
12947
12948 ins_encode %{
12949 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12950 as_Register($src2$$reg), ext::uxtb);
12951 %}
12952 ins_pipe(ialu_reg_reg);
12953 %}
12954
12955 // This pattern is automatically generated from aarch64_ad.m4
12956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12957 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12958 %{
12959 match(Set dst (AddI src1 (AndI src2 mask)));
12960 ins_cost(INSN_COST);
12961 format %{ "addw $dst, $src1, $src2, uxth" %}
12962
12963 ins_encode %{
12964 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12965 as_Register($src2$$reg), ext::uxth);
12966 %}
12967 ins_pipe(ialu_reg_reg);
12968 %}
12969
12970 // This pattern is automatically generated from aarch64_ad.m4
12971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12972 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12973 %{
12974 match(Set dst (AddL src1 (AndL src2 mask)));
12975 ins_cost(INSN_COST);
12976 format %{ "add $dst, $src1, $src2, uxtb" %}
12977
12978 ins_encode %{
12979 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12980 as_Register($src2$$reg), ext::uxtb);
12981 %}
12982 ins_pipe(ialu_reg_reg);
12983 %}
12984
12985 // This pattern is automatically generated from aarch64_ad.m4
12986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12987 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12988 %{
12989 match(Set dst (AddL src1 (AndL src2 mask)));
12990 ins_cost(INSN_COST);
12991 format %{ "add $dst, $src1, $src2, uxth" %}
12992
12993 ins_encode %{
12994 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12995 as_Register($src2$$reg), ext::uxth);
12996 %}
12997 ins_pipe(ialu_reg_reg);
12998 %}
12999
13000 // This pattern is automatically generated from aarch64_ad.m4
13001 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13002 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13003 %{
13004 match(Set dst (AddL src1 (AndL src2 mask)));
13005 ins_cost(INSN_COST);
13006 format %{ "add $dst, $src1, $src2, uxtw" %}
13007
13008 ins_encode %{
13009 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13010 as_Register($src2$$reg), ext::uxtw);
13011 %}
13012 ins_pipe(ialu_reg_reg);
13013 %}
13014
13015 // This pattern is automatically generated from aarch64_ad.m4
13016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13017 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13018 %{
13019 match(Set dst (SubI src1 (AndI src2 mask)));
13020 ins_cost(INSN_COST);
13021 format %{ "subw $dst, $src1, $src2, uxtb" %}
13022
13023 ins_encode %{
13024 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13025 as_Register($src2$$reg), ext::uxtb);
13026 %}
13027 ins_pipe(ialu_reg_reg);
13028 %}
13029
13030 // This pattern is automatically generated from aarch64_ad.m4
13031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13032 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13033 %{
13034 match(Set dst (SubI src1 (AndI src2 mask)));
13035 ins_cost(INSN_COST);
13036 format %{ "subw $dst, $src1, $src2, uxth" %}
13037
13038 ins_encode %{
13039 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13040 as_Register($src2$$reg), ext::uxth);
13041 %}
13042 ins_pipe(ialu_reg_reg);
13043 %}
13044
13045 // This pattern is automatically generated from aarch64_ad.m4
13046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13047 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13048 %{
13049 match(Set dst (SubL src1 (AndL src2 mask)));
13050 ins_cost(INSN_COST);
13051 format %{ "sub $dst, $src1, $src2, uxtb" %}
13052
13053 ins_encode %{
13054 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13055 as_Register($src2$$reg), ext::uxtb);
13056 %}
13057 ins_pipe(ialu_reg_reg);
13058 %}
13059
13060 // This pattern is automatically generated from aarch64_ad.m4
13061 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13062 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13063 %{
13064 match(Set dst (SubL src1 (AndL src2 mask)));
13065 ins_cost(INSN_COST);
13066 format %{ "sub $dst, $src1, $src2, uxth" %}
13067
13068 ins_encode %{
13069 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13070 as_Register($src2$$reg), ext::uxth);
13071 %}
13072 ins_pipe(ialu_reg_reg);
13073 %}
13074
13075 // This pattern is automatically generated from aarch64_ad.m4
13076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13077 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13078 %{
13079 match(Set dst (SubL src1 (AndL src2 mask)));
13080 ins_cost(INSN_COST);
13081 format %{ "sub $dst, $src1, $src2, uxtw" %}
13082
13083 ins_encode %{
13084 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13085 as_Register($src2$$reg), ext::uxtw);
13086 %}
13087 ins_pipe(ialu_reg_reg);
13088 %}
13089
13090
13091 // This pattern is automatically generated from aarch64_ad.m4
13092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13093 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13094 %{
13095 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13096 ins_cost(1.9 * INSN_COST);
13097 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13098
13099 ins_encode %{
13100 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13101 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13102 %}
13103 ins_pipe(ialu_reg_reg_shift);
13104 %}
13105
13106 // This pattern is automatically generated from aarch64_ad.m4
13107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13108 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13109 %{
13110 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13111 ins_cost(1.9 * INSN_COST);
13112 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13113
13114 ins_encode %{
13115 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13116 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13117 %}
13118 ins_pipe(ialu_reg_reg_shift);
13119 %}
13120
13121 // This pattern is automatically generated from aarch64_ad.m4
13122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13123 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13124 %{
13125 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13126 ins_cost(1.9 * INSN_COST);
13127 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13128
13129 ins_encode %{
13130 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13131 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13132 %}
13133 ins_pipe(ialu_reg_reg_shift);
13134 %}
13135
13136 // This pattern is automatically generated from aarch64_ad.m4
13137 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13138 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13139 %{
13140 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13141 ins_cost(1.9 * INSN_COST);
13142 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13143
13144 ins_encode %{
13145 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13146 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13147 %}
13148 ins_pipe(ialu_reg_reg_shift);
13149 %}
13150
13151 // This pattern is automatically generated from aarch64_ad.m4
13152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13153 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13154 %{
13155 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13156 ins_cost(1.9 * INSN_COST);
13157 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13158
13159 ins_encode %{
13160 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13161 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13162 %}
13163 ins_pipe(ialu_reg_reg_shift);
13164 %}
13165
13166 // This pattern is automatically generated from aarch64_ad.m4
13167 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13168 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13169 %{
13170 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13171 ins_cost(1.9 * INSN_COST);
13172 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13173
13174 ins_encode %{
13175 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13176 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13177 %}
13178 ins_pipe(ialu_reg_reg_shift);
13179 %}
13180
13181 // This pattern is automatically generated from aarch64_ad.m4
13182 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13183 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13184 %{
13185 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13186 ins_cost(1.9 * INSN_COST);
13187 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13188
13189 ins_encode %{
13190 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13191 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13192 %}
13193 ins_pipe(ialu_reg_reg_shift);
13194 %}
13195
13196 // This pattern is automatically generated from aarch64_ad.m4
13197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13198 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13199 %{
13200 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13201 ins_cost(1.9 * INSN_COST);
13202 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13203
13204 ins_encode %{
13205 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13206 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13207 %}
13208 ins_pipe(ialu_reg_reg_shift);
13209 %}
13210
13211 // This pattern is automatically generated from aarch64_ad.m4
13212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13213 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13214 %{
13215 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13216 ins_cost(1.9 * INSN_COST);
13217 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13218
13219 ins_encode %{
13220 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13221 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13222 %}
13223 ins_pipe(ialu_reg_reg_shift);
13224 %}
13225
13226 // This pattern is automatically generated from aarch64_ad.m4
13227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13228 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13229 %{
13230 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13231 ins_cost(1.9 * INSN_COST);
13232 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13233
13234 ins_encode %{
13235 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13236 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13237 %}
13238 ins_pipe(ialu_reg_reg_shift);
13239 %}
13240
13241 // This pattern is automatically generated from aarch64_ad.m4
13242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13243 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13244 %{
13245 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13246 ins_cost(1.9 * INSN_COST);
13247 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13248
13249 ins_encode %{
13250 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13251 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13252 %}
13253 ins_pipe(ialu_reg_reg_shift);
13254 %}
13255
13256 // This pattern is automatically generated from aarch64_ad.m4
13257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13258 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13259 %{
13260 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13261 ins_cost(1.9 * INSN_COST);
13262 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13263
13264 ins_encode %{
13265 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13266 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13267 %}
13268 ins_pipe(ialu_reg_reg_shift);
13269 %}
13270
13271 // This pattern is automatically generated from aarch64_ad.m4
13272 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13273 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13274 %{
13275 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13276 ins_cost(1.9 * INSN_COST);
13277 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13278
13279 ins_encode %{
13280 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13281 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13282 %}
13283 ins_pipe(ialu_reg_reg_shift);
13284 %}
13285
13286 // This pattern is automatically generated from aarch64_ad.m4
13287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13288 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13289 %{
13290 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13291 ins_cost(1.9 * INSN_COST);
13292 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13293
13294 ins_encode %{
13295 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13296 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13297 %}
13298 ins_pipe(ialu_reg_reg_shift);
13299 %}
13300
13301 // This pattern is automatically generated from aarch64_ad.m4
13302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13303 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13304 %{
13305 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13306 ins_cost(1.9 * INSN_COST);
13307 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13308
13309 ins_encode %{
13310 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13311 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13312 %}
13313 ins_pipe(ialu_reg_reg_shift);
13314 %}
13315
13316 // This pattern is automatically generated from aarch64_ad.m4
13317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13318 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13319 %{
13320 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13321 ins_cost(1.9 * INSN_COST);
13322 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13323
13324 ins_encode %{
13325 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13326 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13327 %}
13328 ins_pipe(ialu_reg_reg_shift);
13329 %}
13330
13331 // This pattern is automatically generated from aarch64_ad.m4
13332 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13333 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13334 %{
13335 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13336 ins_cost(1.9 * INSN_COST);
13337 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13338
13339 ins_encode %{
13340 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13341 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13342 %}
13343 ins_pipe(ialu_reg_reg_shift);
13344 %}
13345
13346 // This pattern is automatically generated from aarch64_ad.m4
13347 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13348 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13349 %{
13350 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13351 ins_cost(1.9 * INSN_COST);
13352 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13353
13354 ins_encode %{
13355 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13356 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13357 %}
13358 ins_pipe(ialu_reg_reg_shift);
13359 %}
13360
13361 // This pattern is automatically generated from aarch64_ad.m4
13362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13363 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13364 %{
13365 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13366 ins_cost(1.9 * INSN_COST);
13367 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13368
13369 ins_encode %{
13370 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13371 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13372 %}
13373 ins_pipe(ialu_reg_reg_shift);
13374 %}
13375
13376 // This pattern is automatically generated from aarch64_ad.m4
13377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13378 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13379 %{
13380 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13381 ins_cost(1.9 * INSN_COST);
13382 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13383
13384 ins_encode %{
13385 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13386 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13387 %}
13388 ins_pipe(ialu_reg_reg_shift);
13389 %}
13390
13391 // This pattern is automatically generated from aarch64_ad.m4
13392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13393 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13394 %{
13395 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13396 ins_cost(1.9 * INSN_COST);
13397 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13398
13399 ins_encode %{
13400 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13401 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13402 %}
13403 ins_pipe(ialu_reg_reg_shift);
13404 %}
13405
13406 // This pattern is automatically generated from aarch64_ad.m4
13407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13408 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13409 %{
13410 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13411 ins_cost(1.9 * INSN_COST);
13412 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13413
13414 ins_encode %{
13415 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13416 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13417 %}
13418 ins_pipe(ialu_reg_reg_shift);
13419 %}
13420
13421 // This pattern is automatically generated from aarch64_ad.m4
13422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13423 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13424 %{
13425 effect(DEF dst, USE src1, USE src2, USE cr);
13426 ins_cost(INSN_COST * 2);
13427 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13428
13429 ins_encode %{
13430 __ cselw($dst$$Register,
13431 $src1$$Register,
13432 $src2$$Register,
13433 Assembler::LT);
13434 %}
13435 ins_pipe(icond_reg_reg);
13436 %}
13437
13438 // This pattern is automatically generated from aarch64_ad.m4
13439 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13440 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13441 %{
13442 effect(DEF dst, USE src1, USE src2, USE cr);
13443 ins_cost(INSN_COST * 2);
13444 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13445
13446 ins_encode %{
13447 __ cselw($dst$$Register,
13448 $src1$$Register,
13449 $src2$$Register,
13450 Assembler::GT);
13451 %}
13452 ins_pipe(icond_reg_reg);
13453 %}
13454
13455 // This pattern is automatically generated from aarch64_ad.m4
13456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13457 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13458 %{
13459 effect(DEF dst, USE src1, USE cr);
13460 ins_cost(INSN_COST * 2);
13461 format %{ "cselw $dst, $src1, zr lt\t" %}
13462
13463 ins_encode %{
13464 __ cselw($dst$$Register,
13465 $src1$$Register,
13466 zr,
13467 Assembler::LT);
13468 %}
13469 ins_pipe(icond_reg);
13470 %}
13471
13472 // This pattern is automatically generated from aarch64_ad.m4
13473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13474 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13475 %{
13476 effect(DEF dst, USE src1, USE cr);
13477 ins_cost(INSN_COST * 2);
13478 format %{ "cselw $dst, $src1, zr gt\t" %}
13479
13480 ins_encode %{
13481 __ cselw($dst$$Register,
13482 $src1$$Register,
13483 zr,
13484 Assembler::GT);
13485 %}
13486 ins_pipe(icond_reg);
13487 %}
13488
13489 // This pattern is automatically generated from aarch64_ad.m4
13490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13491 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13492 %{
13493 effect(DEF dst, USE src1, USE cr);
13494 ins_cost(INSN_COST * 2);
13495 format %{ "csincw $dst, $src1, zr le\t" %}
13496
13497 ins_encode %{
13498 __ csincw($dst$$Register,
13499 $src1$$Register,
13500 zr,
13501 Assembler::LE);
13502 %}
13503 ins_pipe(icond_reg);
13504 %}
13505
13506 // This pattern is automatically generated from aarch64_ad.m4
13507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13508 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13509 %{
13510 effect(DEF dst, USE src1, USE cr);
13511 ins_cost(INSN_COST * 2);
13512 format %{ "csincw $dst, $src1, zr gt\t" %}
13513
13514 ins_encode %{
13515 __ csincw($dst$$Register,
13516 $src1$$Register,
13517 zr,
13518 Assembler::GT);
13519 %}
13520 ins_pipe(icond_reg);
13521 %}
13522
13523 // This pattern is automatically generated from aarch64_ad.m4
13524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13525 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13526 %{
13527 effect(DEF dst, USE src1, USE cr);
13528 ins_cost(INSN_COST * 2);
13529 format %{ "csinvw $dst, $src1, zr lt\t" %}
13530
13531 ins_encode %{
13532 __ csinvw($dst$$Register,
13533 $src1$$Register,
13534 zr,
13535 Assembler::LT);
13536 %}
13537 ins_pipe(icond_reg);
13538 %}
13539
13540 // This pattern is automatically generated from aarch64_ad.m4
13541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13542 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13543 %{
13544 effect(DEF dst, USE src1, USE cr);
13545 ins_cost(INSN_COST * 2);
13546 format %{ "csinvw $dst, $src1, zr ge\t" %}
13547
13548 ins_encode %{
13549 __ csinvw($dst$$Register,
13550 $src1$$Register,
13551 zr,
13552 Assembler::GE);
13553 %}
13554 ins_pipe(icond_reg);
13555 %}
13556
13557 // This pattern is automatically generated from aarch64_ad.m4
13558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13559 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13560 %{
13561 match(Set dst (MinI src imm));
13562 ins_cost(INSN_COST * 3);
13563 expand %{
13564 rFlagsReg cr;
13565 compI_reg_imm0(cr, src);
13566 cmovI_reg_imm0_lt(dst, src, cr);
13567 %}
13568 %}
13569
13570 // This pattern is automatically generated from aarch64_ad.m4
13571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13572 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13573 %{
13574 match(Set dst (MinI imm src));
13575 ins_cost(INSN_COST * 3);
13576 expand %{
13577 rFlagsReg cr;
13578 compI_reg_imm0(cr, src);
13579 cmovI_reg_imm0_lt(dst, src, cr);
13580 %}
13581 %}
13582
13583 // This pattern is automatically generated from aarch64_ad.m4
13584 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13585 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13586 %{
13587 match(Set dst (MinI src imm));
13588 ins_cost(INSN_COST * 3);
13589 expand %{
13590 rFlagsReg cr;
13591 compI_reg_imm0(cr, src);
13592 cmovI_reg_imm1_le(dst, src, cr);
13593 %}
13594 %}
13595
13596 // This pattern is automatically generated from aarch64_ad.m4
13597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13598 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13599 %{
13600 match(Set dst (MinI imm src));
13601 ins_cost(INSN_COST * 3);
13602 expand %{
13603 rFlagsReg cr;
13604 compI_reg_imm0(cr, src);
13605 cmovI_reg_imm1_le(dst, src, cr);
13606 %}
13607 %}
13608
13609 // This pattern is automatically generated from aarch64_ad.m4
13610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13611 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13612 %{
13613 match(Set dst (MinI src imm));
13614 ins_cost(INSN_COST * 3);
13615 expand %{
13616 rFlagsReg cr;
13617 compI_reg_imm0(cr, src);
13618 cmovI_reg_immM1_lt(dst, src, cr);
13619 %}
13620 %}
13621
13622 // This pattern is automatically generated from aarch64_ad.m4
13623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13624 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13625 %{
13626 match(Set dst (MinI imm src));
13627 ins_cost(INSN_COST * 3);
13628 expand %{
13629 rFlagsReg cr;
13630 compI_reg_imm0(cr, src);
13631 cmovI_reg_immM1_lt(dst, src, cr);
13632 %}
13633 %}
13634
13635 // This pattern is automatically generated from aarch64_ad.m4
13636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13637 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13638 %{
13639 match(Set dst (MaxI src imm));
13640 ins_cost(INSN_COST * 3);
13641 expand %{
13642 rFlagsReg cr;
13643 compI_reg_imm0(cr, src);
13644 cmovI_reg_imm0_gt(dst, src, cr);
13645 %}
13646 %}
13647
13648 // This pattern is automatically generated from aarch64_ad.m4
13649 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13650 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13651 %{
13652 match(Set dst (MaxI imm src));
13653 ins_cost(INSN_COST * 3);
13654 expand %{
13655 rFlagsReg cr;
13656 compI_reg_imm0(cr, src);
13657 cmovI_reg_imm0_gt(dst, src, cr);
13658 %}
13659 %}
13660
13661 // This pattern is automatically generated from aarch64_ad.m4
13662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13663 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13664 %{
13665 match(Set dst (MaxI src imm));
13666 ins_cost(INSN_COST * 3);
13667 expand %{
13668 rFlagsReg cr;
13669 compI_reg_imm0(cr, src);
13670 cmovI_reg_imm1_gt(dst, src, cr);
13671 %}
13672 %}
13673
13674 // This pattern is automatically generated from aarch64_ad.m4
13675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13676 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13677 %{
13678 match(Set dst (MaxI imm src));
13679 ins_cost(INSN_COST * 3);
13680 expand %{
13681 rFlagsReg cr;
13682 compI_reg_imm0(cr, src);
13683 cmovI_reg_imm1_gt(dst, src, cr);
13684 %}
13685 %}
13686
13687 // This pattern is automatically generated from aarch64_ad.m4
13688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13689 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13690 %{
13691 match(Set dst (MaxI src imm));
13692 ins_cost(INSN_COST * 3);
13693 expand %{
13694 rFlagsReg cr;
13695 compI_reg_imm0(cr, src);
13696 cmovI_reg_immM1_ge(dst, src, cr);
13697 %}
13698 %}
13699
13700 // This pattern is automatically generated from aarch64_ad.m4
13701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13702 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13703 %{
13704 match(Set dst (MaxI imm src));
13705 ins_cost(INSN_COST * 3);
13706 expand %{
13707 rFlagsReg cr;
13708 compI_reg_imm0(cr, src);
13709 cmovI_reg_immM1_ge(dst, src, cr);
13710 %}
13711 %}
13712
13713 // This pattern is automatically generated from aarch64_ad.m4
13714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13715 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13716 %{
13717 match(Set dst (ReverseI src));
13718 ins_cost(INSN_COST);
13719 format %{ "rbitw $dst, $src" %}
13720 ins_encode %{
13721 __ rbitw($dst$$Register, $src$$Register);
13722 %}
13723 ins_pipe(ialu_reg);
13724 %}
13725
13726 // This pattern is automatically generated from aarch64_ad.m4
13727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13728 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13729 %{
13730 match(Set dst (ReverseL src));
13731 ins_cost(INSN_COST);
13732 format %{ "rbit $dst, $src" %}
13733 ins_encode %{
13734 __ rbit($dst$$Register, $src$$Register);
13735 %}
13736 ins_pipe(ialu_reg);
13737 %}
13738
13739
13740 // END This section of the file is automatically generated. Do not edit --------------
13741
13742
13743 // ============================================================================
13744 // Floating Point Arithmetic Instructions
13745
13746 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13747 match(Set dst (AddHF src1 src2));
13748 format %{ "faddh $dst, $src1, $src2" %}
13749 ins_encode %{
13750 __ faddh($dst$$FloatRegister,
13751 $src1$$FloatRegister,
13752 $src2$$FloatRegister);
13753 %}
13754 ins_pipe(fp_dop_reg_reg_s);
13755 %}
13756
13757 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13758 match(Set dst (AddF src1 src2));
13759
13760 ins_cost(INSN_COST * 5);
13761 format %{ "fadds $dst, $src1, $src2" %}
13762
13763 ins_encode %{
13764 __ fadds(as_FloatRegister($dst$$reg),
13765 as_FloatRegister($src1$$reg),
13766 as_FloatRegister($src2$$reg));
13767 %}
13768
13769 ins_pipe(fp_dop_reg_reg_s);
13770 %}
13771
13772 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13773 match(Set dst (AddD src1 src2));
13774
13775 ins_cost(INSN_COST * 5);
13776 format %{ "faddd $dst, $src1, $src2" %}
13777
13778 ins_encode %{
13779 __ faddd(as_FloatRegister($dst$$reg),
13780 as_FloatRegister($src1$$reg),
13781 as_FloatRegister($src2$$reg));
13782 %}
13783
13784 ins_pipe(fp_dop_reg_reg_d);
13785 %}
13786
13787 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13788 match(Set dst (SubHF src1 src2));
13789 format %{ "fsubh $dst, $src1, $src2" %}
13790 ins_encode %{
13791 __ fsubh($dst$$FloatRegister,
13792 $src1$$FloatRegister,
13793 $src2$$FloatRegister);
13794 %}
13795 ins_pipe(fp_dop_reg_reg_s);
13796 %}
13797
13798 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13799 match(Set dst (SubF src1 src2));
13800
13801 ins_cost(INSN_COST * 5);
13802 format %{ "fsubs $dst, $src1, $src2" %}
13803
13804 ins_encode %{
13805 __ fsubs(as_FloatRegister($dst$$reg),
13806 as_FloatRegister($src1$$reg),
13807 as_FloatRegister($src2$$reg));
13808 %}
13809
13810 ins_pipe(fp_dop_reg_reg_s);
13811 %}
13812
13813 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13814 match(Set dst (SubD src1 src2));
13815
13816 ins_cost(INSN_COST * 5);
13817 format %{ "fsubd $dst, $src1, $src2" %}
13818
13819 ins_encode %{
13820 __ fsubd(as_FloatRegister($dst$$reg),
13821 as_FloatRegister($src1$$reg),
13822 as_FloatRegister($src2$$reg));
13823 %}
13824
13825 ins_pipe(fp_dop_reg_reg_d);
13826 %}
13827
13828 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13829 match(Set dst (MulHF src1 src2));
13830 format %{ "fmulh $dst, $src1, $src2" %}
13831 ins_encode %{
13832 __ fmulh($dst$$FloatRegister,
13833 $src1$$FloatRegister,
13834 $src2$$FloatRegister);
13835 %}
13836 ins_pipe(fp_dop_reg_reg_s);
13837 %}
13838
13839 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13840 match(Set dst (MulF src1 src2));
13841
13842 ins_cost(INSN_COST * 6);
13843 format %{ "fmuls $dst, $src1, $src2" %}
13844
13845 ins_encode %{
13846 __ fmuls(as_FloatRegister($dst$$reg),
13847 as_FloatRegister($src1$$reg),
13848 as_FloatRegister($src2$$reg));
13849 %}
13850
13851 ins_pipe(fp_dop_reg_reg_s);
13852 %}
13853
13854 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13855 match(Set dst (MulD src1 src2));
13856
13857 ins_cost(INSN_COST * 6);
13858 format %{ "fmuld $dst, $src1, $src2" %}
13859
13860 ins_encode %{
13861 __ fmuld(as_FloatRegister($dst$$reg),
13862 as_FloatRegister($src1$$reg),
13863 as_FloatRegister($src2$$reg));
13864 %}
13865
13866 ins_pipe(fp_dop_reg_reg_d);
13867 %}
13868
13869 // src1 * src2 + src3 (half-precision float)
13870 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13871 match(Set dst (FmaHF src3 (Binary src1 src2)));
13872 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13873 ins_encode %{
13874 assert(UseFMA, "Needs FMA instructions support.");
13875 __ fmaddh($dst$$FloatRegister,
13876 $src1$$FloatRegister,
13877 $src2$$FloatRegister,
13878 $src3$$FloatRegister);
13879 %}
13880 ins_pipe(pipe_class_default);
13881 %}
13882
13883 // src1 * src2 + src3
13884 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13885 match(Set dst (FmaF src3 (Binary src1 src2)));
13886
13887 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13888
13889 ins_encode %{
13890 assert(UseFMA, "Needs FMA instructions support.");
13891 __ fmadds(as_FloatRegister($dst$$reg),
13892 as_FloatRegister($src1$$reg),
13893 as_FloatRegister($src2$$reg),
13894 as_FloatRegister($src3$$reg));
13895 %}
13896
13897 ins_pipe(pipe_class_default);
13898 %}
13899
13900 // src1 * src2 + src3
13901 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13902 match(Set dst (FmaD src3 (Binary src1 src2)));
13903
13904 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13905
13906 ins_encode %{
13907 assert(UseFMA, "Needs FMA instructions support.");
13908 __ fmaddd(as_FloatRegister($dst$$reg),
13909 as_FloatRegister($src1$$reg),
13910 as_FloatRegister($src2$$reg),
13911 as_FloatRegister($src3$$reg));
13912 %}
13913
13914 ins_pipe(pipe_class_default);
13915 %}
13916
13917 // src1 * (-src2) + src3
13918 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13919 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13920 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13921
13922 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13923
13924 ins_encode %{
13925 assert(UseFMA, "Needs FMA instructions support.");
13926 __ fmsubs(as_FloatRegister($dst$$reg),
13927 as_FloatRegister($src1$$reg),
13928 as_FloatRegister($src2$$reg),
13929 as_FloatRegister($src3$$reg));
13930 %}
13931
13932 ins_pipe(pipe_class_default);
13933 %}
13934
13935 // src1 * (-src2) + src3
13936 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13937 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13938 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13939
13940 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13941
13942 ins_encode %{
13943 assert(UseFMA, "Needs FMA instructions support.");
13944 __ fmsubd(as_FloatRegister($dst$$reg),
13945 as_FloatRegister($src1$$reg),
13946 as_FloatRegister($src2$$reg),
13947 as_FloatRegister($src3$$reg));
13948 %}
13949
13950 ins_pipe(pipe_class_default);
13951 %}
13952
13953 // src1 * (-src2) - src3
13954 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13955 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13956 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13957
13958 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13959
13960 ins_encode %{
13961 assert(UseFMA, "Needs FMA instructions support.");
13962 __ fnmadds(as_FloatRegister($dst$$reg),
13963 as_FloatRegister($src1$$reg),
13964 as_FloatRegister($src2$$reg),
13965 as_FloatRegister($src3$$reg));
13966 %}
13967
13968 ins_pipe(pipe_class_default);
13969 %}
13970
13971 // src1 * (-src2) - src3
13972 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13973 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13974 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13975
13976 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13977
13978 ins_encode %{
13979 assert(UseFMA, "Needs FMA instructions support.");
13980 __ fnmaddd(as_FloatRegister($dst$$reg),
13981 as_FloatRegister($src1$$reg),
13982 as_FloatRegister($src2$$reg),
13983 as_FloatRegister($src3$$reg));
13984 %}
13985
13986 ins_pipe(pipe_class_default);
13987 %}
13988
13989 // src1 * src2 - src3
13990 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13991 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13992
13993 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13994
13995 ins_encode %{
13996 assert(UseFMA, "Needs FMA instructions support.");
13997 __ fnmsubs(as_FloatRegister($dst$$reg),
13998 as_FloatRegister($src1$$reg),
13999 as_FloatRegister($src2$$reg),
14000 as_FloatRegister($src3$$reg));
14001 %}
14002
14003 ins_pipe(pipe_class_default);
14004 %}
14005
14006 // src1 * src2 - src3
14007 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14008 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14009
14010 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14011
14012 ins_encode %{
14013 assert(UseFMA, "Needs FMA instructions support.");
14014 // n.b. insn name should be fnmsubd
14015 __ fnmsub(as_FloatRegister($dst$$reg),
14016 as_FloatRegister($src1$$reg),
14017 as_FloatRegister($src2$$reg),
14018 as_FloatRegister($src3$$reg));
14019 %}
14020
14021 ins_pipe(pipe_class_default);
14022 %}
14023
14024 // Math.max(HH)H (half-precision float)
14025 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14026 match(Set dst (MaxHF src1 src2));
14027 format %{ "fmaxh $dst, $src1, $src2" %}
14028 ins_encode %{
14029 __ fmaxh($dst$$FloatRegister,
14030 $src1$$FloatRegister,
14031 $src2$$FloatRegister);
14032 %}
14033 ins_pipe(fp_dop_reg_reg_s);
14034 %}
14035
14036 // Math.min(HH)H (half-precision float)
14037 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14038 match(Set dst (MinHF src1 src2));
14039 format %{ "fminh $dst, $src1, $src2" %}
14040 ins_encode %{
14041 __ fminh($dst$$FloatRegister,
14042 $src1$$FloatRegister,
14043 $src2$$FloatRegister);
14044 %}
14045 ins_pipe(fp_dop_reg_reg_s);
14046 %}
14047
14048 // Math.max(FF)F
14049 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14050 match(Set dst (MaxF src1 src2));
14051
14052 format %{ "fmaxs $dst, $src1, $src2" %}
14053 ins_encode %{
14054 __ fmaxs(as_FloatRegister($dst$$reg),
14055 as_FloatRegister($src1$$reg),
14056 as_FloatRegister($src2$$reg));
14057 %}
14058
14059 ins_pipe(fp_dop_reg_reg_s);
14060 %}
14061
14062 // Math.min(FF)F
14063 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14064 match(Set dst (MinF src1 src2));
14065
14066 format %{ "fmins $dst, $src1, $src2" %}
14067 ins_encode %{
14068 __ fmins(as_FloatRegister($dst$$reg),
14069 as_FloatRegister($src1$$reg),
14070 as_FloatRegister($src2$$reg));
14071 %}
14072
14073 ins_pipe(fp_dop_reg_reg_s);
14074 %}
14075
14076 // Math.max(DD)D
14077 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14078 match(Set dst (MaxD src1 src2));
14079
14080 format %{ "fmaxd $dst, $src1, $src2" %}
14081 ins_encode %{
14082 __ fmaxd(as_FloatRegister($dst$$reg),
14083 as_FloatRegister($src1$$reg),
14084 as_FloatRegister($src2$$reg));
14085 %}
14086
14087 ins_pipe(fp_dop_reg_reg_d);
14088 %}
14089
14090 // Math.min(DD)D
14091 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14092 match(Set dst (MinD src1 src2));
14093
14094 format %{ "fmind $dst, $src1, $src2" %}
14095 ins_encode %{
14096 __ fmind(as_FloatRegister($dst$$reg),
14097 as_FloatRegister($src1$$reg),
14098 as_FloatRegister($src2$$reg));
14099 %}
14100
14101 ins_pipe(fp_dop_reg_reg_d);
14102 %}
14103
14104 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14105 match(Set dst (DivHF src1 src2));
14106 format %{ "fdivh $dst, $src1, $src2" %}
14107 ins_encode %{
14108 __ fdivh($dst$$FloatRegister,
14109 $src1$$FloatRegister,
14110 $src2$$FloatRegister);
14111 %}
14112 ins_pipe(fp_div_s);
14113 %}
14114
14115 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14116 match(Set dst (DivF src1 src2));
14117
14118 ins_cost(INSN_COST * 18);
14119 format %{ "fdivs $dst, $src1, $src2" %}
14120
14121 ins_encode %{
14122 __ fdivs(as_FloatRegister($dst$$reg),
14123 as_FloatRegister($src1$$reg),
14124 as_FloatRegister($src2$$reg));
14125 %}
14126
14127 ins_pipe(fp_div_s);
14128 %}
14129
14130 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14131 match(Set dst (DivD src1 src2));
14132
14133 ins_cost(INSN_COST * 32);
14134 format %{ "fdivd $dst, $src1, $src2" %}
14135
14136 ins_encode %{
14137 __ fdivd(as_FloatRegister($dst$$reg),
14138 as_FloatRegister($src1$$reg),
14139 as_FloatRegister($src2$$reg));
14140 %}
14141
14142 ins_pipe(fp_div_d);
14143 %}
14144
14145 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14146 match(Set dst (NegF src));
14147
14148 ins_cost(INSN_COST * 3);
14149 format %{ "fneg $dst, $src" %}
14150
14151 ins_encode %{
14152 __ fnegs(as_FloatRegister($dst$$reg),
14153 as_FloatRegister($src$$reg));
14154 %}
14155
14156 ins_pipe(fp_uop_s);
14157 %}
14158
14159 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14160 match(Set dst (NegD src));
14161
14162 ins_cost(INSN_COST * 3);
14163 format %{ "fnegd $dst, $src" %}
14164
14165 ins_encode %{
14166 __ fnegd(as_FloatRegister($dst$$reg),
14167 as_FloatRegister($src$$reg));
14168 %}
14169
14170 ins_pipe(fp_uop_d);
14171 %}
14172
14173 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14174 %{
14175 match(Set dst (AbsI src));
14176
14177 effect(KILL cr);
14178 ins_cost(INSN_COST * 2);
14179 format %{ "cmpw $src, zr\n\t"
14180 "cnegw $dst, $src, Assembler::LT\t# int abs"
14181 %}
14182
14183 ins_encode %{
14184 __ cmpw(as_Register($src$$reg), zr);
14185 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14186 %}
14187 ins_pipe(pipe_class_default);
14188 %}
14189
14190 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14191 %{
14192 match(Set dst (AbsL src));
14193
14194 effect(KILL cr);
14195 ins_cost(INSN_COST * 2);
14196 format %{ "cmp $src, zr\n\t"
14197 "cneg $dst, $src, Assembler::LT\t# long abs"
14198 %}
14199
14200 ins_encode %{
14201 __ cmp(as_Register($src$$reg), zr);
14202 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14203 %}
14204 ins_pipe(pipe_class_default);
14205 %}
14206
14207 instruct absF_reg(vRegF dst, vRegF src) %{
14208 match(Set dst (AbsF src));
14209
14210 ins_cost(INSN_COST * 3);
14211 format %{ "fabss $dst, $src" %}
14212 ins_encode %{
14213 __ fabss(as_FloatRegister($dst$$reg),
14214 as_FloatRegister($src$$reg));
14215 %}
14216
14217 ins_pipe(fp_uop_s);
14218 %}
14219
14220 instruct absD_reg(vRegD dst, vRegD src) %{
14221 match(Set dst (AbsD src));
14222
14223 ins_cost(INSN_COST * 3);
14224 format %{ "fabsd $dst, $src" %}
14225 ins_encode %{
14226 __ fabsd(as_FloatRegister($dst$$reg),
14227 as_FloatRegister($src$$reg));
14228 %}
14229
14230 ins_pipe(fp_uop_d);
14231 %}
14232
14233 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14234 match(Set dst (AbsF (SubF src1 src2)));
14235
14236 ins_cost(INSN_COST * 3);
14237 format %{ "fabds $dst, $src1, $src2" %}
14238 ins_encode %{
14239 __ fabds(as_FloatRegister($dst$$reg),
14240 as_FloatRegister($src1$$reg),
14241 as_FloatRegister($src2$$reg));
14242 %}
14243
14244 ins_pipe(fp_uop_s);
14245 %}
14246
14247 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14248 match(Set dst (AbsD (SubD src1 src2)));
14249
14250 ins_cost(INSN_COST * 3);
14251 format %{ "fabdd $dst, $src1, $src2" %}
14252 ins_encode %{
14253 __ fabdd(as_FloatRegister($dst$$reg),
14254 as_FloatRegister($src1$$reg),
14255 as_FloatRegister($src2$$reg));
14256 %}
14257
14258 ins_pipe(fp_uop_d);
14259 %}
14260
14261 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14262 match(Set dst (SqrtD src));
14263
14264 ins_cost(INSN_COST * 50);
14265 format %{ "fsqrtd $dst, $src" %}
14266 ins_encode %{
14267 __ fsqrtd(as_FloatRegister($dst$$reg),
14268 as_FloatRegister($src$$reg));
14269 %}
14270
14271 ins_pipe(fp_div_s);
14272 %}
14273
14274 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14275 match(Set dst (SqrtF src));
14276
14277 ins_cost(INSN_COST * 50);
14278 format %{ "fsqrts $dst, $src" %}
14279 ins_encode %{
14280 __ fsqrts(as_FloatRegister($dst$$reg),
14281 as_FloatRegister($src$$reg));
14282 %}
14283
14284 ins_pipe(fp_div_d);
14285 %}
14286
14287 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14288 match(Set dst (SqrtHF src));
14289 format %{ "fsqrth $dst, $src" %}
14290 ins_encode %{
14291 __ fsqrth($dst$$FloatRegister,
14292 $src$$FloatRegister);
14293 %}
14294 ins_pipe(fp_div_s);
14295 %}
14296
14297 // Math.rint, floor, ceil
14298 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14299 match(Set dst (RoundDoubleMode src rmode));
14300 format %{ "frint $dst, $src, $rmode" %}
14301 ins_encode %{
14302 switch ($rmode$$constant) {
14303 case RoundDoubleModeNode::rmode_rint:
14304 __ frintnd(as_FloatRegister($dst$$reg),
14305 as_FloatRegister($src$$reg));
14306 break;
14307 case RoundDoubleModeNode::rmode_floor:
14308 __ frintmd(as_FloatRegister($dst$$reg),
14309 as_FloatRegister($src$$reg));
14310 break;
14311 case RoundDoubleModeNode::rmode_ceil:
14312 __ frintpd(as_FloatRegister($dst$$reg),
14313 as_FloatRegister($src$$reg));
14314 break;
14315 }
14316 %}
14317 ins_pipe(fp_uop_d);
14318 %}
14319
14320 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14321 match(Set dst (CopySignD src1 (Binary src2 zero)));
14322 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14323 format %{ "CopySignD $dst $src1 $src2" %}
14324 ins_encode %{
14325 FloatRegister dst = as_FloatRegister($dst$$reg),
14326 src1 = as_FloatRegister($src1$$reg),
14327 src2 = as_FloatRegister($src2$$reg),
14328 zero = as_FloatRegister($zero$$reg);
14329 __ fnegd(dst, zero);
14330 __ bsl(dst, __ T8B, src2, src1);
14331 %}
14332 ins_pipe(fp_uop_d);
14333 %}
14334
14335 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14336 match(Set dst (CopySignF src1 src2));
14337 effect(TEMP_DEF dst, USE src1, USE src2);
14338 format %{ "CopySignF $dst $src1 $src2" %}
14339 ins_encode %{
14340 FloatRegister dst = as_FloatRegister($dst$$reg),
14341 src1 = as_FloatRegister($src1$$reg),
14342 src2 = as_FloatRegister($src2$$reg);
14343 __ movi(dst, __ T2S, 0x80, 24);
14344 __ bsl(dst, __ T8B, src2, src1);
14345 %}
14346 ins_pipe(fp_uop_d);
14347 %}
14348
14349 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14350 match(Set dst (SignumD src (Binary zero one)));
14351 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14352 format %{ "signumD $dst, $src" %}
14353 ins_encode %{
14354 FloatRegister src = as_FloatRegister($src$$reg),
14355 dst = as_FloatRegister($dst$$reg),
14356 zero = as_FloatRegister($zero$$reg),
14357 one = as_FloatRegister($one$$reg);
14358 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14359 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14360 // Bit selection instruction gets bit from "one" for each enabled bit in
14361 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14362 // NaN the whole "src" will be copied because "dst" is zero. For all other
14363 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14364 // from "src", and all other bits are copied from 1.0.
14365 __ bsl(dst, __ T8B, one, src);
14366 %}
14367 ins_pipe(fp_uop_d);
14368 %}
14369
14370 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14371 match(Set dst (SignumF src (Binary zero one)));
14372 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14373 format %{ "signumF $dst, $src" %}
14374 ins_encode %{
14375 FloatRegister src = as_FloatRegister($src$$reg),
14376 dst = as_FloatRegister($dst$$reg),
14377 zero = as_FloatRegister($zero$$reg),
14378 one = as_FloatRegister($one$$reg);
14379 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14380 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14381 // Bit selection instruction gets bit from "one" for each enabled bit in
14382 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14383 // NaN the whole "src" will be copied because "dst" is zero. For all other
14384 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14385 // from "src", and all other bits are copied from 1.0.
14386 __ bsl(dst, __ T8B, one, src);
14387 %}
14388 ins_pipe(fp_uop_d);
14389 %}
14390
14391 instruct onspinwait() %{
14392 match(OnSpinWait);
14393 ins_cost(INSN_COST);
14394
14395 format %{ "onspinwait" %}
14396
14397 ins_encode %{
14398 __ spin_wait();
14399 %}
14400 ins_pipe(pipe_class_empty);
14401 %}
14402
14403 // ============================================================================
14404 // Logical Instructions
14405
14406 // Integer Logical Instructions
14407
14408 // And Instructions
14409
14410
14411 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14412 match(Set dst (AndI src1 src2));
14413
14414 format %{ "andw $dst, $src1, $src2\t# int" %}
14415
14416 ins_cost(INSN_COST);
14417 ins_encode %{
14418 __ andw(as_Register($dst$$reg),
14419 as_Register($src1$$reg),
14420 as_Register($src2$$reg));
14421 %}
14422
14423 ins_pipe(ialu_reg_reg);
14424 %}
14425
14426 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14427 match(Set dst (AndI src1 src2));
14428
14429 format %{ "andsw $dst, $src1, $src2\t# int" %}
14430
14431 ins_cost(INSN_COST);
14432 ins_encode %{
14433 __ andw(as_Register($dst$$reg),
14434 as_Register($src1$$reg),
14435 (uint64_t)($src2$$constant));
14436 %}
14437
14438 ins_pipe(ialu_reg_imm);
14439 %}
14440
14441 // Or Instructions
14442
14443 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14444 match(Set dst (OrI src1 src2));
14445
14446 format %{ "orrw $dst, $src1, $src2\t# int" %}
14447
14448 ins_cost(INSN_COST);
14449 ins_encode %{
14450 __ orrw(as_Register($dst$$reg),
14451 as_Register($src1$$reg),
14452 as_Register($src2$$reg));
14453 %}
14454
14455 ins_pipe(ialu_reg_reg);
14456 %}
14457
14458 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14459 match(Set dst (OrI src1 src2));
14460
14461 format %{ "orrw $dst, $src1, $src2\t# int" %}
14462
14463 ins_cost(INSN_COST);
14464 ins_encode %{
14465 __ orrw(as_Register($dst$$reg),
14466 as_Register($src1$$reg),
14467 (uint64_t)($src2$$constant));
14468 %}
14469
14470 ins_pipe(ialu_reg_imm);
14471 %}
14472
14473 // Xor Instructions
14474
14475 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14476 match(Set dst (XorI src1 src2));
14477
14478 format %{ "eorw $dst, $src1, $src2\t# int" %}
14479
14480 ins_cost(INSN_COST);
14481 ins_encode %{
14482 __ eorw(as_Register($dst$$reg),
14483 as_Register($src1$$reg),
14484 as_Register($src2$$reg));
14485 %}
14486
14487 ins_pipe(ialu_reg_reg);
14488 %}
14489
14490 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14491 match(Set dst (XorI src1 src2));
14492
14493 format %{ "eorw $dst, $src1, $src2\t# int" %}
14494
14495 ins_cost(INSN_COST);
14496 ins_encode %{
14497 __ eorw(as_Register($dst$$reg),
14498 as_Register($src1$$reg),
14499 (uint64_t)($src2$$constant));
14500 %}
14501
14502 ins_pipe(ialu_reg_imm);
14503 %}
14504
14505 // Long Logical Instructions
14506 // TODO
14507
14508 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14509 match(Set dst (AndL src1 src2));
14510
14511 format %{ "and $dst, $src1, $src2\t# int" %}
14512
14513 ins_cost(INSN_COST);
14514 ins_encode %{
14515 __ andr(as_Register($dst$$reg),
14516 as_Register($src1$$reg),
14517 as_Register($src2$$reg));
14518 %}
14519
14520 ins_pipe(ialu_reg_reg);
14521 %}
14522
14523 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14524 match(Set dst (AndL src1 src2));
14525
14526 format %{ "and $dst, $src1, $src2\t# int" %}
14527
14528 ins_cost(INSN_COST);
14529 ins_encode %{
14530 __ andr(as_Register($dst$$reg),
14531 as_Register($src1$$reg),
14532 (uint64_t)($src2$$constant));
14533 %}
14534
14535 ins_pipe(ialu_reg_imm);
14536 %}
14537
14538 // Or Instructions
14539
14540 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14541 match(Set dst (OrL src1 src2));
14542
14543 format %{ "orr $dst, $src1, $src2\t# int" %}
14544
14545 ins_cost(INSN_COST);
14546 ins_encode %{
14547 __ orr(as_Register($dst$$reg),
14548 as_Register($src1$$reg),
14549 as_Register($src2$$reg));
14550 %}
14551
14552 ins_pipe(ialu_reg_reg);
14553 %}
14554
14555 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14556 match(Set dst (OrL src1 src2));
14557
14558 format %{ "orr $dst, $src1, $src2\t# int" %}
14559
14560 ins_cost(INSN_COST);
14561 ins_encode %{
14562 __ orr(as_Register($dst$$reg),
14563 as_Register($src1$$reg),
14564 (uint64_t)($src2$$constant));
14565 %}
14566
14567 ins_pipe(ialu_reg_imm);
14568 %}
14569
14570 // Xor Instructions
14571
14572 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14573 match(Set dst (XorL src1 src2));
14574
14575 format %{ "eor $dst, $src1, $src2\t# int" %}
14576
14577 ins_cost(INSN_COST);
14578 ins_encode %{
14579 __ eor(as_Register($dst$$reg),
14580 as_Register($src1$$reg),
14581 as_Register($src2$$reg));
14582 %}
14583
14584 ins_pipe(ialu_reg_reg);
14585 %}
14586
14587 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14588 match(Set dst (XorL src1 src2));
14589
14590 ins_cost(INSN_COST);
14591 format %{ "eor $dst, $src1, $src2\t# int" %}
14592
14593 ins_encode %{
14594 __ eor(as_Register($dst$$reg),
14595 as_Register($src1$$reg),
14596 (uint64_t)($src2$$constant));
14597 %}
14598
14599 ins_pipe(ialu_reg_imm);
14600 %}
14601
14602 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14603 %{
14604 match(Set dst (ConvI2L src));
14605
14606 ins_cost(INSN_COST);
14607 format %{ "sxtw $dst, $src\t# i2l" %}
14608 ins_encode %{
14609 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14610 %}
14611 ins_pipe(ialu_reg_shift);
14612 %}
14613
14614 // this pattern occurs in bigmath arithmetic
14615 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14616 %{
14617 match(Set dst (AndL (ConvI2L src) mask));
14618
14619 ins_cost(INSN_COST);
14620 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14621 ins_encode %{
14622 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14623 %}
14624
14625 ins_pipe(ialu_reg_shift);
14626 %}
14627
14628 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14629 match(Set dst (ConvL2I src));
14630
14631 ins_cost(INSN_COST);
14632 format %{ "movw $dst, $src \t// l2i" %}
14633
14634 ins_encode %{
14635 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14636 %}
14637
14638 ins_pipe(ialu_reg);
14639 %}
14640
14641 instruct convD2F_reg(vRegF dst, vRegD src) %{
14642 match(Set dst (ConvD2F src));
14643
14644 ins_cost(INSN_COST * 5);
14645 format %{ "fcvtd $dst, $src \t// d2f" %}
14646
14647 ins_encode %{
14648 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14649 %}
14650
14651 ins_pipe(fp_d2f);
14652 %}
14653
14654 instruct convF2D_reg(vRegD dst, vRegF src) %{
14655 match(Set dst (ConvF2D src));
14656
14657 ins_cost(INSN_COST * 5);
14658 format %{ "fcvts $dst, $src \t// f2d" %}
14659
14660 ins_encode %{
14661 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14662 %}
14663
14664 ins_pipe(fp_f2d);
14665 %}
14666
14667 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14668 match(Set dst (ConvF2I src));
14669
14670 ins_cost(INSN_COST * 5);
14671 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14672
14673 ins_encode %{
14674 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14675 %}
14676
14677 ins_pipe(fp_f2i);
14678 %}
14679
14680 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14681 match(Set dst (ConvF2L src));
14682
14683 ins_cost(INSN_COST * 5);
14684 format %{ "fcvtzs $dst, $src \t// f2l" %}
14685
14686 ins_encode %{
14687 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14688 %}
14689
14690 ins_pipe(fp_f2l);
14691 %}
14692
14693 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14694 match(Set dst (ConvF2HF src));
14695 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14696 "smov $dst, $tmp\t# move result from $tmp to $dst"
14697 %}
14698 effect(TEMP tmp);
14699 ins_encode %{
14700 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14701 %}
14702 ins_pipe(pipe_slow);
14703 %}
14704
14705 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14706 match(Set dst (ConvHF2F src));
14707 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14708 "fcvt $dst, $tmp\t# convert half to single precision"
14709 %}
14710 effect(TEMP tmp);
14711 ins_encode %{
14712 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14713 %}
14714 ins_pipe(pipe_slow);
14715 %}
14716
14717 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14718 match(Set dst (ConvI2F src));
14719
14720 ins_cost(INSN_COST * 5);
14721 format %{ "scvtfws $dst, $src \t// i2f" %}
14722
14723 ins_encode %{
14724 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14725 %}
14726
14727 ins_pipe(fp_i2f);
14728 %}
14729
14730 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14731 match(Set dst (ConvL2F src));
14732
14733 ins_cost(INSN_COST * 5);
14734 format %{ "scvtfs $dst, $src \t// l2f" %}
14735
14736 ins_encode %{
14737 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14738 %}
14739
14740 ins_pipe(fp_l2f);
14741 %}
14742
14743 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14744 match(Set dst (ConvD2I src));
14745
14746 ins_cost(INSN_COST * 5);
14747 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14748
14749 ins_encode %{
14750 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14751 %}
14752
14753 ins_pipe(fp_d2i);
14754 %}
14755
14756 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14757 match(Set dst (ConvD2L src));
14758
14759 ins_cost(INSN_COST * 5);
14760 format %{ "fcvtzd $dst, $src \t// d2l" %}
14761
14762 ins_encode %{
14763 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14764 %}
14765
14766 ins_pipe(fp_d2l);
14767 %}
14768
14769 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14770 match(Set dst (ConvI2D src));
14771
14772 ins_cost(INSN_COST * 5);
14773 format %{ "scvtfwd $dst, $src \t// i2d" %}
14774
14775 ins_encode %{
14776 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14777 %}
14778
14779 ins_pipe(fp_i2d);
14780 %}
14781
14782 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14783 match(Set dst (ConvL2D src));
14784
14785 ins_cost(INSN_COST * 5);
14786 format %{ "scvtfd $dst, $src \t// l2d" %}
14787
14788 ins_encode %{
14789 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14790 %}
14791
14792 ins_pipe(fp_l2d);
14793 %}
14794
14795 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14796 %{
14797 match(Set dst (RoundD src));
14798 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14799 format %{ "java_round_double $dst,$src"%}
14800 ins_encode %{
14801 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14802 as_FloatRegister($ftmp$$reg));
14803 %}
14804 ins_pipe(pipe_slow);
14805 %}
14806
14807 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14808 %{
14809 match(Set dst (RoundF src));
14810 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14811 format %{ "java_round_float $dst,$src"%}
14812 ins_encode %{
14813 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14814 as_FloatRegister($ftmp$$reg));
14815 %}
14816 ins_pipe(pipe_slow);
14817 %}
14818
14819 // stack <-> reg and reg <-> reg shuffles with no conversion
14820
14821 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14822
14823 match(Set dst (MoveF2I src));
14824
14825 effect(DEF dst, USE src);
14826
14827 ins_cost(4 * INSN_COST);
14828
14829 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14830
14831 ins_encode %{
14832 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14833 %}
14834
14835 ins_pipe(iload_reg_reg);
14836
14837 %}
14838
14839 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14840
14841 match(Set dst (MoveI2F src));
14842
14843 effect(DEF dst, USE src);
14844
14845 ins_cost(4 * INSN_COST);
14846
14847 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14848
14849 ins_encode %{
14850 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14851 %}
14852
14853 ins_pipe(pipe_class_memory);
14854
14855 %}
14856
14857 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14858
14859 match(Set dst (MoveD2L src));
14860
14861 effect(DEF dst, USE src);
14862
14863 ins_cost(4 * INSN_COST);
14864
14865 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14866
14867 ins_encode %{
14868 __ ldr($dst$$Register, Address(sp, $src$$disp));
14869 %}
14870
14871 ins_pipe(iload_reg_reg);
14872
14873 %}
14874
14875 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14876
14877 match(Set dst (MoveL2D src));
14878
14879 effect(DEF dst, USE src);
14880
14881 ins_cost(4 * INSN_COST);
14882
14883 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14884
14885 ins_encode %{
14886 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14887 %}
14888
14889 ins_pipe(pipe_class_memory);
14890
14891 %}
14892
14893 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14894
14895 match(Set dst (MoveF2I src));
14896
14897 effect(DEF dst, USE src);
14898
14899 ins_cost(INSN_COST);
14900
14901 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14902
14903 ins_encode %{
14904 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14905 %}
14906
14907 ins_pipe(pipe_class_memory);
14908
14909 %}
14910
14911 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14912
14913 match(Set dst (MoveI2F src));
14914
14915 effect(DEF dst, USE src);
14916
14917 ins_cost(INSN_COST);
14918
14919 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14920
14921 ins_encode %{
14922 __ strw($src$$Register, Address(sp, $dst$$disp));
14923 %}
14924
14925 ins_pipe(istore_reg_reg);
14926
14927 %}
14928
14929 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14930
14931 match(Set dst (MoveD2L src));
14932
14933 effect(DEF dst, USE src);
14934
14935 ins_cost(INSN_COST);
14936
14937 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14938
14939 ins_encode %{
14940 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14941 %}
14942
14943 ins_pipe(pipe_class_memory);
14944
14945 %}
14946
14947 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14948
14949 match(Set dst (MoveL2D src));
14950
14951 effect(DEF dst, USE src);
14952
14953 ins_cost(INSN_COST);
14954
14955 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14956
14957 ins_encode %{
14958 __ str($src$$Register, Address(sp, $dst$$disp));
14959 %}
14960
14961 ins_pipe(istore_reg_reg);
14962
14963 %}
14964
14965 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14966
14967 match(Set dst (MoveF2I src));
14968
14969 effect(DEF dst, USE src);
14970
14971 ins_cost(INSN_COST);
14972
14973 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14974
14975 ins_encode %{
14976 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14977 %}
14978
14979 ins_pipe(fp_f2i);
14980
14981 %}
14982
14983 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14984
14985 match(Set dst (MoveI2F src));
14986
14987 effect(DEF dst, USE src);
14988
14989 ins_cost(INSN_COST);
14990
14991 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14992
14993 ins_encode %{
14994 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14995 %}
14996
14997 ins_pipe(fp_i2f);
14998
14999 %}
15000
15001 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15002
15003 match(Set dst (MoveD2L src));
15004
15005 effect(DEF dst, USE src);
15006
15007 ins_cost(INSN_COST);
15008
15009 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15010
15011 ins_encode %{
15012 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15013 %}
15014
15015 ins_pipe(fp_d2l);
15016
15017 %}
15018
15019 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15020
15021 match(Set dst (MoveL2D src));
15022
15023 effect(DEF dst, USE src);
15024
15025 ins_cost(INSN_COST);
15026
15027 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15028
15029 ins_encode %{
15030 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15031 %}
15032
15033 ins_pipe(fp_l2d);
15034
15035 %}
15036
15037 // ============================================================================
15038 // clearing of an array
15039
15040 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
15041 %{
15042 match(Set dummy (ClearArray (Binary cnt base) zero));
15043 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15044
15045 ins_cost(4 * INSN_COST);
15046 format %{ "ClearArray $cnt, $base" %}
15047
15048 ins_encode %{
15049 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15050 if (tpc == nullptr) {
15051 ciEnv::current()->record_failure("CodeCache is full");
15052 return;
15053 }
15054 %}
15055
15056 ins_pipe(pipe_class_memory);
15057 %}
15058
15059 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
15060 %{
15061 predicate(((ClearArrayNode*)n)->word_copy_only());
15062 match(Set dummy (ClearArray (Binary cnt base) val));
15063 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15064
15065 ins_cost(4 * INSN_COST);
15066 format %{ "ClearArray $cnt, $base, $val" %}
15067
15068 ins_encode %{
15069 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
15070 %}
15071
15072 ins_pipe(pipe_class_memory);
15073 %}
15074
15075 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15076 %{
15077 predicate((uint64_t)n->in(2)->get_long()
15078 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
15079 && !((ClearArrayNode*)n)->word_copy_only());
15080 match(Set dummy (ClearArray cnt base));
15081 effect(TEMP temp, USE_KILL base, KILL cr);
15082
15083 ins_cost(4 * INSN_COST);
15084 format %{ "ClearArray $cnt, $base" %}
15085
15086 ins_encode %{
15087 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15088 if (tpc == nullptr) {
15089 ciEnv::current()->record_failure("CodeCache is full");
15090 return;
15091 }
15092 %}
15093
15094 ins_pipe(pipe_class_memory);
15095 %}
15096
15097 // ============================================================================
15098 // Overflow Math Instructions
15099
15100 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15101 %{
15102 match(Set cr (OverflowAddI op1 op2));
15103
15104 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15105 ins_cost(INSN_COST);
15106 ins_encode %{
15107 __ cmnw($op1$$Register, $op2$$Register);
15108 %}
15109
15110 ins_pipe(icmp_reg_reg);
15111 %}
15112
15113 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15114 %{
15115 match(Set cr (OverflowAddI op1 op2));
15116
15117 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15118 ins_cost(INSN_COST);
15119 ins_encode %{
15120 __ cmnw($op1$$Register, $op2$$constant);
15121 %}
15122
15123 ins_pipe(icmp_reg_imm);
15124 %}
15125
15126 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15127 %{
15128 match(Set cr (OverflowAddL op1 op2));
15129
15130 format %{ "cmn $op1, $op2\t# overflow check long" %}
15131 ins_cost(INSN_COST);
15132 ins_encode %{
15133 __ cmn($op1$$Register, $op2$$Register);
15134 %}
15135
15136 ins_pipe(icmp_reg_reg);
15137 %}
15138
15139 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15140 %{
15141 match(Set cr (OverflowAddL op1 op2));
15142
15143 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15144 ins_cost(INSN_COST);
15145 ins_encode %{
15146 __ adds(zr, $op1$$Register, $op2$$constant);
15147 %}
15148
15149 ins_pipe(icmp_reg_imm);
15150 %}
15151
15152 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15153 %{
15154 match(Set cr (OverflowSubI op1 op2));
15155
15156 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15157 ins_cost(INSN_COST);
15158 ins_encode %{
15159 __ cmpw($op1$$Register, $op2$$Register);
15160 %}
15161
15162 ins_pipe(icmp_reg_reg);
15163 %}
15164
15165 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15166 %{
15167 match(Set cr (OverflowSubI op1 op2));
15168
15169 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15170 ins_cost(INSN_COST);
15171 ins_encode %{
15172 __ cmpw($op1$$Register, $op2$$constant);
15173 %}
15174
15175 ins_pipe(icmp_reg_imm);
15176 %}
15177
15178 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15179 %{
15180 match(Set cr (OverflowSubL op1 op2));
15181
15182 format %{ "cmp $op1, $op2\t# overflow check long" %}
15183 ins_cost(INSN_COST);
15184 ins_encode %{
15185 __ cmp($op1$$Register, $op2$$Register);
15186 %}
15187
15188 ins_pipe(icmp_reg_reg);
15189 %}
15190
15191 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15192 %{
15193 match(Set cr (OverflowSubL op1 op2));
15194
15195 format %{ "cmp $op1, $op2\t# overflow check long" %}
15196 ins_cost(INSN_COST);
15197 ins_encode %{
15198 __ subs(zr, $op1$$Register, $op2$$constant);
15199 %}
15200
15201 ins_pipe(icmp_reg_imm);
15202 %}
15203
15204 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15205 %{
15206 match(Set cr (OverflowSubI zero op1));
15207
15208 format %{ "cmpw zr, $op1\t# overflow check int" %}
15209 ins_cost(INSN_COST);
15210 ins_encode %{
15211 __ cmpw(zr, $op1$$Register);
15212 %}
15213
15214 ins_pipe(icmp_reg_imm);
15215 %}
15216
15217 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15218 %{
15219 match(Set cr (OverflowSubL zero op1));
15220
15221 format %{ "cmp zr, $op1\t# overflow check long" %}
15222 ins_cost(INSN_COST);
15223 ins_encode %{
15224 __ cmp(zr, $op1$$Register);
15225 %}
15226
15227 ins_pipe(icmp_reg_imm);
15228 %}
15229
15230 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15231 %{
15232 match(Set cr (OverflowMulI op1 op2));
15233
15234 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15235 "cmp rscratch1, rscratch1, sxtw\n\t"
15236 "movw rscratch1, #0x80000000\n\t"
15237 "cselw rscratch1, rscratch1, zr, NE\n\t"
15238 "cmpw rscratch1, #1" %}
15239 ins_cost(5 * INSN_COST);
15240 ins_encode %{
15241 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15242 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15243 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15244 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15245 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15246 %}
15247
15248 ins_pipe(pipe_slow);
15249 %}
15250
15251 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15252 %{
15253 match(If cmp (OverflowMulI op1 op2));
15254 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15255 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15256 effect(USE labl, KILL cr);
15257
15258 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15259 "cmp rscratch1, rscratch1, sxtw\n\t"
15260 "b$cmp $labl" %}
15261 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15262 ins_encode %{
15263 Label* L = $labl$$label;
15264 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15265 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15266 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15267 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15268 %}
15269
15270 ins_pipe(pipe_serial);
15271 %}
15272
15273 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15274 %{
15275 match(Set cr (OverflowMulL op1 op2));
15276
15277 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15278 "smulh rscratch2, $op1, $op2\n\t"
15279 "cmp rscratch2, rscratch1, ASR #63\n\t"
15280 "movw rscratch1, #0x80000000\n\t"
15281 "cselw rscratch1, rscratch1, zr, NE\n\t"
15282 "cmpw rscratch1, #1" %}
15283 ins_cost(6 * INSN_COST);
15284 ins_encode %{
15285 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15286 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15287 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15288 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15289 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15290 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15291 %}
15292
15293 ins_pipe(pipe_slow);
15294 %}
15295
15296 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15297 %{
15298 match(If cmp (OverflowMulL op1 op2));
15299 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15300 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15301 effect(USE labl, KILL cr);
15302
15303 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15304 "smulh rscratch2, $op1, $op2\n\t"
15305 "cmp rscratch2, rscratch1, ASR #63\n\t"
15306 "b$cmp $labl" %}
15307 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15308 ins_encode %{
15309 Label* L = $labl$$label;
15310 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15311 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15312 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15313 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15314 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15315 %}
15316
15317 ins_pipe(pipe_serial);
15318 %}
15319
15320 // ============================================================================
15321 // Compare Instructions
15322
15323 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15324 %{
15325 match(Set cr (CmpI op1 op2));
15326
15327 effect(DEF cr, USE op1, USE op2);
15328
15329 ins_cost(INSN_COST);
15330 format %{ "cmpw $op1, $op2" %}
15331
15332 ins_encode(aarch64_enc_cmpw(op1, op2));
15333
15334 ins_pipe(icmp_reg_reg);
15335 %}
15336
15337 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15338 %{
15339 match(Set cr (CmpI op1 zero));
15340
15341 effect(DEF cr, USE op1);
15342
15343 ins_cost(INSN_COST);
15344 format %{ "cmpw $op1, 0" %}
15345
15346 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15347
15348 ins_pipe(icmp_reg_imm);
15349 %}
15350
15351 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15352 %{
15353 match(Set cr (CmpI op1 op2));
15354
15355 effect(DEF cr, USE op1);
15356
15357 ins_cost(INSN_COST);
15358 format %{ "cmpw $op1, $op2" %}
15359
15360 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15361
15362 ins_pipe(icmp_reg_imm);
15363 %}
15364
15365 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15366 %{
15367 match(Set cr (CmpI op1 op2));
15368
15369 effect(DEF cr, USE op1);
15370
15371 ins_cost(INSN_COST * 2);
15372 format %{ "cmpw $op1, $op2" %}
15373
15374 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15375
15376 ins_pipe(icmp_reg_imm);
15377 %}
15378
15379 // Unsigned compare Instructions; really, same as signed compare
15380 // except it should only be used to feed an If or a CMovI which takes a
15381 // cmpOpU.
15382
15383 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15384 %{
15385 match(Set cr (CmpU op1 op2));
15386
15387 effect(DEF cr, USE op1, USE op2);
15388
15389 ins_cost(INSN_COST);
15390 format %{ "cmpw $op1, $op2\t# unsigned" %}
15391
15392 ins_encode(aarch64_enc_cmpw(op1, op2));
15393
15394 ins_pipe(icmp_reg_reg);
15395 %}
15396
15397 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15398 %{
15399 match(Set cr (CmpU op1 zero));
15400
15401 effect(DEF cr, USE op1);
15402
15403 ins_cost(INSN_COST);
15404 format %{ "cmpw $op1, #0\t# unsigned" %}
15405
15406 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15407
15408 ins_pipe(icmp_reg_imm);
15409 %}
15410
15411 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15412 %{
15413 match(Set cr (CmpU op1 op2));
15414
15415 effect(DEF cr, USE op1);
15416
15417 ins_cost(INSN_COST);
15418 format %{ "cmpw $op1, $op2\t# unsigned" %}
15419
15420 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15421
15422 ins_pipe(icmp_reg_imm);
15423 %}
15424
15425 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15426 %{
15427 match(Set cr (CmpU op1 op2));
15428
15429 effect(DEF cr, USE op1);
15430
15431 ins_cost(INSN_COST * 2);
15432 format %{ "cmpw $op1, $op2\t# unsigned" %}
15433
15434 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15435
15436 ins_pipe(icmp_reg_imm);
15437 %}
15438
15439 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15440 %{
15441 match(Set cr (CmpL op1 op2));
15442
15443 effect(DEF cr, USE op1, USE op2);
15444
15445 ins_cost(INSN_COST);
15446 format %{ "cmp $op1, $op2" %}
15447
15448 ins_encode(aarch64_enc_cmp(op1, op2));
15449
15450 ins_pipe(icmp_reg_reg);
15451 %}
15452
15453 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15454 %{
15455 match(Set cr (CmpL op1 zero));
15456
15457 effect(DEF cr, USE op1);
15458
15459 ins_cost(INSN_COST);
15460 format %{ "tst $op1" %}
15461
15462 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15463
15464 ins_pipe(icmp_reg_imm);
15465 %}
15466
15467 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15468 %{
15469 match(Set cr (CmpL op1 op2));
15470
15471 effect(DEF cr, USE op1);
15472
15473 ins_cost(INSN_COST);
15474 format %{ "cmp $op1, $op2" %}
15475
15476 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15477
15478 ins_pipe(icmp_reg_imm);
15479 %}
15480
15481 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15482 %{
15483 match(Set cr (CmpL op1 op2));
15484
15485 effect(DEF cr, USE op1);
15486
15487 ins_cost(INSN_COST * 2);
15488 format %{ "cmp $op1, $op2" %}
15489
15490 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15491
15492 ins_pipe(icmp_reg_imm);
15493 %}
15494
15495 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15496 %{
15497 match(Set cr (CmpUL op1 op2));
15498
15499 effect(DEF cr, USE op1, USE op2);
15500
15501 ins_cost(INSN_COST);
15502 format %{ "cmp $op1, $op2" %}
15503
15504 ins_encode(aarch64_enc_cmp(op1, op2));
15505
15506 ins_pipe(icmp_reg_reg);
15507 %}
15508
15509 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15510 %{
15511 match(Set cr (CmpUL op1 zero));
15512
15513 effect(DEF cr, USE op1);
15514
15515 ins_cost(INSN_COST);
15516 format %{ "tst $op1" %}
15517
15518 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15519
15520 ins_pipe(icmp_reg_imm);
15521 %}
15522
15523 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15524 %{
15525 match(Set cr (CmpUL op1 op2));
15526
15527 effect(DEF cr, USE op1);
15528
15529 ins_cost(INSN_COST);
15530 format %{ "cmp $op1, $op2" %}
15531
15532 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15533
15534 ins_pipe(icmp_reg_imm);
15535 %}
15536
15537 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15538 %{
15539 match(Set cr (CmpUL op1 op2));
15540
15541 effect(DEF cr, USE op1);
15542
15543 ins_cost(INSN_COST * 2);
15544 format %{ "cmp $op1, $op2" %}
15545
15546 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15547
15548 ins_pipe(icmp_reg_imm);
15549 %}
15550
15551 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15552 %{
15553 match(Set cr (CmpP op1 op2));
15554
15555 effect(DEF cr, USE op1, USE op2);
15556
15557 ins_cost(INSN_COST);
15558 format %{ "cmp $op1, $op2\t // ptr" %}
15559
15560 ins_encode(aarch64_enc_cmpp(op1, op2));
15561
15562 ins_pipe(icmp_reg_reg);
15563 %}
15564
15565 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15566 %{
15567 match(Set cr (CmpN op1 op2));
15568
15569 effect(DEF cr, USE op1, USE op2);
15570
15571 ins_cost(INSN_COST);
15572 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15573
15574 ins_encode(aarch64_enc_cmpn(op1, op2));
15575
15576 ins_pipe(icmp_reg_reg);
15577 %}
15578
15579 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15580 %{
15581 match(Set cr (CmpP op1 zero));
15582
15583 effect(DEF cr, USE op1, USE zero);
15584
15585 ins_cost(INSN_COST);
15586 format %{ "cmp $op1, 0\t // ptr" %}
15587
15588 ins_encode(aarch64_enc_testp(op1));
15589
15590 ins_pipe(icmp_reg_imm);
15591 %}
15592
15593 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15594 %{
15595 match(Set cr (CmpN op1 zero));
15596
15597 effect(DEF cr, USE op1, USE zero);
15598
15599 ins_cost(INSN_COST);
15600 format %{ "cmp $op1, 0\t // compressed ptr" %}
15601
15602 ins_encode(aarch64_enc_testn(op1));
15603
15604 ins_pipe(icmp_reg_imm);
15605 %}
15606
15607 // FP comparisons
15608 //
15609 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15610 // using normal cmpOp. See declaration of rFlagsReg for details.
15611
15612 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15613 %{
15614 match(Set cr (CmpF src1 src2));
15615
15616 ins_cost(3 * INSN_COST);
15617 format %{ "fcmps $src1, $src2" %}
15618
15619 ins_encode %{
15620 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15621 %}
15622
15623 ins_pipe(pipe_class_compare);
15624 %}
15625
15626 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15627 %{
15628 match(Set cr (CmpF src1 src2));
15629
15630 ins_cost(3 * INSN_COST);
15631 format %{ "fcmps $src1, 0.0" %}
15632
15633 ins_encode %{
15634 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15635 %}
15636
15637 ins_pipe(pipe_class_compare);
15638 %}
15639 // FROM HERE
15640
15641 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15642 %{
15643 match(Set cr (CmpD src1 src2));
15644
15645 ins_cost(3 * INSN_COST);
15646 format %{ "fcmpd $src1, $src2" %}
15647
15648 ins_encode %{
15649 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15650 %}
15651
15652 ins_pipe(pipe_class_compare);
15653 %}
15654
15655 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15656 %{
15657 match(Set cr (CmpD src1 src2));
15658
15659 ins_cost(3 * INSN_COST);
15660 format %{ "fcmpd $src1, 0.0" %}
15661
15662 ins_encode %{
15663 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15664 %}
15665
15666 ins_pipe(pipe_class_compare);
15667 %}
15668
15669 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15670 %{
15671 match(Set dst (CmpF3 src1 src2));
15672 effect(KILL cr);
15673
15674 ins_cost(5 * INSN_COST);
15675 format %{ "fcmps $src1, $src2\n\t"
15676 "csinvw($dst, zr, zr, eq\n\t"
15677 "csnegw($dst, $dst, $dst, lt)"
15678 %}
15679
15680 ins_encode %{
15681 Label done;
15682 FloatRegister s1 = as_FloatRegister($src1$$reg);
15683 FloatRegister s2 = as_FloatRegister($src2$$reg);
15684 Register d = as_Register($dst$$reg);
15685 __ fcmps(s1, s2);
15686 // installs 0 if EQ else -1
15687 __ csinvw(d, zr, zr, Assembler::EQ);
15688 // keeps -1 if less or unordered else installs 1
15689 __ csnegw(d, d, d, Assembler::LT);
15690 __ bind(done);
15691 %}
15692
15693 ins_pipe(pipe_class_default);
15694
15695 %}
15696
15697 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15698 %{
15699 match(Set dst (CmpD3 src1 src2));
15700 effect(KILL cr);
15701
15702 ins_cost(5 * INSN_COST);
15703 format %{ "fcmpd $src1, $src2\n\t"
15704 "csinvw($dst, zr, zr, eq\n\t"
15705 "csnegw($dst, $dst, $dst, lt)"
15706 %}
15707
15708 ins_encode %{
15709 Label done;
15710 FloatRegister s1 = as_FloatRegister($src1$$reg);
15711 FloatRegister s2 = as_FloatRegister($src2$$reg);
15712 Register d = as_Register($dst$$reg);
15713 __ fcmpd(s1, s2);
15714 // installs 0 if EQ else -1
15715 __ csinvw(d, zr, zr, Assembler::EQ);
15716 // keeps -1 if less or unordered else installs 1
15717 __ csnegw(d, d, d, Assembler::LT);
15718 __ bind(done);
15719 %}
15720 ins_pipe(pipe_class_default);
15721
15722 %}
15723
15724 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15725 %{
15726 match(Set dst (CmpF3 src1 zero));
15727 effect(KILL cr);
15728
15729 ins_cost(5 * INSN_COST);
15730 format %{ "fcmps $src1, 0.0\n\t"
15731 "csinvw($dst, zr, zr, eq\n\t"
15732 "csnegw($dst, $dst, $dst, lt)"
15733 %}
15734
15735 ins_encode %{
15736 Label done;
15737 FloatRegister s1 = as_FloatRegister($src1$$reg);
15738 Register d = as_Register($dst$$reg);
15739 __ fcmps(s1, 0.0);
15740 // installs 0 if EQ else -1
15741 __ csinvw(d, zr, zr, Assembler::EQ);
15742 // keeps -1 if less or unordered else installs 1
15743 __ csnegw(d, d, d, Assembler::LT);
15744 __ bind(done);
15745 %}
15746
15747 ins_pipe(pipe_class_default);
15748
15749 %}
15750
15751 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15752 %{
15753 match(Set dst (CmpD3 src1 zero));
15754 effect(KILL cr);
15755
15756 ins_cost(5 * INSN_COST);
15757 format %{ "fcmpd $src1, 0.0\n\t"
15758 "csinvw($dst, zr, zr, eq\n\t"
15759 "csnegw($dst, $dst, $dst, lt)"
15760 %}
15761
15762 ins_encode %{
15763 Label done;
15764 FloatRegister s1 = as_FloatRegister($src1$$reg);
15765 Register d = as_Register($dst$$reg);
15766 __ fcmpd(s1, 0.0);
15767 // installs 0 if EQ else -1
15768 __ csinvw(d, zr, zr, Assembler::EQ);
15769 // keeps -1 if less or unordered else installs 1
15770 __ csnegw(d, d, d, Assembler::LT);
15771 __ bind(done);
15772 %}
15773 ins_pipe(pipe_class_default);
15774
15775 %}
15776
15777 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15778 %{
15779 match(Set dst (CmpLTMask p q));
15780 effect(KILL cr);
15781
15782 ins_cost(3 * INSN_COST);
15783
15784 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15785 "csetw $dst, lt\n\t"
15786 "subw $dst, zr, $dst"
15787 %}
15788
15789 ins_encode %{
15790 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15791 __ csetw(as_Register($dst$$reg), Assembler::LT);
15792 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15793 %}
15794
15795 ins_pipe(ialu_reg_reg);
15796 %}
15797
15798 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15799 %{
15800 match(Set dst (CmpLTMask src zero));
15801 effect(KILL cr);
15802
15803 ins_cost(INSN_COST);
15804
15805 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15806
15807 ins_encode %{
15808 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15809 %}
15810
15811 ins_pipe(ialu_reg_shift);
15812 %}
15813
15814 // ============================================================================
15815 // Max and Min
15816
15817 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15818
15819 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15820 %{
15821 effect(DEF cr, USE src);
15822 ins_cost(INSN_COST);
15823 format %{ "cmpw $src, 0" %}
15824
15825 ins_encode %{
15826 __ cmpw($src$$Register, 0);
15827 %}
15828 ins_pipe(icmp_reg_imm);
15829 %}
15830
15831 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15832 %{
15833 match(Set dst (MinI src1 src2));
15834 ins_cost(INSN_COST * 3);
15835
15836 expand %{
15837 rFlagsReg cr;
15838 compI_reg_reg(cr, src1, src2);
15839 cmovI_reg_reg_lt(dst, src1, src2, cr);
15840 %}
15841 %}
15842
15843 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15844 %{
15845 match(Set dst (MaxI src1 src2));
15846 ins_cost(INSN_COST * 3);
15847
15848 expand %{
15849 rFlagsReg cr;
15850 compI_reg_reg(cr, src1, src2);
15851 cmovI_reg_reg_gt(dst, src1, src2, cr);
15852 %}
15853 %}
15854
15855
15856 // ============================================================================
15857 // Branch Instructions
15858
15859 // Direct Branch.
15860 instruct branch(label lbl)
15861 %{
15862 match(Goto);
15863
15864 effect(USE lbl);
15865
15866 ins_cost(BRANCH_COST);
15867 format %{ "b $lbl" %}
15868
15869 ins_encode(aarch64_enc_b(lbl));
15870
15871 ins_pipe(pipe_branch);
15872 %}
15873
15874 // Conditional Near Branch
15875 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15876 %{
15877 // Same match rule as `branchConFar'.
15878 match(If cmp cr);
15879
15880 effect(USE lbl);
15881
15882 ins_cost(BRANCH_COST);
15883 // If set to 1 this indicates that the current instruction is a
15884 // short variant of a long branch. This avoids using this
15885 // instruction in first-pass matching. It will then only be used in
15886 // the `Shorten_branches' pass.
15887 // ins_short_branch(1);
15888 format %{ "b$cmp $lbl" %}
15889
15890 ins_encode(aarch64_enc_br_con(cmp, lbl));
15891
15892 ins_pipe(pipe_branch_cond);
15893 %}
15894
15895 // Conditional Near Branch Unsigned
15896 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15897 %{
15898 // Same match rule as `branchConFar'.
15899 match(If cmp cr);
15900
15901 effect(USE lbl);
15902
15903 ins_cost(BRANCH_COST);
15904 // If set to 1 this indicates that the current instruction is a
15905 // short variant of a long branch. This avoids using this
15906 // instruction in first-pass matching. It will then only be used in
15907 // the `Shorten_branches' pass.
15908 // ins_short_branch(1);
15909 format %{ "b$cmp $lbl\t# unsigned" %}
15910
15911 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15912
15913 ins_pipe(pipe_branch_cond);
15914 %}
15915
15916 // Make use of CBZ and CBNZ. These instructions, as well as being
15917 // shorter than (cmp; branch), have the additional benefit of not
15918 // killing the flags.
15919
15920 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15921 match(If cmp (CmpI op1 op2));
15922 effect(USE labl);
15923
15924 ins_cost(BRANCH_COST);
15925 format %{ "cbw$cmp $op1, $labl" %}
15926 ins_encode %{
15927 Label* L = $labl$$label;
15928 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15929 if (cond == Assembler::EQ)
15930 __ cbzw($op1$$Register, *L);
15931 else
15932 __ cbnzw($op1$$Register, *L);
15933 %}
15934 ins_pipe(pipe_cmp_branch);
15935 %}
15936
15937 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15938 match(If cmp (CmpL op1 op2));
15939 effect(USE labl);
15940
15941 ins_cost(BRANCH_COST);
15942 format %{ "cb$cmp $op1, $labl" %}
15943 ins_encode %{
15944 Label* L = $labl$$label;
15945 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15946 if (cond == Assembler::EQ)
15947 __ cbz($op1$$Register, *L);
15948 else
15949 __ cbnz($op1$$Register, *L);
15950 %}
15951 ins_pipe(pipe_cmp_branch);
15952 %}
15953
15954 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15955 match(If cmp (CmpP op1 op2));
15956 effect(USE labl);
15957
15958 ins_cost(BRANCH_COST);
15959 format %{ "cb$cmp $op1, $labl" %}
15960 ins_encode %{
15961 Label* L = $labl$$label;
15962 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15963 if (cond == Assembler::EQ)
15964 __ cbz($op1$$Register, *L);
15965 else
15966 __ cbnz($op1$$Register, *L);
15967 %}
15968 ins_pipe(pipe_cmp_branch);
15969 %}
15970
15971 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15972 match(If cmp (CmpN op1 op2));
15973 effect(USE labl);
15974
15975 ins_cost(BRANCH_COST);
15976 format %{ "cbw$cmp $op1, $labl" %}
15977 ins_encode %{
15978 Label* L = $labl$$label;
15979 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15980 if (cond == Assembler::EQ)
15981 __ cbzw($op1$$Register, *L);
15982 else
15983 __ cbnzw($op1$$Register, *L);
15984 %}
15985 ins_pipe(pipe_cmp_branch);
15986 %}
15987
15988 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15989 match(If cmp (CmpP (DecodeN oop) zero));
15990 effect(USE labl);
15991
15992 ins_cost(BRANCH_COST);
15993 format %{ "cb$cmp $oop, $labl" %}
15994 ins_encode %{
15995 Label* L = $labl$$label;
15996 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15997 if (cond == Assembler::EQ)
15998 __ cbzw($oop$$Register, *L);
15999 else
16000 __ cbnzw($oop$$Register, *L);
16001 %}
16002 ins_pipe(pipe_cmp_branch);
16003 %}
16004
16005 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16006 match(If cmp (CmpU op1 op2));
16007 effect(USE labl);
16008
16009 ins_cost(BRANCH_COST);
16010 format %{ "cbw$cmp $op1, $labl" %}
16011 ins_encode %{
16012 Label* L = $labl$$label;
16013 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16014 if (cond == Assembler::EQ || cond == Assembler::LS) {
16015 __ cbzw($op1$$Register, *L);
16016 } else {
16017 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16018 __ cbnzw($op1$$Register, *L);
16019 }
16020 %}
16021 ins_pipe(pipe_cmp_branch);
16022 %}
16023
16024 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
16025 match(If cmp (CmpUL op1 op2));
16026 effect(USE labl);
16027
16028 ins_cost(BRANCH_COST);
16029 format %{ "cb$cmp $op1, $labl" %}
16030 ins_encode %{
16031 Label* L = $labl$$label;
16032 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16033 if (cond == Assembler::EQ || cond == Assembler::LS) {
16034 __ cbz($op1$$Register, *L);
16035 } else {
16036 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16037 __ cbnz($op1$$Register, *L);
16038 }
16039 %}
16040 ins_pipe(pipe_cmp_branch);
16041 %}
16042
16043 // Test bit and Branch
16044
16045 // Patterns for short (< 32KiB) variants
16046 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16047 match(If cmp (CmpL op1 op2));
16048 effect(USE labl);
16049
16050 ins_cost(BRANCH_COST);
16051 format %{ "cb$cmp $op1, $labl # long" %}
16052 ins_encode %{
16053 Label* L = $labl$$label;
16054 Assembler::Condition cond =
16055 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16056 __ tbr(cond, $op1$$Register, 63, *L);
16057 %}
16058 ins_pipe(pipe_cmp_branch);
16059 ins_short_branch(1);
16060 %}
16061
16062 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16063 match(If cmp (CmpI op1 op2));
16064 effect(USE labl);
16065
16066 ins_cost(BRANCH_COST);
16067 format %{ "cb$cmp $op1, $labl # int" %}
16068 ins_encode %{
16069 Label* L = $labl$$label;
16070 Assembler::Condition cond =
16071 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16072 __ tbr(cond, $op1$$Register, 31, *L);
16073 %}
16074 ins_pipe(pipe_cmp_branch);
16075 ins_short_branch(1);
16076 %}
16077
16078 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16079 match(If cmp (CmpL (AndL op1 op2) op3));
16080 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16081 effect(USE labl);
16082
16083 ins_cost(BRANCH_COST);
16084 format %{ "tb$cmp $op1, $op2, $labl" %}
16085 ins_encode %{
16086 Label* L = $labl$$label;
16087 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16088 int bit = exact_log2_long($op2$$constant);
16089 __ tbr(cond, $op1$$Register, bit, *L);
16090 %}
16091 ins_pipe(pipe_cmp_branch);
16092 ins_short_branch(1);
16093 %}
16094
16095 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16096 match(If cmp (CmpI (AndI op1 op2) op3));
16097 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16098 effect(USE labl);
16099
16100 ins_cost(BRANCH_COST);
16101 format %{ "tb$cmp $op1, $op2, $labl" %}
16102 ins_encode %{
16103 Label* L = $labl$$label;
16104 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16105 int bit = exact_log2((juint)$op2$$constant);
16106 __ tbr(cond, $op1$$Register, bit, *L);
16107 %}
16108 ins_pipe(pipe_cmp_branch);
16109 ins_short_branch(1);
16110 %}
16111
16112 // And far variants
16113 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16114 match(If cmp (CmpL op1 op2));
16115 effect(USE labl);
16116
16117 ins_cost(BRANCH_COST);
16118 format %{ "cb$cmp $op1, $labl # long" %}
16119 ins_encode %{
16120 Label* L = $labl$$label;
16121 Assembler::Condition cond =
16122 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16123 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16124 %}
16125 ins_pipe(pipe_cmp_branch);
16126 %}
16127
16128 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16129 match(If cmp (CmpI op1 op2));
16130 effect(USE labl);
16131
16132 ins_cost(BRANCH_COST);
16133 format %{ "cb$cmp $op1, $labl # int" %}
16134 ins_encode %{
16135 Label* L = $labl$$label;
16136 Assembler::Condition cond =
16137 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16138 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16139 %}
16140 ins_pipe(pipe_cmp_branch);
16141 %}
16142
16143 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16144 match(If cmp (CmpL (AndL op1 op2) op3));
16145 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16146 effect(USE labl);
16147
16148 ins_cost(BRANCH_COST);
16149 format %{ "tb$cmp $op1, $op2, $labl" %}
16150 ins_encode %{
16151 Label* L = $labl$$label;
16152 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16153 int bit = exact_log2_long($op2$$constant);
16154 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16155 %}
16156 ins_pipe(pipe_cmp_branch);
16157 %}
16158
16159 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16160 match(If cmp (CmpI (AndI op1 op2) op3));
16161 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16162 effect(USE labl);
16163
16164 ins_cost(BRANCH_COST);
16165 format %{ "tb$cmp $op1, $op2, $labl" %}
16166 ins_encode %{
16167 Label* L = $labl$$label;
16168 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16169 int bit = exact_log2((juint)$op2$$constant);
16170 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16171 %}
16172 ins_pipe(pipe_cmp_branch);
16173 %}
16174
16175 // Test bits
16176
16177 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16178 match(Set cr (CmpL (AndL op1 op2) op3));
16179 predicate(Assembler::operand_valid_for_logical_immediate
16180 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16181
16182 ins_cost(INSN_COST);
16183 format %{ "tst $op1, $op2 # long" %}
16184 ins_encode %{
16185 __ tst($op1$$Register, $op2$$constant);
16186 %}
16187 ins_pipe(ialu_reg_reg);
16188 %}
16189
16190 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16191 match(Set cr (CmpI (AndI op1 op2) op3));
16192 predicate(Assembler::operand_valid_for_logical_immediate
16193 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16194
16195 ins_cost(INSN_COST);
16196 format %{ "tst $op1, $op2 # int" %}
16197 ins_encode %{
16198 __ tstw($op1$$Register, $op2$$constant);
16199 %}
16200 ins_pipe(ialu_reg_reg);
16201 %}
16202
16203 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16204 match(Set cr (CmpL (AndL op1 op2) op3));
16205
16206 ins_cost(INSN_COST);
16207 format %{ "tst $op1, $op2 # long" %}
16208 ins_encode %{
16209 __ tst($op1$$Register, $op2$$Register);
16210 %}
16211 ins_pipe(ialu_reg_reg);
16212 %}
16213
16214 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16215 match(Set cr (CmpI (AndI op1 op2) op3));
16216
16217 ins_cost(INSN_COST);
16218 format %{ "tstw $op1, $op2 # int" %}
16219 ins_encode %{
16220 __ tstw($op1$$Register, $op2$$Register);
16221 %}
16222 ins_pipe(ialu_reg_reg);
16223 %}
16224
16225
16226 // Conditional Far Branch
16227 // Conditional Far Branch Unsigned
16228 // TODO: fixme
16229
16230 // counted loop end branch near
16231 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16232 %{
16233 match(CountedLoopEnd cmp cr);
16234
16235 effect(USE lbl);
16236
16237 ins_cost(BRANCH_COST);
16238 // short variant.
16239 // ins_short_branch(1);
16240 format %{ "b$cmp $lbl \t// counted loop end" %}
16241
16242 ins_encode(aarch64_enc_br_con(cmp, lbl));
16243
16244 ins_pipe(pipe_branch);
16245 %}
16246
16247 // counted loop end branch far
16248 // TODO: fixme
16249
16250 // ============================================================================
16251 // inlined locking and unlocking
16252
16253 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16254 %{
16255 predicate(LockingMode != LM_LIGHTWEIGHT);
16256 match(Set cr (FastLock object box));
16257 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16258
16259 ins_cost(5 * INSN_COST);
16260 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16261
16262 ins_encode %{
16263 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16264 %}
16265
16266 ins_pipe(pipe_serial);
16267 %}
16268
16269 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16270 %{
16271 predicate(LockingMode != LM_LIGHTWEIGHT);
16272 match(Set cr (FastUnlock object box));
16273 effect(TEMP tmp, TEMP tmp2);
16274
16275 ins_cost(5 * INSN_COST);
16276 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16277
16278 ins_encode %{
16279 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16280 %}
16281
16282 ins_pipe(pipe_serial);
16283 %}
16284
16285 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16286 %{
16287 predicate(LockingMode == LM_LIGHTWEIGHT);
16288 match(Set cr (FastLock object box));
16289 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16290
16291 ins_cost(5 * INSN_COST);
16292 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16293
16294 ins_encode %{
16295 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16296 %}
16297
16298 ins_pipe(pipe_serial);
16299 %}
16300
16301 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16302 %{
16303 predicate(LockingMode == LM_LIGHTWEIGHT);
16304 match(Set cr (FastUnlock object box));
16305 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16306
16307 ins_cost(5 * INSN_COST);
16308 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16309
16310 ins_encode %{
16311 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16312 %}
16313
16314 ins_pipe(pipe_serial);
16315 %}
16316
16317 // ============================================================================
16318 // Safepoint Instructions
16319
16320 // TODO
16321 // provide a near and far version of this code
16322
16323 instruct safePoint(rFlagsReg cr, iRegP poll)
16324 %{
16325 match(SafePoint poll);
16326 effect(KILL cr);
16327
16328 format %{
16329 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16330 %}
16331 ins_encode %{
16332 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16333 %}
16334 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16335 %}
16336
16337
16338 // ============================================================================
16339 // Procedure Call/Return Instructions
16340
16341 // Call Java Static Instruction
16342
16343 instruct CallStaticJavaDirect(method meth)
16344 %{
16345 match(CallStaticJava);
16346
16347 effect(USE meth);
16348
16349 ins_cost(CALL_COST);
16350
16351 format %{ "call,static $meth \t// ==> " %}
16352
16353 ins_encode(aarch64_enc_java_static_call(meth),
16354 aarch64_enc_call_epilog);
16355
16356 ins_pipe(pipe_class_call);
16357 %}
16358
16359 // TO HERE
16360
16361 // Call Java Dynamic Instruction
16362 instruct CallDynamicJavaDirect(method meth)
16363 %{
16364 match(CallDynamicJava);
16365
16366 effect(USE meth);
16367
16368 ins_cost(CALL_COST);
16369
16370 format %{ "CALL,dynamic $meth \t// ==> " %}
16371
16372 ins_encode(aarch64_enc_java_dynamic_call(meth),
16373 aarch64_enc_call_epilog);
16374
16375 ins_pipe(pipe_class_call);
16376 %}
16377
16378 // Call Runtime Instruction
16379
16380 instruct CallRuntimeDirect(method meth)
16381 %{
16382 match(CallRuntime);
16383
16384 effect(USE meth);
16385
16386 ins_cost(CALL_COST);
16387
16388 format %{ "CALL, runtime $meth" %}
16389
16390 ins_encode( aarch64_enc_java_to_runtime(meth) );
16391
16392 ins_pipe(pipe_class_call);
16393 %}
16394
16395 // Call Runtime Instruction
16396
16397 instruct CallLeafDirect(method meth)
16398 %{
16399 match(CallLeaf);
16400
16401 effect(USE meth);
16402
16403 ins_cost(CALL_COST);
16404
16405 format %{ "CALL, runtime leaf $meth" %}
16406
16407 ins_encode( aarch64_enc_java_to_runtime(meth) );
16408
16409 ins_pipe(pipe_class_call);
16410 %}
16411
16412 // Call Runtime Instruction without safepoint and with vector arguments
16413 instruct CallLeafDirectVector(method meth)
16414 %{
16415 match(CallLeafVector);
16416
16417 effect(USE meth);
16418
16419 ins_cost(CALL_COST);
16420
16421 format %{ "CALL, runtime leaf vector $meth" %}
16422
16423 ins_encode(aarch64_enc_java_to_runtime(meth));
16424
16425 ins_pipe(pipe_class_call);
16426 %}
16427
16428 // Call Runtime Instruction
16429
16430 // entry point is null, target holds the address to call
16431 instruct CallLeafNoFPIndirect(iRegP target)
16432 %{
16433 predicate(n->as_Call()->entry_point() == nullptr);
16434
16435 match(CallLeafNoFP target);
16436
16437 ins_cost(CALL_COST);
16438
16439 format %{ "CALL, runtime leaf nofp indirect $target" %}
16440
16441 ins_encode %{
16442 __ blr($target$$Register);
16443 %}
16444
16445 ins_pipe(pipe_class_call);
16446 %}
16447
16448 instruct CallLeafNoFPDirect(method meth)
16449 %{
16450 predicate(n->as_Call()->entry_point() != nullptr);
16451
16452 match(CallLeafNoFP);
16453
16454 effect(USE meth);
16455
16456 ins_cost(CALL_COST);
16457
16458 format %{ "CALL, runtime leaf nofp $meth" %}
16459
16460 ins_encode( aarch64_enc_java_to_runtime(meth) );
16461
16462 ins_pipe(pipe_class_call);
16463 %}
16464
16465 // Tail Call; Jump from runtime stub to Java code.
16466 // Also known as an 'interprocedural jump'.
16467 // Target of jump will eventually return to caller.
16468 // TailJump below removes the return address.
16469 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16470 // emitted just above the TailCall which has reset rfp to the caller state.
16471 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16472 %{
16473 match(TailCall jump_target method_ptr);
16474
16475 ins_cost(CALL_COST);
16476
16477 format %{ "br $jump_target\t# $method_ptr holds method" %}
16478
16479 ins_encode(aarch64_enc_tail_call(jump_target));
16480
16481 ins_pipe(pipe_class_call);
16482 %}
16483
16484 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16485 %{
16486 match(TailJump jump_target ex_oop);
16487
16488 ins_cost(CALL_COST);
16489
16490 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16491
16492 ins_encode(aarch64_enc_tail_jmp(jump_target));
16493
16494 ins_pipe(pipe_class_call);
16495 %}
16496
16497 // Forward exception.
16498 instruct ForwardExceptionjmp()
16499 %{
16500 match(ForwardException);
16501 ins_cost(CALL_COST);
16502
16503 format %{ "b forward_exception_stub" %}
16504 ins_encode %{
16505 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16506 %}
16507 ins_pipe(pipe_class_call);
16508 %}
16509
16510 // Create exception oop: created by stack-crawling runtime code.
16511 // Created exception is now available to this handler, and is setup
16512 // just prior to jumping to this handler. No code emitted.
16513 // TODO check
16514 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16515 instruct CreateException(iRegP_R0 ex_oop)
16516 %{
16517 match(Set ex_oop (CreateEx));
16518
16519 format %{ " -- \t// exception oop; no code emitted" %}
16520
16521 size(0);
16522
16523 ins_encode( /*empty*/ );
16524
16525 ins_pipe(pipe_class_empty);
16526 %}
16527
16528 // Rethrow exception: The exception oop will come in the first
16529 // argument position. Then JUMP (not call) to the rethrow stub code.
16530 instruct RethrowException() %{
16531 match(Rethrow);
16532 ins_cost(CALL_COST);
16533
16534 format %{ "b rethrow_stub" %}
16535
16536 ins_encode( aarch64_enc_rethrow() );
16537
16538 ins_pipe(pipe_class_call);
16539 %}
16540
16541
16542 // Return Instruction
16543 // epilog node loads ret address into lr as part of frame pop
16544 instruct Ret()
16545 %{
16546 match(Return);
16547
16548 format %{ "ret\t// return register" %}
16549
16550 ins_encode( aarch64_enc_ret() );
16551
16552 ins_pipe(pipe_branch);
16553 %}
16554
16555 // Die now.
16556 instruct ShouldNotReachHere() %{
16557 match(Halt);
16558
16559 ins_cost(CALL_COST);
16560 format %{ "ShouldNotReachHere" %}
16561
16562 ins_encode %{
16563 if (is_reachable()) {
16564 const char* str = __ code_string(_halt_reason);
16565 __ stop(str);
16566 }
16567 %}
16568
16569 ins_pipe(pipe_class_default);
16570 %}
16571
16572 // ============================================================================
16573 // Partial Subtype Check
16574 //
16575 // superklass array for an instance of the superklass. Set a hidden
16576 // internal cache on a hit (cache is checked with exposed code in
16577 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16578 // encoding ALSO sets flags.
16579
16580 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16581 %{
16582 match(Set result (PartialSubtypeCheck sub super));
16583 predicate(!UseSecondarySupersTable);
16584 effect(KILL cr, KILL temp);
16585
16586 ins_cost(20 * INSN_COST); // slightly larger than the next version
16587 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16588
16589 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16590
16591 opcode(0x1); // Force zero of result reg on hit
16592
16593 ins_pipe(pipe_class_memory);
16594 %}
16595
16596 // Two versions of partialSubtypeCheck, both used when we need to
16597 // search for a super class in the secondary supers array. The first
16598 // is used when we don't know _a priori_ the class being searched
16599 // for. The second, far more common, is used when we do know: this is
16600 // used for instanceof, checkcast, and any case where C2 can determine
16601 // it by constant propagation.
16602
16603 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16604 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16605 rFlagsReg cr)
16606 %{
16607 match(Set result (PartialSubtypeCheck sub super));
16608 predicate(UseSecondarySupersTable);
16609 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16610
16611 ins_cost(10 * INSN_COST); // slightly larger than the next version
16612 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16613
16614 ins_encode %{
16615 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16616 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16617 $vtemp$$FloatRegister,
16618 $result$$Register, /*L_success*/nullptr);
16619 %}
16620
16621 ins_pipe(pipe_class_memory);
16622 %}
16623
16624 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16625 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16626 rFlagsReg cr)
16627 %{
16628 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16629 predicate(UseSecondarySupersTable);
16630 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16631
16632 ins_cost(5 * INSN_COST); // smaller than the next version
16633 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16634
16635 ins_encode %{
16636 bool success = false;
16637 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16638 if (InlineSecondarySupersTest) {
16639 success =
16640 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16641 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16642 $vtemp$$FloatRegister,
16643 $result$$Register,
16644 super_klass_slot);
16645 } else {
16646 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16647 success = (call != nullptr);
16648 }
16649 if (!success) {
16650 ciEnv::current()->record_failure("CodeCache is full");
16651 return;
16652 }
16653 %}
16654
16655 ins_pipe(pipe_class_memory);
16656 %}
16657
16658 // Intrisics for String.compareTo()
16659
16660 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16661 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16662 %{
16663 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16664 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16665 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16666
16667 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16668 ins_encode %{
16669 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16670 __ string_compare($str1$$Register, $str2$$Register,
16671 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16672 $tmp1$$Register, $tmp2$$Register,
16673 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16674 %}
16675 ins_pipe(pipe_class_memory);
16676 %}
16677
16678 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16679 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16680 %{
16681 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16682 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16683 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16684
16685 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16686 ins_encode %{
16687 __ string_compare($str1$$Register, $str2$$Register,
16688 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16689 $tmp1$$Register, $tmp2$$Register,
16690 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16691 %}
16692 ins_pipe(pipe_class_memory);
16693 %}
16694
16695 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16696 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16697 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16698 %{
16699 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16700 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16701 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16702 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16703
16704 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16705 ins_encode %{
16706 __ string_compare($str1$$Register, $str2$$Register,
16707 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16708 $tmp1$$Register, $tmp2$$Register,
16709 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16710 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16711 %}
16712 ins_pipe(pipe_class_memory);
16713 %}
16714
16715 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16716 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16717 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16718 %{
16719 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16720 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16721 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16722 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16723
16724 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16725 ins_encode %{
16726 __ string_compare($str1$$Register, $str2$$Register,
16727 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16728 $tmp1$$Register, $tmp2$$Register,
16729 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16730 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16731 %}
16732 ins_pipe(pipe_class_memory);
16733 %}
16734
16735 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16736 // these string_compare variants as NEON register type for convenience so that the prototype of
16737 // string_compare can be shared with all variants.
16738
16739 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16740 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16741 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16742 pRegGov_P1 pgtmp2, rFlagsReg cr)
16743 %{
16744 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16745 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16746 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16747 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16748
16749 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16750 ins_encode %{
16751 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16752 __ string_compare($str1$$Register, $str2$$Register,
16753 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16754 $tmp1$$Register, $tmp2$$Register,
16755 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16756 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16757 StrIntrinsicNode::LL);
16758 %}
16759 ins_pipe(pipe_class_memory);
16760 %}
16761
16762 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16763 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16764 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16765 pRegGov_P1 pgtmp2, rFlagsReg cr)
16766 %{
16767 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16768 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16769 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16770 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16771
16772 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16773 ins_encode %{
16774 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16775 __ string_compare($str1$$Register, $str2$$Register,
16776 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16777 $tmp1$$Register, $tmp2$$Register,
16778 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16779 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16780 StrIntrinsicNode::LU);
16781 %}
16782 ins_pipe(pipe_class_memory);
16783 %}
16784
16785 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16786 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16787 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16788 pRegGov_P1 pgtmp2, rFlagsReg cr)
16789 %{
16790 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16791 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16792 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16793 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16794
16795 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16796 ins_encode %{
16797 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16798 __ string_compare($str1$$Register, $str2$$Register,
16799 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16800 $tmp1$$Register, $tmp2$$Register,
16801 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16802 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16803 StrIntrinsicNode::UL);
16804 %}
16805 ins_pipe(pipe_class_memory);
16806 %}
16807
16808 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16809 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16810 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16811 pRegGov_P1 pgtmp2, rFlagsReg cr)
16812 %{
16813 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16814 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16815 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16816 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16817
16818 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16819 ins_encode %{
16820 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16821 __ string_compare($str1$$Register, $str2$$Register,
16822 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16823 $tmp1$$Register, $tmp2$$Register,
16824 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16825 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16826 StrIntrinsicNode::UU);
16827 %}
16828 ins_pipe(pipe_class_memory);
16829 %}
16830
16831 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16832 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16833 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16834 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16835 %{
16836 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16837 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16838 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16839 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16840 TEMP vtmp0, TEMP vtmp1, KILL cr);
16841 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16842 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16843
16844 ins_encode %{
16845 __ string_indexof($str1$$Register, $str2$$Register,
16846 $cnt1$$Register, $cnt2$$Register,
16847 $tmp1$$Register, $tmp2$$Register,
16848 $tmp3$$Register, $tmp4$$Register,
16849 $tmp5$$Register, $tmp6$$Register,
16850 -1, $result$$Register, StrIntrinsicNode::UU);
16851 %}
16852 ins_pipe(pipe_class_memory);
16853 %}
16854
16855 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16856 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16857 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16858 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16859 %{
16860 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16861 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16862 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16863 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16864 TEMP vtmp0, TEMP vtmp1, KILL cr);
16865 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16866 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16867
16868 ins_encode %{
16869 __ string_indexof($str1$$Register, $str2$$Register,
16870 $cnt1$$Register, $cnt2$$Register,
16871 $tmp1$$Register, $tmp2$$Register,
16872 $tmp3$$Register, $tmp4$$Register,
16873 $tmp5$$Register, $tmp6$$Register,
16874 -1, $result$$Register, StrIntrinsicNode::LL);
16875 %}
16876 ins_pipe(pipe_class_memory);
16877 %}
16878
16879 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16880 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16881 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16882 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16883 %{
16884 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16885 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16886 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16887 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16888 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16889 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16890 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16891
16892 ins_encode %{
16893 __ string_indexof($str1$$Register, $str2$$Register,
16894 $cnt1$$Register, $cnt2$$Register,
16895 $tmp1$$Register, $tmp2$$Register,
16896 $tmp3$$Register, $tmp4$$Register,
16897 $tmp5$$Register, $tmp6$$Register,
16898 -1, $result$$Register, StrIntrinsicNode::UL);
16899 %}
16900 ins_pipe(pipe_class_memory);
16901 %}
16902
16903 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16904 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16905 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16906 %{
16907 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16908 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16909 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16910 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16911 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16912 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16913
16914 ins_encode %{
16915 int icnt2 = (int)$int_cnt2$$constant;
16916 __ string_indexof($str1$$Register, $str2$$Register,
16917 $cnt1$$Register, zr,
16918 $tmp1$$Register, $tmp2$$Register,
16919 $tmp3$$Register, $tmp4$$Register, zr, zr,
16920 icnt2, $result$$Register, StrIntrinsicNode::UU);
16921 %}
16922 ins_pipe(pipe_class_memory);
16923 %}
16924
16925 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16926 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16927 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16928 %{
16929 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16930 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16931 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16932 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16933 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16934 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16935
16936 ins_encode %{
16937 int icnt2 = (int)$int_cnt2$$constant;
16938 __ string_indexof($str1$$Register, $str2$$Register,
16939 $cnt1$$Register, zr,
16940 $tmp1$$Register, $tmp2$$Register,
16941 $tmp3$$Register, $tmp4$$Register, zr, zr,
16942 icnt2, $result$$Register, StrIntrinsicNode::LL);
16943 %}
16944 ins_pipe(pipe_class_memory);
16945 %}
16946
16947 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16948 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16949 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16950 %{
16951 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16952 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16953 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16954 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16955 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16956 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16957
16958 ins_encode %{
16959 int icnt2 = (int)$int_cnt2$$constant;
16960 __ string_indexof($str1$$Register, $str2$$Register,
16961 $cnt1$$Register, zr,
16962 $tmp1$$Register, $tmp2$$Register,
16963 $tmp3$$Register, $tmp4$$Register, zr, zr,
16964 icnt2, $result$$Register, StrIntrinsicNode::UL);
16965 %}
16966 ins_pipe(pipe_class_memory);
16967 %}
16968
16969 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16970 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16971 iRegINoSp tmp3, rFlagsReg cr)
16972 %{
16973 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16974 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16975 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16976 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16977
16978 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16979
16980 ins_encode %{
16981 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16982 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16983 $tmp3$$Register);
16984 %}
16985 ins_pipe(pipe_class_memory);
16986 %}
16987
16988 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16989 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16990 iRegINoSp tmp3, rFlagsReg cr)
16991 %{
16992 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16993 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16994 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16995 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16996
16997 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16998
16999 ins_encode %{
17000 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17001 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17002 $tmp3$$Register);
17003 %}
17004 ins_pipe(pipe_class_memory);
17005 %}
17006
17007 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17008 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17009 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17010 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17011 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17012 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17013 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17014 ins_encode %{
17015 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17016 $result$$Register, $ztmp1$$FloatRegister,
17017 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17018 $ptmp$$PRegister, true /* isL */);
17019 %}
17020 ins_pipe(pipe_class_memory);
17021 %}
17022
17023 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17024 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17025 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17026 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17027 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17028 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17029 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17030 ins_encode %{
17031 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17032 $result$$Register, $ztmp1$$FloatRegister,
17033 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17034 $ptmp$$PRegister, false /* isL */);
17035 %}
17036 ins_pipe(pipe_class_memory);
17037 %}
17038
17039 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17040 iRegI_R0 result, rFlagsReg cr)
17041 %{
17042 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17043 match(Set result (StrEquals (Binary str1 str2) cnt));
17044 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17045
17046 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17047 ins_encode %{
17048 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17049 __ string_equals($str1$$Register, $str2$$Register,
17050 $result$$Register, $cnt$$Register);
17051 %}
17052 ins_pipe(pipe_class_memory);
17053 %}
17054
17055 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17056 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17057 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17058 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17059 iRegP_R10 tmp, rFlagsReg cr)
17060 %{
17061 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17062 match(Set result (AryEq ary1 ary2));
17063 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17064 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17065 TEMP vtmp6, TEMP vtmp7, KILL cr);
17066
17067 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17068 ins_encode %{
17069 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17070 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17071 $result$$Register, $tmp$$Register, 1);
17072 if (tpc == nullptr) {
17073 ciEnv::current()->record_failure("CodeCache is full");
17074 return;
17075 }
17076 %}
17077 ins_pipe(pipe_class_memory);
17078 %}
17079
17080 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17081 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17082 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17083 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17084 iRegP_R10 tmp, rFlagsReg cr)
17085 %{
17086 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17087 match(Set result (AryEq ary1 ary2));
17088 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17089 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17090 TEMP vtmp6, TEMP vtmp7, KILL cr);
17091
17092 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17093 ins_encode %{
17094 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17095 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17096 $result$$Register, $tmp$$Register, 2);
17097 if (tpc == nullptr) {
17098 ciEnv::current()->record_failure("CodeCache is full");
17099 return;
17100 }
17101 %}
17102 ins_pipe(pipe_class_memory);
17103 %}
17104
17105 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17106 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17107 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17108 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17109 %{
17110 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17111 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17112 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17113
17114 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
17115 ins_encode %{
17116 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17117 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17118 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17119 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17120 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17121 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17122 (BasicType)$basic_type$$constant);
17123 if (tpc == nullptr) {
17124 ciEnv::current()->record_failure("CodeCache is full");
17125 return;
17126 }
17127 %}
17128 ins_pipe(pipe_class_memory);
17129 %}
17130
17131 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17132 %{
17133 match(Set result (CountPositives ary1 len));
17134 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17135 format %{ "count positives byte[] $ary1,$len -> $result" %}
17136 ins_encode %{
17137 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17138 if (tpc == nullptr) {
17139 ciEnv::current()->record_failure("CodeCache is full");
17140 return;
17141 }
17142 %}
17143 ins_pipe( pipe_slow );
17144 %}
17145
17146 // fast char[] to byte[] compression
17147 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17148 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17149 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17150 iRegI_R0 result, rFlagsReg cr)
17151 %{
17152 match(Set result (StrCompressedCopy src (Binary dst len)));
17153 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17154 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17155
17156 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17157 ins_encode %{
17158 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17159 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17160 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17161 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17162 %}
17163 ins_pipe(pipe_slow);
17164 %}
17165
17166 // fast byte[] to char[] inflation
17167 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17168 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17169 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17170 %{
17171 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17172 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17173 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17174 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17175
17176 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17177 ins_encode %{
17178 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17179 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17180 $vtmp2$$FloatRegister, $tmp$$Register);
17181 if (tpc == nullptr) {
17182 ciEnv::current()->record_failure("CodeCache is full");
17183 return;
17184 }
17185 %}
17186 ins_pipe(pipe_class_memory);
17187 %}
17188
17189 // encode char[] to byte[] in ISO_8859_1
17190 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17191 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17192 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17193 iRegI_R0 result, rFlagsReg cr)
17194 %{
17195 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17196 match(Set result (EncodeISOArray src (Binary dst len)));
17197 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17198 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17199
17200 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17201 ins_encode %{
17202 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17203 $result$$Register, false,
17204 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17205 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17206 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17207 %}
17208 ins_pipe(pipe_class_memory);
17209 %}
17210
17211 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17212 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17213 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17214 iRegI_R0 result, rFlagsReg cr)
17215 %{
17216 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17217 match(Set result (EncodeISOArray src (Binary dst len)));
17218 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17219 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17220
17221 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17222 ins_encode %{
17223 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17224 $result$$Register, true,
17225 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17226 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17227 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17228 %}
17229 ins_pipe(pipe_class_memory);
17230 %}
17231
17232 //----------------------------- CompressBits/ExpandBits ------------------------
17233
17234 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17235 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17236 match(Set dst (CompressBits src mask));
17237 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17238 format %{ "mov $tsrc, $src\n\t"
17239 "mov $tmask, $mask\n\t"
17240 "bext $tdst, $tsrc, $tmask\n\t"
17241 "mov $dst, $tdst"
17242 %}
17243 ins_encode %{
17244 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17245 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17246 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17247 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17248 %}
17249 ins_pipe(pipe_slow);
17250 %}
17251
17252 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17253 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17254 match(Set dst (CompressBits (LoadI mem) mask));
17255 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17256 format %{ "ldrs $tsrc, $mem\n\t"
17257 "ldrs $tmask, $mask\n\t"
17258 "bext $tdst, $tsrc, $tmask\n\t"
17259 "mov $dst, $tdst"
17260 %}
17261 ins_encode %{
17262 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17263 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17264 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17265 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17266 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17267 %}
17268 ins_pipe(pipe_slow);
17269 %}
17270
17271 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17272 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17273 match(Set dst (CompressBits src mask));
17274 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17275 format %{ "mov $tsrc, $src\n\t"
17276 "mov $tmask, $mask\n\t"
17277 "bext $tdst, $tsrc, $tmask\n\t"
17278 "mov $dst, $tdst"
17279 %}
17280 ins_encode %{
17281 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17282 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17283 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17284 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17285 %}
17286 ins_pipe(pipe_slow);
17287 %}
17288
17289 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17290 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17291 match(Set dst (CompressBits (LoadL mem) mask));
17292 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17293 format %{ "ldrd $tsrc, $mem\n\t"
17294 "ldrd $tmask, $mask\n\t"
17295 "bext $tdst, $tsrc, $tmask\n\t"
17296 "mov $dst, $tdst"
17297 %}
17298 ins_encode %{
17299 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17300 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17301 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17302 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17303 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17304 %}
17305 ins_pipe(pipe_slow);
17306 %}
17307
17308 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17309 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17310 match(Set dst (ExpandBits src mask));
17311 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17312 format %{ "mov $tsrc, $src\n\t"
17313 "mov $tmask, $mask\n\t"
17314 "bdep $tdst, $tsrc, $tmask\n\t"
17315 "mov $dst, $tdst"
17316 %}
17317 ins_encode %{
17318 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17319 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17320 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17321 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17322 %}
17323 ins_pipe(pipe_slow);
17324 %}
17325
17326 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17327 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17328 match(Set dst (ExpandBits (LoadI mem) mask));
17329 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17330 format %{ "ldrs $tsrc, $mem\n\t"
17331 "ldrs $tmask, $mask\n\t"
17332 "bdep $tdst, $tsrc, $tmask\n\t"
17333 "mov $dst, $tdst"
17334 %}
17335 ins_encode %{
17336 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17337 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17338 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17339 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17340 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17341 %}
17342 ins_pipe(pipe_slow);
17343 %}
17344
17345 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17346 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17347 match(Set dst (ExpandBits src mask));
17348 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17349 format %{ "mov $tsrc, $src\n\t"
17350 "mov $tmask, $mask\n\t"
17351 "bdep $tdst, $tsrc, $tmask\n\t"
17352 "mov $dst, $tdst"
17353 %}
17354 ins_encode %{
17355 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17356 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17357 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17358 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17359 %}
17360 ins_pipe(pipe_slow);
17361 %}
17362
17363
17364 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17365 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17366 match(Set dst (ExpandBits (LoadL mem) mask));
17367 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17368 format %{ "ldrd $tsrc, $mem\n\t"
17369 "ldrd $tmask, $mask\n\t"
17370 "bdep $tdst, $tsrc, $tmask\n\t"
17371 "mov $dst, $tdst"
17372 %}
17373 ins_encode %{
17374 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17375 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17376 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17377 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17378 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17379 %}
17380 ins_pipe(pipe_slow);
17381 %}
17382
17383 //----------------------------- Reinterpret ----------------------------------
17384 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17385 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17386 match(Set dst (ReinterpretHF2S src));
17387 format %{ "reinterpretHF2S $dst, $src" %}
17388 ins_encode %{
17389 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17390 %}
17391 ins_pipe(pipe_slow);
17392 %}
17393
17394 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17395 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17396 match(Set dst (ReinterpretS2HF src));
17397 format %{ "reinterpretS2HF $dst, $src" %}
17398 ins_encode %{
17399 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17400 %}
17401 ins_pipe(pipe_slow);
17402 %}
17403
17404 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17405 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17406 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
17407 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
17408 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
17409 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17410 // can be omitted in this pattern, resulting in -
17411 // fcvt $dst, $src // Convert float to half-precision float
17412 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17413 %{
17414 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17415 format %{ "convF2HFAndS2HF $dst, $src" %}
17416 ins_encode %{
17417 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17418 %}
17419 ins_pipe(pipe_slow);
17420 %}
17421
17422 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17423 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17424 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
17425 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17426 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
17427 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17428 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17429 // resulting in -
17430 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
17431 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17432 %{
17433 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17434 format %{ "convHF2SAndHF2F $dst, $src" %}
17435 ins_encode %{
17436 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17437 %}
17438 ins_pipe(pipe_slow);
17439 %}
17440
17441 // ============================================================================
17442 // This name is KNOWN by the ADLC and cannot be changed.
17443 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17444 // for this guy.
17445 instruct tlsLoadP(thread_RegP dst)
17446 %{
17447 match(Set dst (ThreadLocal));
17448
17449 ins_cost(0);
17450
17451 format %{ " -- \t// $dst=Thread::current(), empty" %}
17452
17453 size(0);
17454
17455 ins_encode( /*empty*/ );
17456
17457 ins_pipe(pipe_class_empty);
17458 %}
17459
17460 //----------PEEPHOLE RULES-----------------------------------------------------
17461 // These must follow all instruction definitions as they use the names
17462 // defined in the instructions definitions.
17463 //
17464 // peepmatch ( root_instr_name [preceding_instruction]* );
17465 //
17466 // peepconstraint %{
17467 // (instruction_number.operand_name relational_op instruction_number.operand_name
17468 // [, ...] );
17469 // // instruction numbers are zero-based using left to right order in peepmatch
17470 //
17471 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17472 // // provide an instruction_number.operand_name for each operand that appears
17473 // // in the replacement instruction's match rule
17474 //
17475 // ---------VM FLAGS---------------------------------------------------------
17476 //
17477 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17478 //
17479 // Each peephole rule is given an identifying number starting with zero and
17480 // increasing by one in the order seen by the parser. An individual peephole
17481 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17482 // on the command-line.
17483 //
17484 // ---------CURRENT LIMITATIONS----------------------------------------------
17485 //
17486 // Only match adjacent instructions in same basic block
17487 // Only equality constraints
17488 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17489 // Only one replacement instruction
17490 //
17491 // ---------EXAMPLE----------------------------------------------------------
17492 //
17493 // // pertinent parts of existing instructions in architecture description
17494 // instruct movI(iRegINoSp dst, iRegI src)
17495 // %{
17496 // match(Set dst (CopyI src));
17497 // %}
17498 //
17499 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17500 // %{
17501 // match(Set dst (AddI dst src));
17502 // effect(KILL cr);
17503 // %}
17504 //
17505 // // Change (inc mov) to lea
17506 // peephole %{
17507 // // increment preceded by register-register move
17508 // peepmatch ( incI_iReg movI );
17509 // // require that the destination register of the increment
17510 // // match the destination register of the move
17511 // peepconstraint ( 0.dst == 1.dst );
17512 // // construct a replacement instruction that sets
17513 // // the destination to ( move's source register + one )
17514 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17515 // %}
17516 //
17517
17518 // Implementation no longer uses movX instructions since
17519 // machine-independent system no longer uses CopyX nodes.
17520 //
17521 // peephole
17522 // %{
17523 // peepmatch (incI_iReg movI);
17524 // peepconstraint (0.dst == 1.dst);
17525 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17526 // %}
17527
17528 // peephole
17529 // %{
17530 // peepmatch (decI_iReg movI);
17531 // peepconstraint (0.dst == 1.dst);
17532 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17533 // %}
17534
17535 // peephole
17536 // %{
17537 // peepmatch (addI_iReg_imm movI);
17538 // peepconstraint (0.dst == 1.dst);
17539 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17540 // %}
17541
17542 // peephole
17543 // %{
17544 // peepmatch (incL_iReg movL);
17545 // peepconstraint (0.dst == 1.dst);
17546 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17547 // %}
17548
17549 // peephole
17550 // %{
17551 // peepmatch (decL_iReg movL);
17552 // peepconstraint (0.dst == 1.dst);
17553 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17554 // %}
17555
17556 // peephole
17557 // %{
17558 // peepmatch (addL_iReg_imm movL);
17559 // peepconstraint (0.dst == 1.dst);
17560 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17561 // %}
17562
17563 // peephole
17564 // %{
17565 // peepmatch (addP_iReg_imm movP);
17566 // peepconstraint (0.dst == 1.dst);
17567 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17568 // %}
17569
17570 // // Change load of spilled value to only a spill
17571 // instruct storeI(memory mem, iRegI src)
17572 // %{
17573 // match(Set mem (StoreI mem src));
17574 // %}
17575 //
17576 // instruct loadI(iRegINoSp dst, memory mem)
17577 // %{
17578 // match(Set dst (LoadI mem));
17579 // %}
17580 //
17581
17582 //----------SMARTSPILL RULES---------------------------------------------------
17583 // These must follow all instruction definitions as they use the names
17584 // defined in the instructions definitions.
17585
17586 // Local Variables:
17587 // mode: c++
17588 // End: