1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return 0; // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1652 // lea(rscratch1, RuntimeAddress(addr)
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else if (_entry_point == nullptr) {
1658 // See CallLeafNoFPIndirect
1659 return 1 * NativeInstruction::instruction_size;
1660 } else {
1661 return 6 * NativeInstruction::instruction_size;
1662 }
1663 }
1664
1665 //=============================================================================
1666
1667 #ifndef PRODUCT
1668 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1669 st->print("BREAKPOINT");
1670 }
1671 #endif
1672
1673 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1674 __ brk(0);
1675 }
1676
1677 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1678 return MachNode::size(ra_);
1679 }
1680
1681 //=============================================================================
1682
1683 #ifndef PRODUCT
1684 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1685 st->print("nop \t# %d bytes pad for loops and calls", _count);
1686 }
1687 #endif
1688
1689 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1690 for (int i = 0; i < _count; i++) {
1691 __ nop();
1692 }
1693 }
1694
1695 uint MachNopNode::size(PhaseRegAlloc*) const {
1696 return _count * NativeInstruction::instruction_size;
1697 }
1698
1699 //=============================================================================
1700 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1701
1702 int ConstantTable::calculate_table_base_offset() const {
1703 return 0; // absolute addressing, no offset
1704 }
1705
1706 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1707 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1708 ShouldNotReachHere();
1709 }
1710
1711 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1712 // Empty encoding
1713 }
1714
1715 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1716 return 0;
1717 }
1718
1719 #ifndef PRODUCT
1720 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1721 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1722 }
1723 #endif
1724
1725 #ifndef PRODUCT
1726 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1727 Compile* C = ra_->C;
1728
1729 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1730
1731 if (C->output()->need_stack_bang(framesize))
1732 st->print("# stack bang size=%d\n\t", framesize);
1733
1734 if (VM_Version::use_rop_protection()) {
1735 st->print("ldr zr, [lr]\n\t");
1736 st->print("paciaz\n\t");
1737 }
1738 if (framesize < ((1 << 9) + 2 * wordSize)) {
1739 st->print("sub sp, sp, #%d\n\t", framesize);
1740 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1741 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1742 } else {
1743 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1744 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1745 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1746 st->print("sub sp, sp, rscratch1");
1747 }
1748 if (C->stub_function() == nullptr) {
1749 st->print("\n\t");
1750 st->print("ldr rscratch1, [guard]\n\t");
1751 st->print("dmb ishld\n\t");
1752 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1753 st->print("cmp rscratch1, rscratch2\n\t");
1754 st->print("b.eq skip");
1755 st->print("\n\t");
1756 st->print("blr #nmethod_entry_barrier_stub\n\t");
1757 st->print("b skip\n\t");
1758 st->print("guard: int\n\t");
1759 st->print("\n\t");
1760 st->print("skip:\n\t");
1761 }
1762 }
1763 #endif
1764
1765 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1766 Compile* C = ra_->C;
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 __ verified_entry(C, 0);
1773
1774 if (C->stub_function() == nullptr) {
1775 __ entry_barrier();
1776 }
1777
1778 if (!Compile::current()->output()->in_scratch_emit_size()) {
1779 __ bind(*_verified_entry);
1780 }
1781
1782 if (VerifyStackAtCalls) {
1783 Unimplemented();
1784 }
1785
1786 C->output()->set_frame_complete(__ offset());
1787
1788 if (C->has_mach_constant_base_node()) {
1789 // NOTE: We set the table base offset here because users might be
1790 // emitted before MachConstantBaseNode.
1791 ConstantTable& constant_table = C->output()->constant_table();
1792 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1793 }
1794 }
1795
1796 int MachPrologNode::reloc() const
1797 {
1798 return 0;
1799 }
1800
1801 //=============================================================================
1802
1803 #ifndef PRODUCT
1804 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1805 Compile* C = ra_->C;
1806 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1807
1808 st->print("# pop frame %d\n\t",framesize);
1809
1810 if (framesize == 0) {
1811 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1812 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1813 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1814 st->print("add sp, sp, #%d\n\t", framesize);
1815 } else {
1816 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1817 st->print("add sp, sp, rscratch1\n\t");
1818 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1819 }
1820 if (VM_Version::use_rop_protection()) {
1821 st->print("autiaz\n\t");
1822 st->print("ldr zr, [lr]\n\t");
1823 }
1824
1825 if (do_polling() && C->is_method_compilation()) {
1826 st->print("# test polling word\n\t");
1827 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1828 st->print("cmp sp, rscratch1\n\t");
1829 st->print("bhi #slow_path");
1830 }
1831 }
1832 #endif
1833
1834 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1835 Compile* C = ra_->C;
1836 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1837
1838 __ remove_frame(framesize, C->needs_stack_repair());
1839
1840 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1841 __ reserved_stack_check();
1842 }
1843
1844 if (do_polling() && C->is_method_compilation()) {
1845 Label dummy_label;
1846 Label* code_stub = &dummy_label;
1847 if (!C->output()->in_scratch_emit_size()) {
1848 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1849 C->output()->add_stub(stub);
1850 code_stub = &stub->entry();
1851 }
1852 __ relocate(relocInfo::poll_return_type);
1853 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1854 }
1855 }
1856
1857 int MachEpilogNode::reloc() const {
1858 // Return number of relocatable values contained in this instruction.
1859 return 1; // 1 for polling page.
1860 }
1861
1862 const Pipeline * MachEpilogNode::pipeline() const {
1863 return MachNode::pipeline_class();
1864 }
1865
1866 //=============================================================================
1867
1868 static enum RC rc_class(OptoReg::Name reg) {
1869
1870 if (reg == OptoReg::Bad) {
1871 return rc_bad;
1872 }
1873
1874 // we have 32 int registers * 2 halves
1875 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1876
1877 if (reg < slots_of_int_registers) {
1878 return rc_int;
1879 }
1880
1881 // we have 32 float register * 8 halves
1882 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1883 if (reg < slots_of_int_registers + slots_of_float_registers) {
1884 return rc_float;
1885 }
1886
1887 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1888 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1889 return rc_predicate;
1890 }
1891
1892 // Between predicate regs & stack is the flags.
1893 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1894
1895 return rc_stack;
1896 }
1897
1898 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1899 Compile* C = ra_->C;
1900
1901 // Get registers to move.
1902 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1903 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1904 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1905 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1906
1907 enum RC src_hi_rc = rc_class(src_hi);
1908 enum RC src_lo_rc = rc_class(src_lo);
1909 enum RC dst_hi_rc = rc_class(dst_hi);
1910 enum RC dst_lo_rc = rc_class(dst_lo);
1911
1912 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1913
1914 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1915 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1916 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1917 "expected aligned-adjacent pairs");
1918 }
1919
1920 if (src_lo == dst_lo && src_hi == dst_hi) {
1921 return 0; // Self copy, no move.
1922 }
1923
1924 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1925 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1926 int src_offset = ra_->reg2offset(src_lo);
1927 int dst_offset = ra_->reg2offset(dst_lo);
1928
1929 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1930 uint ireg = ideal_reg();
1931 if (ireg == Op_VecA && masm) {
1932 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1933 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1934 // stack->stack
1935 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1936 sve_vector_reg_size_in_bytes);
1937 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1938 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1939 sve_vector_reg_size_in_bytes);
1940 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1941 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1942 sve_vector_reg_size_in_bytes);
1943 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1944 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1945 as_FloatRegister(Matcher::_regEncode[src_lo]),
1946 as_FloatRegister(Matcher::_regEncode[src_lo]));
1947 } else {
1948 ShouldNotReachHere();
1949 }
1950 } else if (masm) {
1951 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1952 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1953 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1954 // stack->stack
1955 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1956 if (ireg == Op_VecD) {
1957 __ unspill(rscratch1, true, src_offset);
1958 __ spill(rscratch1, true, dst_offset);
1959 } else {
1960 __ spill_copy128(src_offset, dst_offset);
1961 }
1962 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1963 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1964 ireg == Op_VecD ? __ T8B : __ T16B,
1965 as_FloatRegister(Matcher::_regEncode[src_lo]));
1966 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1967 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1968 ireg == Op_VecD ? __ D : __ Q,
1969 ra_->reg2offset(dst_lo));
1970 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1971 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1972 ireg == Op_VecD ? __ D : __ Q,
1973 ra_->reg2offset(src_lo));
1974 } else {
1975 ShouldNotReachHere();
1976 }
1977 }
1978 } else if (masm) {
1979 switch (src_lo_rc) {
1980 case rc_int:
1981 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
1982 if (is64) {
1983 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1984 as_Register(Matcher::_regEncode[src_lo]));
1985 } else {
1986 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1987 as_Register(Matcher::_regEncode[src_lo]));
1988 }
1989 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1990 if (is64) {
1991 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1992 as_Register(Matcher::_regEncode[src_lo]));
1993 } else {
1994 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1995 as_Register(Matcher::_regEncode[src_lo]));
1996 }
1997 } else { // gpr --> stack spill
1998 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1999 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2000 }
2001 break;
2002 case rc_float:
2003 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2004 if (is64) {
2005 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2006 as_FloatRegister(Matcher::_regEncode[src_lo]));
2007 } else {
2008 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2009 as_FloatRegister(Matcher::_regEncode[src_lo]));
2010 }
2011 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2012 if (is64) {
2013 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2014 as_FloatRegister(Matcher::_regEncode[src_lo]));
2015 } else {
2016 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 as_FloatRegister(Matcher::_regEncode[src_lo]));
2018 }
2019 } else { // fpr --> stack spill
2020 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2021 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2022 is64 ? __ D : __ S, dst_offset);
2023 }
2024 break;
2025 case rc_stack:
2026 if (dst_lo_rc == rc_int) { // stack --> gpr load
2027 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2028 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2029 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2030 is64 ? __ D : __ S, src_offset);
2031 } else if (dst_lo_rc == rc_predicate) {
2032 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2033 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2034 } else { // stack --> stack copy
2035 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2036 if (ideal_reg() == Op_RegVectMask) {
2037 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2038 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2039 } else {
2040 __ unspill(rscratch1, is64, src_offset);
2041 __ spill(rscratch1, is64, dst_offset);
2042 }
2043 }
2044 break;
2045 case rc_predicate:
2046 if (dst_lo_rc == rc_predicate) {
2047 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2048 } else if (dst_lo_rc == rc_stack) {
2049 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2050 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2051 } else {
2052 assert(false, "bad src and dst rc_class combination.");
2053 ShouldNotReachHere();
2054 }
2055 break;
2056 default:
2057 assert(false, "bad rc_class for spill");
2058 ShouldNotReachHere();
2059 }
2060 }
2061
2062 if (st) {
2063 st->print("spill ");
2064 if (src_lo_rc == rc_stack) {
2065 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2066 } else {
2067 st->print("%s -> ", Matcher::regName[src_lo]);
2068 }
2069 if (dst_lo_rc == rc_stack) {
2070 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2071 } else {
2072 st->print("%s", Matcher::regName[dst_lo]);
2073 }
2074 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2075 int vsize = 0;
2076 switch (ideal_reg()) {
2077 case Op_VecD:
2078 vsize = 64;
2079 break;
2080 case Op_VecX:
2081 vsize = 128;
2082 break;
2083 case Op_VecA:
2084 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2085 break;
2086 default:
2087 assert(false, "bad register type for spill");
2088 ShouldNotReachHere();
2089 }
2090 st->print("\t# vector spill size = %d", vsize);
2091 } else if (ideal_reg() == Op_RegVectMask) {
2092 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2093 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2094 st->print("\t# predicate spill size = %d", vsize);
2095 } else {
2096 st->print("\t# spill size = %d", is64 ? 64 : 32);
2097 }
2098 }
2099
2100 return 0;
2101
2102 }
2103
2104 #ifndef PRODUCT
2105 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2106 if (!ra_)
2107 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2108 else
2109 implementation(nullptr, ra_, false, st);
2110 }
2111 #endif
2112
2113 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2114 implementation(masm, ra_, false, nullptr);
2115 }
2116
2117 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2118 return MachNode::size(ra_);
2119 }
2120
2121 //=============================================================================
2122
2123 #ifndef PRODUCT
2124 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2125 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2126 int reg = ra_->get_reg_first(this);
2127 st->print("add %s, rsp, #%d]\t# box lock",
2128 Matcher::regName[reg], offset);
2129 }
2130 #endif
2131
2132 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2133 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2134 int reg = ra_->get_encode(this);
2135
2136 // This add will handle any 24-bit signed offset. 24 bits allows an
2137 // 8 megabyte stack frame.
2138 __ add(as_Register(reg), sp, offset);
2139 }
2140
2141 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2142 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2143 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2144
2145 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2146 return NativeInstruction::instruction_size;
2147 } else {
2148 return 2 * NativeInstruction::instruction_size;
2149 }
2150 }
2151
2152 ///=============================================================================
2153 #ifndef PRODUCT
2154 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2155 {
2156 st->print_cr("# MachVEPNode");
2157 if (!_verified) {
2158 st->print_cr("\t load_class");
2159 } else {
2160 st->print_cr("\t unpack_inline_arg");
2161 }
2162 }
2163 #endif
2164
2165 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2166 {
2167 if (!_verified) {
2168 __ ic_check(1);
2169 } else {
2170 // insert a nop at the start of the prolog so we can patch in a
2171 // branch if we need to invalidate the method later
2172 __ nop();
2173
2174 // TODO 8284443 Avoid creation of temporary frame
2175 if (ra_->C->stub_function() == nullptr) {
2176 __ verified_entry(ra_->C, 0);
2177 __ entry_barrier();
2178 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2179 __ remove_frame(framesize, false);
2180 }
2181 // Unpack inline type args passed as oop and then jump to
2182 // the verified entry point (skipping the unverified entry).
2183 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2184 // Emit code for verified entry and save increment for stack repair on return
2185 __ verified_entry(ra_->C, sp_inc);
2186 if (Compile::current()->output()->in_scratch_emit_size()) {
2187 Label dummy_verified_entry;
2188 __ b(dummy_verified_entry);
2189 } else {
2190 __ b(*_verified_entry);
2191 }
2192 }
2193 }
2194
2195 //=============================================================================
2196 #ifndef PRODUCT
2197 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2198 {
2199 st->print_cr("# MachUEPNode");
2200 if (UseCompressedClassPointers) {
2201 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2202 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2203 st->print_cr("\tcmpw rscratch1, r10");
2204 } else {
2205 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2206 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2207 st->print_cr("\tcmp rscratch1, r10");
2208 }
2209 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2210 }
2211 #endif
2212
2213 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2214 {
2215 __ ic_check(InteriorEntryAlignment);
2216 }
2217
2218 // REQUIRED EMIT CODE
2219
2220 //=============================================================================
2221
2222 // Emit exception handler code.
2223 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2224 {
2225 // mov rscratch1 #exception_blob_entry_point
2226 // br rscratch1
2227 // Note that the code buffer's insts_mark is always relative to insts.
2228 // That's why we must use the macroassembler to generate a handler.
2229 address base = __ start_a_stub(size_exception_handler());
2230 if (base == nullptr) {
2231 ciEnv::current()->record_failure("CodeCache is full");
2232 return 0; // CodeBuffer::expand failed
2233 }
2234 int offset = __ offset();
2235 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2236 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2237 __ end_a_stub();
2238 return offset;
2239 }
2240
2241 // Emit deopt handler code.
2242 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2243 {
2244 // Note that the code buffer's insts_mark is always relative to insts.
2245 // That's why we must use the macroassembler to generate a handler.
2246 address base = __ start_a_stub(size_deopt_handler());
2247 if (base == nullptr) {
2248 ciEnv::current()->record_failure("CodeCache is full");
2249 return 0; // CodeBuffer::expand failed
2250 }
2251 int offset = __ offset();
2252
2253 __ adr(lr, __ pc());
2254 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2255
2256 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2257 __ end_a_stub();
2258 return offset;
2259 }
2260
2261 // REQUIRED MATCHER CODE
2262
2263 //=============================================================================
2264
2265 bool Matcher::match_rule_supported(int opcode) {
2266 if (!has_match_rule(opcode))
2267 return false;
2268
2269 switch (opcode) {
2270 case Op_OnSpinWait:
2271 return VM_Version::supports_on_spin_wait();
2272 case Op_CacheWB:
2273 case Op_CacheWBPreSync:
2274 case Op_CacheWBPostSync:
2275 if (!VM_Version::supports_data_cache_line_flush()) {
2276 return false;
2277 }
2278 break;
2279 case Op_ExpandBits:
2280 case Op_CompressBits:
2281 if (!VM_Version::supports_svebitperm()) {
2282 return false;
2283 }
2284 break;
2285 case Op_FmaF:
2286 case Op_FmaD:
2287 case Op_FmaVF:
2288 case Op_FmaVD:
2289 if (!UseFMA) {
2290 return false;
2291 }
2292 break;
2293 case Op_FmaHF:
2294 // UseFMA flag also needs to be checked along with FEAT_FP16
2295 if (!UseFMA || !is_feat_fp16_supported()) {
2296 return false;
2297 }
2298 break;
2299 case Op_AddHF:
2300 case Op_SubHF:
2301 case Op_MulHF:
2302 case Op_DivHF:
2303 case Op_MinHF:
2304 case Op_MaxHF:
2305 case Op_SqrtHF:
2306 // Half-precision floating point scalar operations require FEAT_FP16
2307 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2308 // features are supported.
2309 if (!is_feat_fp16_supported()) {
2310 return false;
2311 }
2312 break;
2313 }
2314
2315 return true; // Per default match rules are supported.
2316 }
2317
2318 const RegMask* Matcher::predicate_reg_mask(void) {
2319 return &_PR_REG_mask;
2320 }
2321
2322 bool Matcher::supports_vector_calling_convention(void) {
2323 return EnableVectorSupport;
2324 }
2325
2326 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2327 assert(EnableVectorSupport, "sanity");
2328 int lo = V0_num;
2329 int hi = V0_H_num;
2330 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2331 hi = V0_K_num;
2332 }
2333 return OptoRegPair(hi, lo);
2334 }
2335
2336 // Is this branch offset short enough that a short branch can be used?
2337 //
2338 // NOTE: If the platform does not provide any short branch variants, then
2339 // this method should return false for offset 0.
2340 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2341 // The passed offset is relative to address of the branch.
2342
2343 return (-32768 <= offset && offset < 32768);
2344 }
2345
2346 // Vector width in bytes.
2347 int Matcher::vector_width_in_bytes(BasicType bt) {
2348 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2349 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2350 // Minimum 2 values in vector
2351 if (size < 2*type2aelembytes(bt)) size = 0;
2352 // But never < 4
2353 if (size < 4) size = 0;
2354 return size;
2355 }
2356
2357 // Limits on vector size (number of elements) loaded into vector.
2358 int Matcher::max_vector_size(const BasicType bt) {
2359 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2360 }
2361
2362 int Matcher::min_vector_size(const BasicType bt) {
2363 int max_size = max_vector_size(bt);
2364 // Limit the min vector size to 8 bytes.
2365 int size = 8 / type2aelembytes(bt);
2366 if (bt == T_BYTE) {
2367 // To support vector api shuffle/rearrange.
2368 size = 4;
2369 } else if (bt == T_BOOLEAN) {
2370 // To support vector api load/store mask.
2371 size = 2;
2372 }
2373 if (size < 2) size = 2;
2374 return MIN2(size, max_size);
2375 }
2376
2377 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2378 return Matcher::max_vector_size(bt);
2379 }
2380
2381 // Actual max scalable vector register length.
2382 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2383 return Matcher::max_vector_size(bt);
2384 }
2385
2386 // Vector ideal reg.
2387 uint Matcher::vector_ideal_reg(int len) {
2388 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2389 return Op_VecA;
2390 }
2391 switch(len) {
2392 // For 16-bit/32-bit mask vector, reuse VecD.
2393 case 2:
2394 case 4:
2395 case 8: return Op_VecD;
2396 case 16: return Op_VecX;
2397 }
2398 ShouldNotReachHere();
2399 return 0;
2400 }
2401
2402 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2403 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2404 switch (ideal_reg) {
2405 case Op_VecA: return new vecAOper();
2406 case Op_VecD: return new vecDOper();
2407 case Op_VecX: return new vecXOper();
2408 }
2409 ShouldNotReachHere();
2410 return nullptr;
2411 }
2412
2413 bool Matcher::is_reg2reg_move(MachNode* m) {
2414 return false;
2415 }
2416
2417 bool Matcher::is_generic_vector(MachOper* opnd) {
2418 return opnd->opcode() == VREG;
2419 }
2420
2421 // Return whether or not this register is ever used as an argument.
2422 // This function is used on startup to build the trampoline stubs in
2423 // generateOptoStub. Registers not mentioned will be killed by the VM
2424 // call in the trampoline, and arguments in those registers not be
2425 // available to the callee.
2426 bool Matcher::can_be_java_arg(int reg)
2427 {
2428 return
2429 reg == R0_num || reg == R0_H_num ||
2430 reg == R1_num || reg == R1_H_num ||
2431 reg == R2_num || reg == R2_H_num ||
2432 reg == R3_num || reg == R3_H_num ||
2433 reg == R4_num || reg == R4_H_num ||
2434 reg == R5_num || reg == R5_H_num ||
2435 reg == R6_num || reg == R6_H_num ||
2436 reg == R7_num || reg == R7_H_num ||
2437 reg == V0_num || reg == V0_H_num ||
2438 reg == V1_num || reg == V1_H_num ||
2439 reg == V2_num || reg == V2_H_num ||
2440 reg == V3_num || reg == V3_H_num ||
2441 reg == V4_num || reg == V4_H_num ||
2442 reg == V5_num || reg == V5_H_num ||
2443 reg == V6_num || reg == V6_H_num ||
2444 reg == V7_num || reg == V7_H_num;
2445 }
2446
2447 bool Matcher::is_spillable_arg(int reg)
2448 {
2449 return can_be_java_arg(reg);
2450 }
2451
2452 uint Matcher::int_pressure_limit()
2453 {
2454 // JDK-8183543: When taking the number of available registers as int
2455 // register pressure threshold, the jtreg test:
2456 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2457 // failed due to C2 compilation failure with
2458 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2459 //
2460 // A derived pointer is live at CallNode and then is flagged by RA
2461 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2462 // derived pointers and lastly fail to spill after reaching maximum
2463 // number of iterations. Lowering the default pressure threshold to
2464 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2465 // a high register pressure area of the code so that split_DEF can
2466 // generate DefinitionSpillCopy for the derived pointer.
2467 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2468 if (!PreserveFramePointer) {
2469 // When PreserveFramePointer is off, frame pointer is allocatable,
2470 // but different from other SOC registers, it is excluded from
2471 // fatproj's mask because its save type is No-Save. Decrease 1 to
2472 // ensure high pressure at fatproj when PreserveFramePointer is off.
2473 // See check_pressure_at_fatproj().
2474 default_int_pressure_threshold--;
2475 }
2476 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2477 }
2478
2479 uint Matcher::float_pressure_limit()
2480 {
2481 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2482 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2483 }
2484
2485 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2486 return false;
2487 }
2488
2489 RegMask Matcher::divI_proj_mask() {
2490 ShouldNotReachHere();
2491 return RegMask();
2492 }
2493
2494 // Register for MODI projection of divmodI.
2495 RegMask Matcher::modI_proj_mask() {
2496 ShouldNotReachHere();
2497 return RegMask();
2498 }
2499
2500 // Register for DIVL projection of divmodL.
2501 RegMask Matcher::divL_proj_mask() {
2502 ShouldNotReachHere();
2503 return RegMask();
2504 }
2505
2506 // Register for MODL projection of divmodL.
2507 RegMask Matcher::modL_proj_mask() {
2508 ShouldNotReachHere();
2509 return RegMask();
2510 }
2511
2512 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2513 return FP_REG_mask();
2514 }
2515
2516 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2517 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2518 Node* u = addp->fast_out(i);
2519 if (u->is_LoadStore()) {
2520 // On AArch64, LoadStoreNodes (i.e. compare and swap
2521 // instructions) only take register indirect as an operand, so
2522 // any attempt to use an AddPNode as an input to a LoadStoreNode
2523 // must fail.
2524 return false;
2525 }
2526 if (u->is_Mem()) {
2527 int opsize = u->as_Mem()->memory_size();
2528 assert(opsize > 0, "unexpected memory operand size");
2529 if (u->as_Mem()->memory_size() != (1<<shift)) {
2530 return false;
2531 }
2532 }
2533 }
2534 return true;
2535 }
2536
2537 // Convert BootTest condition to Assembler condition.
2538 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2539 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2540 Assembler::Condition result;
2541 switch(cond) {
2542 case BoolTest::eq:
2543 result = Assembler::EQ; break;
2544 case BoolTest::ne:
2545 result = Assembler::NE; break;
2546 case BoolTest::le:
2547 result = Assembler::LE; break;
2548 case BoolTest::ge:
2549 result = Assembler::GE; break;
2550 case BoolTest::lt:
2551 result = Assembler::LT; break;
2552 case BoolTest::gt:
2553 result = Assembler::GT; break;
2554 case BoolTest::ule:
2555 result = Assembler::LS; break;
2556 case BoolTest::uge:
2557 result = Assembler::HS; break;
2558 case BoolTest::ult:
2559 result = Assembler::LO; break;
2560 case BoolTest::ugt:
2561 result = Assembler::HI; break;
2562 case BoolTest::overflow:
2563 result = Assembler::VS; break;
2564 case BoolTest::no_overflow:
2565 result = Assembler::VC; break;
2566 default:
2567 ShouldNotReachHere();
2568 return Assembler::Condition(-1);
2569 }
2570
2571 // Check conversion
2572 if (cond & BoolTest::unsigned_compare) {
2573 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2574 } else {
2575 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2576 }
2577
2578 return result;
2579 }
2580
2581 // Binary src (Replicate con)
2582 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2583 if (n == nullptr || m == nullptr) {
2584 return false;
2585 }
2586
2587 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2588 return false;
2589 }
2590
2591 Node* imm_node = m->in(1);
2592 if (!imm_node->is_Con()) {
2593 return false;
2594 }
2595
2596 const Type* t = imm_node->bottom_type();
2597 if (!(t->isa_int() || t->isa_long())) {
2598 return false;
2599 }
2600
2601 switch (n->Opcode()) {
2602 case Op_AndV:
2603 case Op_OrV:
2604 case Op_XorV: {
2605 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2606 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2607 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2608 }
2609 case Op_AddVB:
2610 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2611 case Op_AddVS:
2612 case Op_AddVI:
2613 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2614 case Op_AddVL:
2615 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2616 default:
2617 return false;
2618 }
2619 }
2620
2621 // (XorV src (Replicate m1))
2622 // (XorVMask src (MaskAll m1))
2623 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2624 if (n != nullptr && m != nullptr) {
2625 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2626 VectorNode::is_all_ones_vector(m);
2627 }
2628 return false;
2629 }
2630
2631 // Should the matcher clone input 'm' of node 'n'?
2632 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2633 if (is_vshift_con_pattern(n, m) ||
2634 is_vector_bitwise_not_pattern(n, m) ||
2635 is_valid_sve_arith_imm_pattern(n, m) ||
2636 is_encode_and_store_pattern(n, m)) {
2637 mstack.push(m, Visit);
2638 return true;
2639 }
2640 return false;
2641 }
2642
2643 // Should the Matcher clone shifts on addressing modes, expecting them
2644 // to be subsumed into complex addressing expressions or compute them
2645 // into registers?
2646 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2647
2648 // Loads and stores with indirect memory input (e.g., volatile loads and
2649 // stores) do not subsume the input into complex addressing expressions. If
2650 // the addressing expression is input to at least one such load or store, do
2651 // not clone the addressing expression. Query needs_acquiring_load and
2652 // needs_releasing_store as a proxy for indirect memory input, as it is not
2653 // possible to directly query for indirect memory input at this stage.
2654 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2655 Node* n = m->fast_out(i);
2656 if (n->is_Load() && needs_acquiring_load(n)) {
2657 return false;
2658 }
2659 if (n->is_Store() && needs_releasing_store(n)) {
2660 return false;
2661 }
2662 }
2663
2664 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2665 return true;
2666 }
2667
2668 Node *off = m->in(AddPNode::Offset);
2669 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2670 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2671 // Are there other uses besides address expressions?
2672 !is_visited(off)) {
2673 address_visited.set(off->_idx); // Flag as address_visited
2674 mstack.push(off->in(2), Visit);
2675 Node *conv = off->in(1);
2676 if (conv->Opcode() == Op_ConvI2L &&
2677 // Are there other uses besides address expressions?
2678 !is_visited(conv)) {
2679 address_visited.set(conv->_idx); // Flag as address_visited
2680 mstack.push(conv->in(1), Pre_Visit);
2681 } else {
2682 mstack.push(conv, Pre_Visit);
2683 }
2684 address_visited.test_set(m->_idx); // Flag as address_visited
2685 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2686 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2687 return true;
2688 } else if (off->Opcode() == Op_ConvI2L &&
2689 // Are there other uses besides address expressions?
2690 !is_visited(off)) {
2691 address_visited.test_set(m->_idx); // Flag as address_visited
2692 address_visited.set(off->_idx); // Flag as address_visited
2693 mstack.push(off->in(1), Pre_Visit);
2694 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2695 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2696 return true;
2697 }
2698 return false;
2699 }
2700
2701 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2702 { \
2703 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2704 guarantee(DISP == 0, "mode not permitted for volatile"); \
2705 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2706 __ INSN(REG, as_Register(BASE)); \
2707 }
2708
2709
2710 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2711 {
2712 Address::extend scale;
2713
2714 // Hooboy, this is fugly. We need a way to communicate to the
2715 // encoder that the index needs to be sign extended, so we have to
2716 // enumerate all the cases.
2717 switch (opcode) {
2718 case INDINDEXSCALEDI2L:
2719 case INDINDEXSCALEDI2LN:
2720 case INDINDEXI2L:
2721 case INDINDEXI2LN:
2722 scale = Address::sxtw(size);
2723 break;
2724 default:
2725 scale = Address::lsl(size);
2726 }
2727
2728 if (index == -1) {
2729 return Address(base, disp);
2730 } else {
2731 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2732 return Address(base, as_Register(index), scale);
2733 }
2734 }
2735
2736
2737 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2738 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2739 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2740 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2741 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2742
2743 // Used for all non-volatile memory accesses. The use of
2744 // $mem->opcode() to discover whether this pattern uses sign-extended
2745 // offsets is something of a kludge.
2746 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2747 Register reg, int opcode,
2748 Register base, int index, int scale, int disp,
2749 int size_in_memory)
2750 {
2751 Address addr = mem2address(opcode, base, index, scale, disp);
2752 if (addr.getMode() == Address::base_plus_offset) {
2753 /* Fix up any out-of-range offsets. */
2754 assert_different_registers(rscratch1, base);
2755 assert_different_registers(rscratch1, reg);
2756 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2757 }
2758 (masm->*insn)(reg, addr);
2759 }
2760
2761 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2762 FloatRegister reg, int opcode,
2763 Register base, int index, int size, int disp,
2764 int size_in_memory)
2765 {
2766 Address::extend scale;
2767
2768 switch (opcode) {
2769 case INDINDEXSCALEDI2L:
2770 case INDINDEXSCALEDI2LN:
2771 scale = Address::sxtw(size);
2772 break;
2773 default:
2774 scale = Address::lsl(size);
2775 }
2776
2777 if (index == -1) {
2778 // Fix up any out-of-range offsets.
2779 assert_different_registers(rscratch1, base);
2780 Address addr = Address(base, disp);
2781 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2782 (masm->*insn)(reg, addr);
2783 } else {
2784 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2785 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2786 }
2787 }
2788
2789 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2790 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2791 int opcode, Register base, int index, int size, int disp)
2792 {
2793 if (index == -1) {
2794 (masm->*insn)(reg, T, Address(base, disp));
2795 } else {
2796 assert(disp == 0, "unsupported address mode");
2797 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2798 }
2799 }
2800
2801 %}
2802
2803
2804
2805 //----------ENCODING BLOCK-----------------------------------------------------
2806 // This block specifies the encoding classes used by the compiler to
2807 // output byte streams. Encoding classes are parameterized macros
2808 // used by Machine Instruction Nodes in order to generate the bit
2809 // encoding of the instruction. Operands specify their base encoding
2810 // interface with the interface keyword. There are currently
2811 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2812 // COND_INTER. REG_INTER causes an operand to generate a function
2813 // which returns its register number when queried. CONST_INTER causes
2814 // an operand to generate a function which returns the value of the
2815 // constant when queried. MEMORY_INTER causes an operand to generate
2816 // four functions which return the Base Register, the Index Register,
2817 // the Scale Value, and the Offset Value of the operand when queried.
2818 // COND_INTER causes an operand to generate six functions which return
2819 // the encoding code (ie - encoding bits for the instruction)
2820 // associated with each basic boolean condition for a conditional
2821 // instruction.
2822 //
2823 // Instructions specify two basic values for encoding. Again, a
2824 // function is available to check if the constant displacement is an
2825 // oop. They use the ins_encode keyword to specify their encoding
2826 // classes (which must be a sequence of enc_class names, and their
2827 // parameters, specified in the encoding block), and they use the
2828 // opcode keyword to specify, in order, their primary, secondary, and
2829 // tertiary opcode. Only the opcode sections which a particular
2830 // instruction needs for encoding need to be specified.
2831 encode %{
2832 // Build emit functions for each basic byte or larger field in the
2833 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2834 // from C++ code in the enc_class source block. Emit functions will
2835 // live in the main source block for now. In future, we can
2836 // generalize this by adding a syntax that specifies the sizes of
2837 // fields in an order, so that the adlc can build the emit functions
2838 // automagically
2839
2840 // catch all for unimplemented encodings
2841 enc_class enc_unimplemented %{
2842 __ unimplemented("C2 catch all");
2843 %}
2844
2845 // BEGIN Non-volatile memory access
2846
2847 // This encoding class is generated automatically from ad_encode.m4.
2848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2849 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2850 Register dst_reg = as_Register($dst$$reg);
2851 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2852 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2853 %}
2854
2855 // This encoding class is generated automatically from ad_encode.m4.
2856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2857 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2858 Register dst_reg = as_Register($dst$$reg);
2859 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2860 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2861 %}
2862
2863 // This encoding class is generated automatically from ad_encode.m4.
2864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2865 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2866 Register dst_reg = as_Register($dst$$reg);
2867 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2868 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2869 %}
2870
2871 // This encoding class is generated automatically from ad_encode.m4.
2872 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2873 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2874 Register dst_reg = as_Register($dst$$reg);
2875 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2876 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2877 %}
2878
2879 // This encoding class is generated automatically from ad_encode.m4.
2880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2881 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2882 Register dst_reg = as_Register($dst$$reg);
2883 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2884 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2885 %}
2886
2887 // This encoding class is generated automatically from ad_encode.m4.
2888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2889 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2890 Register dst_reg = as_Register($dst$$reg);
2891 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2892 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2893 %}
2894
2895 // This encoding class is generated automatically from ad_encode.m4.
2896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2897 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2898 Register dst_reg = as_Register($dst$$reg);
2899 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2900 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2901 %}
2902
2903 // This encoding class is generated automatically from ad_encode.m4.
2904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2905 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2906 Register dst_reg = as_Register($dst$$reg);
2907 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2908 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2909 %}
2910
2911 // This encoding class is generated automatically from ad_encode.m4.
2912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2913 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2914 Register dst_reg = as_Register($dst$$reg);
2915 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2916 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2917 %}
2918
2919 // This encoding class is generated automatically from ad_encode.m4.
2920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2921 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2922 Register dst_reg = as_Register($dst$$reg);
2923 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2924 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2925 %}
2926
2927 // This encoding class is generated automatically from ad_encode.m4.
2928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2929 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2930 Register dst_reg = as_Register($dst$$reg);
2931 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2932 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2933 %}
2934
2935 // This encoding class is generated automatically from ad_encode.m4.
2936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2937 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2938 Register dst_reg = as_Register($dst$$reg);
2939 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2940 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2941 %}
2942
2943 // This encoding class is generated automatically from ad_encode.m4.
2944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2945 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2946 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2947 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2948 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2949 %}
2950
2951 // This encoding class is generated automatically from ad_encode.m4.
2952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2953 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2954 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2955 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2956 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2957 %}
2958
2959 // This encoding class is generated automatically from ad_encode.m4.
2960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2961 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2962 Register src_reg = as_Register($src$$reg);
2963 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2964 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2965 %}
2966
2967 // This encoding class is generated automatically from ad_encode.m4.
2968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2969 enc_class aarch64_enc_strb0(memory1 mem) %{
2970 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2971 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2972 %}
2973
2974 // This encoding class is generated automatically from ad_encode.m4.
2975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2976 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2977 Register src_reg = as_Register($src$$reg);
2978 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2979 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2980 %}
2981
2982 // This encoding class is generated automatically from ad_encode.m4.
2983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2984 enc_class aarch64_enc_strh0(memory2 mem) %{
2985 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2986 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2987 %}
2988
2989 // This encoding class is generated automatically from ad_encode.m4.
2990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2991 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2992 Register src_reg = as_Register($src$$reg);
2993 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
2994 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2995 %}
2996
2997 // This encoding class is generated automatically from ad_encode.m4.
2998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2999 enc_class aarch64_enc_strw0(memory4 mem) %{
3000 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3001 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3002 %}
3003
3004 // This encoding class is generated automatically from ad_encode.m4.
3005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3006 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3007 Register src_reg = as_Register($src$$reg);
3008 // we sometimes get asked to store the stack pointer into the
3009 // current thread -- we cannot do that directly on AArch64
3010 if (src_reg == r31_sp) {
3011 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3012 __ mov(rscratch2, sp);
3013 src_reg = rscratch2;
3014 }
3015 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3016 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3017 %}
3018
3019 // This encoding class is generated automatically from ad_encode.m4.
3020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3021 enc_class aarch64_enc_str0(memory8 mem) %{
3022 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3023 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3024 %}
3025
3026 // This encoding class is generated automatically from ad_encode.m4.
3027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3028 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3029 FloatRegister src_reg = as_FloatRegister($src$$reg);
3030 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3031 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3032 %}
3033
3034 // This encoding class is generated automatically from ad_encode.m4.
3035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3036 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3037 FloatRegister src_reg = as_FloatRegister($src$$reg);
3038 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3039 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3040 %}
3041
3042 // This encoding class is generated automatically from ad_encode.m4.
3043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3044 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3045 __ membar(Assembler::StoreStore);
3046 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3047 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3048 %}
3049
3050 // END Non-volatile memory access
3051
3052 // Vector loads and stores
3053 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3054 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3055 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3056 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3057 %}
3058
3059 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3060 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3061 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3062 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3063 %}
3064
3065 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3066 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3067 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3068 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3069 %}
3070
3071 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3072 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3073 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3074 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3075 %}
3076
3077 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3078 FloatRegister src_reg = as_FloatRegister($src$$reg);
3079 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3080 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3081 %}
3082
3083 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3084 FloatRegister src_reg = as_FloatRegister($src$$reg);
3085 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3086 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3087 %}
3088
3089 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3090 FloatRegister src_reg = as_FloatRegister($src$$reg);
3091 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3092 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3093 %}
3094
3095 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3096 FloatRegister src_reg = as_FloatRegister($src$$reg);
3097 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3098 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3099 %}
3100
3101 // volatile loads and stores
3102
3103 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3104 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3105 rscratch1, stlrb);
3106 %}
3107
3108 enc_class aarch64_enc_stlrb0(memory mem) %{
3109 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3110 rscratch1, stlrb);
3111 %}
3112
3113 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3114 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3115 rscratch1, stlrh);
3116 %}
3117
3118 enc_class aarch64_enc_stlrh0(memory mem) %{
3119 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3120 rscratch1, stlrh);
3121 %}
3122
3123 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3124 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3125 rscratch1, stlrw);
3126 %}
3127
3128 enc_class aarch64_enc_stlrw0(memory mem) %{
3129 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3130 rscratch1, stlrw);
3131 %}
3132
3133 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3134 Register dst_reg = as_Register($dst$$reg);
3135 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3136 rscratch1, ldarb);
3137 __ sxtbw(dst_reg, dst_reg);
3138 %}
3139
3140 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3141 Register dst_reg = as_Register($dst$$reg);
3142 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3143 rscratch1, ldarb);
3144 __ sxtb(dst_reg, dst_reg);
3145 %}
3146
3147 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3148 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3149 rscratch1, ldarb);
3150 %}
3151
3152 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3153 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3154 rscratch1, ldarb);
3155 %}
3156
3157 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3158 Register dst_reg = as_Register($dst$$reg);
3159 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, ldarh);
3161 __ sxthw(dst_reg, dst_reg);
3162 %}
3163
3164 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3165 Register dst_reg = as_Register($dst$$reg);
3166 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3167 rscratch1, ldarh);
3168 __ sxth(dst_reg, dst_reg);
3169 %}
3170
3171 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3172 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3173 rscratch1, ldarh);
3174 %}
3175
3176 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3177 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3178 rscratch1, ldarh);
3179 %}
3180
3181 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3182 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3183 rscratch1, ldarw);
3184 %}
3185
3186 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3187 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3188 rscratch1, ldarw);
3189 %}
3190
3191 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3192 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3193 rscratch1, ldar);
3194 %}
3195
3196 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3197 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3198 rscratch1, ldarw);
3199 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3200 %}
3201
3202 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3203 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3204 rscratch1, ldar);
3205 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3206 %}
3207
3208 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3209 Register src_reg = as_Register($src$$reg);
3210 // we sometimes get asked to store the stack pointer into the
3211 // current thread -- we cannot do that directly on AArch64
3212 if (src_reg == r31_sp) {
3213 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3214 __ mov(rscratch2, sp);
3215 src_reg = rscratch2;
3216 }
3217 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3218 rscratch1, stlr);
3219 %}
3220
3221 enc_class aarch64_enc_stlr0(memory mem) %{
3222 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3223 rscratch1, stlr);
3224 %}
3225
3226 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3227 {
3228 FloatRegister src_reg = as_FloatRegister($src$$reg);
3229 __ fmovs(rscratch2, src_reg);
3230 }
3231 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3232 rscratch1, stlrw);
3233 %}
3234
3235 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3236 {
3237 FloatRegister src_reg = as_FloatRegister($src$$reg);
3238 __ fmovd(rscratch2, src_reg);
3239 }
3240 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3241 rscratch1, stlr);
3242 %}
3243
3244 // synchronized read/update encodings
3245
3246 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3247 Register dst_reg = as_Register($dst$$reg);
3248 Register base = as_Register($mem$$base);
3249 int index = $mem$$index;
3250 int scale = $mem$$scale;
3251 int disp = $mem$$disp;
3252 if (index == -1) {
3253 if (disp != 0) {
3254 __ lea(rscratch1, Address(base, disp));
3255 __ ldaxr(dst_reg, rscratch1);
3256 } else {
3257 // TODO
3258 // should we ever get anything other than this case?
3259 __ ldaxr(dst_reg, base);
3260 }
3261 } else {
3262 Register index_reg = as_Register(index);
3263 if (disp == 0) {
3264 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3265 __ ldaxr(dst_reg, rscratch1);
3266 } else {
3267 __ lea(rscratch1, Address(base, disp));
3268 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3269 __ ldaxr(dst_reg, rscratch1);
3270 }
3271 }
3272 %}
3273
3274 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3275 Register src_reg = as_Register($src$$reg);
3276 Register base = as_Register($mem$$base);
3277 int index = $mem$$index;
3278 int scale = $mem$$scale;
3279 int disp = $mem$$disp;
3280 if (index == -1) {
3281 if (disp != 0) {
3282 __ lea(rscratch2, Address(base, disp));
3283 __ stlxr(rscratch1, src_reg, rscratch2);
3284 } else {
3285 // TODO
3286 // should we ever get anything other than this case?
3287 __ stlxr(rscratch1, src_reg, base);
3288 }
3289 } else {
3290 Register index_reg = as_Register(index);
3291 if (disp == 0) {
3292 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3293 __ stlxr(rscratch1, src_reg, rscratch2);
3294 } else {
3295 __ lea(rscratch2, Address(base, disp));
3296 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3297 __ stlxr(rscratch1, src_reg, rscratch2);
3298 }
3299 }
3300 __ cmpw(rscratch1, zr);
3301 %}
3302
3303 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3304 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3305 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3306 Assembler::xword, /*acquire*/ false, /*release*/ true,
3307 /*weak*/ false, noreg);
3308 %}
3309
3310 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3311 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3312 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3313 Assembler::word, /*acquire*/ false, /*release*/ true,
3314 /*weak*/ false, noreg);
3315 %}
3316
3317 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3318 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3319 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3320 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3321 /*weak*/ false, noreg);
3322 %}
3323
3324 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3325 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3326 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3327 Assembler::byte, /*acquire*/ false, /*release*/ true,
3328 /*weak*/ false, noreg);
3329 %}
3330
3331
3332 // The only difference between aarch64_enc_cmpxchg and
3333 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3334 // CompareAndSwap sequence to serve as a barrier on acquiring a
3335 // lock.
3336 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3337 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3338 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3339 Assembler::xword, /*acquire*/ true, /*release*/ true,
3340 /*weak*/ false, noreg);
3341 %}
3342
3343 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3344 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3345 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3346 Assembler::word, /*acquire*/ true, /*release*/ true,
3347 /*weak*/ false, noreg);
3348 %}
3349
3350 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3351 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3352 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3353 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3354 /*weak*/ false, noreg);
3355 %}
3356
3357 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3358 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3359 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3360 Assembler::byte, /*acquire*/ true, /*release*/ true,
3361 /*weak*/ false, noreg);
3362 %}
3363
3364 // auxiliary used for CompareAndSwapX to set result register
3365 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3366 Register res_reg = as_Register($res$$reg);
3367 __ cset(res_reg, Assembler::EQ);
3368 %}
3369
3370 // prefetch encodings
3371
3372 enc_class aarch64_enc_prefetchw(memory mem) %{
3373 Register base = as_Register($mem$$base);
3374 int index = $mem$$index;
3375 int scale = $mem$$scale;
3376 int disp = $mem$$disp;
3377 if (index == -1) {
3378 // Fix up any out-of-range offsets.
3379 assert_different_registers(rscratch1, base);
3380 Address addr = Address(base, disp);
3381 addr = __ legitimize_address(addr, 8, rscratch1);
3382 __ prfm(addr, PSTL1KEEP);
3383 } else {
3384 Register index_reg = as_Register(index);
3385 if (disp == 0) {
3386 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3387 } else {
3388 __ lea(rscratch1, Address(base, disp));
3389 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3390 }
3391 }
3392 %}
3393
3394 // mov encodings
3395
3396 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3397 uint32_t con = (uint32_t)$src$$constant;
3398 Register dst_reg = as_Register($dst$$reg);
3399 if (con == 0) {
3400 __ movw(dst_reg, zr);
3401 } else {
3402 __ movw(dst_reg, con);
3403 }
3404 %}
3405
3406 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3407 Register dst_reg = as_Register($dst$$reg);
3408 uint64_t con = (uint64_t)$src$$constant;
3409 if (con == 0) {
3410 __ mov(dst_reg, zr);
3411 } else {
3412 __ mov(dst_reg, con);
3413 }
3414 %}
3415
3416 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3417 Register dst_reg = as_Register($dst$$reg);
3418 address con = (address)$src$$constant;
3419 if (con == nullptr || con == (address)1) {
3420 ShouldNotReachHere();
3421 } else {
3422 relocInfo::relocType rtype = $src->constant_reloc();
3423 if (rtype == relocInfo::oop_type) {
3424 __ movoop(dst_reg, (jobject)con);
3425 } else if (rtype == relocInfo::metadata_type) {
3426 __ mov_metadata(dst_reg, (Metadata*)con);
3427 } else {
3428 assert(rtype == relocInfo::none, "unexpected reloc type");
3429 if (! __ is_valid_AArch64_address(con) ||
3430 con < (address)(uintptr_t)os::vm_page_size()) {
3431 __ mov(dst_reg, con);
3432 } else {
3433 uint64_t offset;
3434 __ adrp(dst_reg, con, offset);
3435 __ add(dst_reg, dst_reg, offset);
3436 }
3437 }
3438 }
3439 %}
3440
3441 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3442 Register dst_reg = as_Register($dst$$reg);
3443 __ mov(dst_reg, zr);
3444 %}
3445
3446 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3447 Register dst_reg = as_Register($dst$$reg);
3448 __ mov(dst_reg, (uint64_t)1);
3449 %}
3450
3451 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3452 __ load_byte_map_base($dst$$Register);
3453 %}
3454
3455 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3456 Register dst_reg = as_Register($dst$$reg);
3457 address con = (address)$src$$constant;
3458 if (con == nullptr) {
3459 ShouldNotReachHere();
3460 } else {
3461 relocInfo::relocType rtype = $src->constant_reloc();
3462 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3463 __ set_narrow_oop(dst_reg, (jobject)con);
3464 }
3465 %}
3466
3467 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3468 Register dst_reg = as_Register($dst$$reg);
3469 __ mov(dst_reg, zr);
3470 %}
3471
3472 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3473 Register dst_reg = as_Register($dst$$reg);
3474 address con = (address)$src$$constant;
3475 if (con == nullptr) {
3476 ShouldNotReachHere();
3477 } else {
3478 relocInfo::relocType rtype = $src->constant_reloc();
3479 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3480 __ set_narrow_klass(dst_reg, (Klass *)con);
3481 }
3482 %}
3483
3484 // arithmetic encodings
3485
3486 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3487 Register dst_reg = as_Register($dst$$reg);
3488 Register src_reg = as_Register($src1$$reg);
3489 int32_t con = (int32_t)$src2$$constant;
3490 // add has primary == 0, subtract has primary == 1
3491 if ($primary) { con = -con; }
3492 if (con < 0) {
3493 __ subw(dst_reg, src_reg, -con);
3494 } else {
3495 __ addw(dst_reg, src_reg, con);
3496 }
3497 %}
3498
3499 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3500 Register dst_reg = as_Register($dst$$reg);
3501 Register src_reg = as_Register($src1$$reg);
3502 int32_t con = (int32_t)$src2$$constant;
3503 // add has primary == 0, subtract has primary == 1
3504 if ($primary) { con = -con; }
3505 if (con < 0) {
3506 __ sub(dst_reg, src_reg, -con);
3507 } else {
3508 __ add(dst_reg, src_reg, con);
3509 }
3510 %}
3511
3512 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3513 Register dst_reg = as_Register($dst$$reg);
3514 Register src1_reg = as_Register($src1$$reg);
3515 Register src2_reg = as_Register($src2$$reg);
3516 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3517 %}
3518
3519 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3520 Register dst_reg = as_Register($dst$$reg);
3521 Register src1_reg = as_Register($src1$$reg);
3522 Register src2_reg = as_Register($src2$$reg);
3523 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3524 %}
3525
3526 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3527 Register dst_reg = as_Register($dst$$reg);
3528 Register src1_reg = as_Register($src1$$reg);
3529 Register src2_reg = as_Register($src2$$reg);
3530 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3531 %}
3532
3533 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3534 Register dst_reg = as_Register($dst$$reg);
3535 Register src1_reg = as_Register($src1$$reg);
3536 Register src2_reg = as_Register($src2$$reg);
3537 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3538 %}
3539
3540 // compare instruction encodings
3541
3542 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3543 Register reg1 = as_Register($src1$$reg);
3544 Register reg2 = as_Register($src2$$reg);
3545 __ cmpw(reg1, reg2);
3546 %}
3547
3548 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3549 Register reg = as_Register($src1$$reg);
3550 int32_t val = $src2$$constant;
3551 if (val >= 0) {
3552 __ subsw(zr, reg, val);
3553 } else {
3554 __ addsw(zr, reg, -val);
3555 }
3556 %}
3557
3558 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3559 Register reg1 = as_Register($src1$$reg);
3560 uint32_t val = (uint32_t)$src2$$constant;
3561 __ movw(rscratch1, val);
3562 __ cmpw(reg1, rscratch1);
3563 %}
3564
3565 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3566 Register reg1 = as_Register($src1$$reg);
3567 Register reg2 = as_Register($src2$$reg);
3568 __ cmp(reg1, reg2);
3569 %}
3570
3571 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3572 Register reg = as_Register($src1$$reg);
3573 int64_t val = $src2$$constant;
3574 if (val >= 0) {
3575 __ subs(zr, reg, val);
3576 } else if (val != -val) {
3577 __ adds(zr, reg, -val);
3578 } else {
3579 // aargh, Long.MIN_VALUE is a special case
3580 __ orr(rscratch1, zr, (uint64_t)val);
3581 __ subs(zr, reg, rscratch1);
3582 }
3583 %}
3584
3585 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3586 Register reg1 = as_Register($src1$$reg);
3587 uint64_t val = (uint64_t)$src2$$constant;
3588 __ mov(rscratch1, val);
3589 __ cmp(reg1, rscratch1);
3590 %}
3591
3592 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3593 Register reg1 = as_Register($src1$$reg);
3594 Register reg2 = as_Register($src2$$reg);
3595 __ cmp(reg1, reg2);
3596 %}
3597
3598 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3599 Register reg1 = as_Register($src1$$reg);
3600 Register reg2 = as_Register($src2$$reg);
3601 __ cmpw(reg1, reg2);
3602 %}
3603
3604 enc_class aarch64_enc_testp(iRegP src) %{
3605 Register reg = as_Register($src$$reg);
3606 __ cmp(reg, zr);
3607 %}
3608
3609 enc_class aarch64_enc_testn(iRegN src) %{
3610 Register reg = as_Register($src$$reg);
3611 __ cmpw(reg, zr);
3612 %}
3613
3614 enc_class aarch64_enc_b(label lbl) %{
3615 Label *L = $lbl$$label;
3616 __ b(*L);
3617 %}
3618
3619 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3620 Label *L = $lbl$$label;
3621 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3622 %}
3623
3624 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3625 Label *L = $lbl$$label;
3626 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3627 %}
3628
3629 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3630 %{
3631 Register sub_reg = as_Register($sub$$reg);
3632 Register super_reg = as_Register($super$$reg);
3633 Register temp_reg = as_Register($temp$$reg);
3634 Register result_reg = as_Register($result$$reg);
3635
3636 Label miss;
3637 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3638 nullptr, &miss,
3639 /*set_cond_codes:*/ true);
3640 if ($primary) {
3641 __ mov(result_reg, zr);
3642 }
3643 __ bind(miss);
3644 %}
3645
3646 enc_class aarch64_enc_java_static_call(method meth) %{
3647 address addr = (address)$meth$$method;
3648 address call;
3649 if (!_method) {
3650 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3651 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3652 if (call == nullptr) {
3653 ciEnv::current()->record_failure("CodeCache is full");
3654 return;
3655 }
3656 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3657 // The NOP here is purely to ensure that eliding a call to
3658 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3659 __ nop();
3660 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3661 } else {
3662 int method_index = resolved_method_index(masm);
3663 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3664 : static_call_Relocation::spec(method_index);
3665 call = __ trampoline_call(Address(addr, rspec));
3666 if (call == nullptr) {
3667 ciEnv::current()->record_failure("CodeCache is full");
3668 return;
3669 }
3670 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3671 // Calls of the same statically bound method can share
3672 // a stub to the interpreter.
3673 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3674 } else {
3675 // Emit stub for static call
3676 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3677 if (stub == nullptr) {
3678 ciEnv::current()->record_failure("CodeCache is full");
3679 return;
3680 }
3681 }
3682 }
3683
3684 __ post_call_nop();
3685
3686 // Only non uncommon_trap calls need to reinitialize ptrue.
3687 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3688 __ reinitialize_ptrue();
3689 }
3690 %}
3691
3692 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3693 int method_index = resolved_method_index(masm);
3694 address call = __ ic_call((address)$meth$$method, method_index);
3695 if (call == nullptr) {
3696 ciEnv::current()->record_failure("CodeCache is full");
3697 return;
3698 }
3699 __ post_call_nop();
3700 if (Compile::current()->max_vector_size() > 0) {
3701 __ reinitialize_ptrue();
3702 }
3703 %}
3704
3705 enc_class aarch64_enc_call_epilog() %{
3706 if (VerifyStackAtCalls) {
3707 // Check that stack depth is unchanged: find majik cookie on stack
3708 __ call_Unimplemented();
3709 }
3710 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
3711 // The last return value is not set by the callee but used to pass the null marker to compiled code.
3712 // Search for the corresponding projection, get the register and emit code that initialized it.
3713 uint con = (tf()->range_cc()->cnt() - 1);
3714 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3715 ProjNode* proj = fast_out(i)->as_Proj();
3716 if (proj->_con == con) {
3717 // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
3718 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3719 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3720 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3721 __ cmp(r0, zr);
3722 __ cset(toReg, Assembler::NE);
3723 if (reg->is_stack()) {
3724 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3725 __ str(toReg, Address(sp, st_off));
3726 }
3727 break;
3728 }
3729 }
3730 if (return_value_is_used()) {
3731 // An inline type is returned as fields in multiple registers.
3732 // R0 either contains an oop if the inline type is buffered or a pointer
3733 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3734 // if the lowest bit is set to allow C2 to use the oop after null checking.
3735 // r0 &= (r0 & 1) - 1
3736 __ andr(rscratch1, r0, 0x1);
3737 __ sub(rscratch1, rscratch1, 0x1);
3738 __ andr(r0, r0, rscratch1);
3739 }
3740 }
3741 %}
3742
3743 enc_class aarch64_enc_java_to_runtime(method meth) %{
3744 // some calls to generated routines (arraycopy code) are scheduled
3745 // by C2 as runtime calls. if so we can call them using a br (they
3746 // will be in a reachable segment) otherwise we have to use a blr
3747 // which loads the absolute address into a register.
3748 address entry = (address)$meth$$method;
3749 CodeBlob *cb = CodeCache::find_blob(entry);
3750 if (cb) {
3751 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3752 if (call == nullptr) {
3753 ciEnv::current()->record_failure("CodeCache is full");
3754 return;
3755 }
3756 __ post_call_nop();
3757 } else {
3758 Label retaddr;
3759 // Make the anchor frame walkable
3760 __ adr(rscratch2, retaddr);
3761 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3762 __ lea(rscratch1, RuntimeAddress(entry));
3763 __ blr(rscratch1);
3764 __ bind(retaddr);
3765 __ post_call_nop();
3766 }
3767 if (Compile::current()->max_vector_size() > 0) {
3768 __ reinitialize_ptrue();
3769 }
3770 %}
3771
3772 enc_class aarch64_enc_rethrow() %{
3773 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3774 %}
3775
3776 enc_class aarch64_enc_ret() %{
3777 #ifdef ASSERT
3778 if (Compile::current()->max_vector_size() > 0) {
3779 __ verify_ptrue();
3780 }
3781 #endif
3782 __ ret(lr);
3783 %}
3784
3785 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3786 Register target_reg = as_Register($jump_target$$reg);
3787 __ br(target_reg);
3788 %}
3789
3790 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3791 Register target_reg = as_Register($jump_target$$reg);
3792 // exception oop should be in r0
3793 // ret addr has been popped into lr
3794 // callee expects it in r3
3795 __ mov(r3, lr);
3796 __ br(target_reg);
3797 %}
3798
3799 %}
3800
3801 //----------FRAME--------------------------------------------------------------
3802 // Definition of frame structure and management information.
3803 //
3804 // S T A C K L A Y O U T Allocators stack-slot number
3805 // | (to get allocators register number
3806 // G Owned by | | v add OptoReg::stack0())
3807 // r CALLER | |
3808 // o | +--------+ pad to even-align allocators stack-slot
3809 // w V | pad0 | numbers; owned by CALLER
3810 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3811 // h ^ | in | 5
3812 // | | args | 4 Holes in incoming args owned by SELF
3813 // | | | | 3
3814 // | | +--------+
3815 // V | | old out| Empty on Intel, window on Sparc
3816 // | old |preserve| Must be even aligned.
3817 // | SP-+--------+----> Matcher::_old_SP, even aligned
3818 // | | in | 3 area for Intel ret address
3819 // Owned by |preserve| Empty on Sparc.
3820 // SELF +--------+
3821 // | | pad2 | 2 pad to align old SP
3822 // | +--------+ 1
3823 // | | locks | 0
3824 // | +--------+----> OptoReg::stack0(), even aligned
3825 // | | pad1 | 11 pad to align new SP
3826 // | +--------+
3827 // | | | 10
3828 // | | spills | 9 spills
3829 // V | | 8 (pad0 slot for callee)
3830 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3831 // ^ | out | 7
3832 // | | args | 6 Holes in outgoing args owned by CALLEE
3833 // Owned by +--------+
3834 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3835 // | new |preserve| Must be even-aligned.
3836 // | SP-+--------+----> Matcher::_new_SP, even aligned
3837 // | | |
3838 //
3839 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3840 // known from SELF's arguments and the Java calling convention.
3841 // Region 6-7 is determined per call site.
3842 // Note 2: If the calling convention leaves holes in the incoming argument
3843 // area, those holes are owned by SELF. Holes in the outgoing area
3844 // are owned by the CALLEE. Holes should not be necessary in the
3845 // incoming area, as the Java calling convention is completely under
3846 // the control of the AD file. Doubles can be sorted and packed to
3847 // avoid holes. Holes in the outgoing arguments may be necessary for
3848 // varargs C calling conventions.
3849 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3850 // even aligned with pad0 as needed.
3851 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3852 // (the latter is true on Intel but is it false on AArch64?)
3853 // region 6-11 is even aligned; it may be padded out more so that
3854 // the region from SP to FP meets the minimum stack alignment.
3855 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3856 // alignment. Region 11, pad1, may be dynamically extended so that
3857 // SP meets the minimum alignment.
3858
3859 frame %{
3860 // These three registers define part of the calling convention
3861 // between compiled code and the interpreter.
3862
3863 // Inline Cache Register or Method for I2C.
3864 inline_cache_reg(R12);
3865
3866 // Number of stack slots consumed by locking an object
3867 sync_stack_slots(2);
3868
3869 // Compiled code's Frame Pointer
3870 frame_pointer(R31);
3871
3872 // Interpreter stores its frame pointer in a register which is
3873 // stored to the stack by I2CAdaptors.
3874 // I2CAdaptors convert from interpreted java to compiled java.
3875 interpreter_frame_pointer(R29);
3876
3877 // Stack alignment requirement
3878 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3879
3880 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3881 // for calls to C. Supports the var-args backing area for register parms.
3882 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3883
3884 // The after-PROLOG location of the return address. Location of
3885 // return address specifies a type (REG or STACK) and a number
3886 // representing the register number (i.e. - use a register name) or
3887 // stack slot.
3888 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3889 // Otherwise, it is above the locks and verification slot and alignment word
3890 // TODO this may well be correct but need to check why that - 2 is there
3891 // ppc port uses 0 but we definitely need to allow for fixed_slots
3892 // which folds in the space used for monitors
3893 return_addr(STACK - 2 +
3894 align_up((Compile::current()->in_preserve_stack_slots() +
3895 Compile::current()->fixed_slots()),
3896 stack_alignment_in_slots()));
3897
3898 // Location of compiled Java return values. Same as C for now.
3899 return_value
3900 %{
3901 // TODO do we allow ideal_reg == Op_RegN???
3902 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3903 "only return normal values");
3904
3905 static const int lo[Op_RegL + 1] = { // enum name
3906 0, // Op_Node
3907 0, // Op_Set
3908 R0_num, // Op_RegN
3909 R0_num, // Op_RegI
3910 R0_num, // Op_RegP
3911 V0_num, // Op_RegF
3912 V0_num, // Op_RegD
3913 R0_num // Op_RegL
3914 };
3915
3916 static const int hi[Op_RegL + 1] = { // enum name
3917 0, // Op_Node
3918 0, // Op_Set
3919 OptoReg::Bad, // Op_RegN
3920 OptoReg::Bad, // Op_RegI
3921 R0_H_num, // Op_RegP
3922 OptoReg::Bad, // Op_RegF
3923 V0_H_num, // Op_RegD
3924 R0_H_num // Op_RegL
3925 };
3926
3927 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3928 %}
3929 %}
3930
3931 //----------ATTRIBUTES---------------------------------------------------------
3932 //----------Operand Attributes-------------------------------------------------
3933 op_attrib op_cost(1); // Required cost attribute
3934
3935 //----------Instruction Attributes---------------------------------------------
3936 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3937 ins_attrib ins_size(32); // Required size attribute (in bits)
3938 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3939 // a non-matching short branch variant
3940 // of some long branch?
3941 ins_attrib ins_alignment(4); // Required alignment attribute (must
3942 // be a power of 2) specifies the
3943 // alignment that some part of the
3944 // instruction (not necessarily the
3945 // start) requires. If > 1, a
3946 // compute_padding() function must be
3947 // provided for the instruction
3948
3949 //----------OPERANDS-----------------------------------------------------------
3950 // Operand definitions must precede instruction definitions for correct parsing
3951 // in the ADLC because operands constitute user defined types which are used in
3952 // instruction definitions.
3953
3954 //----------Simple Operands----------------------------------------------------
3955
3956 // Integer operands 32 bit
3957 // 32 bit immediate
3958 operand immI()
3959 %{
3960 match(ConI);
3961
3962 op_cost(0);
3963 format %{ %}
3964 interface(CONST_INTER);
3965 %}
3966
3967 // 32 bit zero
3968 operand immI0()
3969 %{
3970 predicate(n->get_int() == 0);
3971 match(ConI);
3972
3973 op_cost(0);
3974 format %{ %}
3975 interface(CONST_INTER);
3976 %}
3977
3978 // 32 bit unit increment
3979 operand immI_1()
3980 %{
3981 predicate(n->get_int() == 1);
3982 match(ConI);
3983
3984 op_cost(0);
3985 format %{ %}
3986 interface(CONST_INTER);
3987 %}
3988
3989 // 32 bit unit decrement
3990 operand immI_M1()
3991 %{
3992 predicate(n->get_int() == -1);
3993 match(ConI);
3994
3995 op_cost(0);
3996 format %{ %}
3997 interface(CONST_INTER);
3998 %}
3999
4000 // Shift values for add/sub extension shift
4001 operand immIExt()
4002 %{
4003 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4004 match(ConI);
4005
4006 op_cost(0);
4007 format %{ %}
4008 interface(CONST_INTER);
4009 %}
4010
4011 operand immI_gt_1()
4012 %{
4013 predicate(n->get_int() > 1);
4014 match(ConI);
4015
4016 op_cost(0);
4017 format %{ %}
4018 interface(CONST_INTER);
4019 %}
4020
4021 operand immI_le_4()
4022 %{
4023 predicate(n->get_int() <= 4);
4024 match(ConI);
4025
4026 op_cost(0);
4027 format %{ %}
4028 interface(CONST_INTER);
4029 %}
4030
4031 operand immI_16()
4032 %{
4033 predicate(n->get_int() == 16);
4034 match(ConI);
4035
4036 op_cost(0);
4037 format %{ %}
4038 interface(CONST_INTER);
4039 %}
4040
4041 operand immI_24()
4042 %{
4043 predicate(n->get_int() == 24);
4044 match(ConI);
4045
4046 op_cost(0);
4047 format %{ %}
4048 interface(CONST_INTER);
4049 %}
4050
4051 operand immI_32()
4052 %{
4053 predicate(n->get_int() == 32);
4054 match(ConI);
4055
4056 op_cost(0);
4057 format %{ %}
4058 interface(CONST_INTER);
4059 %}
4060
4061 operand immI_48()
4062 %{
4063 predicate(n->get_int() == 48);
4064 match(ConI);
4065
4066 op_cost(0);
4067 format %{ %}
4068 interface(CONST_INTER);
4069 %}
4070
4071 operand immI_56()
4072 %{
4073 predicate(n->get_int() == 56);
4074 match(ConI);
4075
4076 op_cost(0);
4077 format %{ %}
4078 interface(CONST_INTER);
4079 %}
4080
4081 operand immI_255()
4082 %{
4083 predicate(n->get_int() == 255);
4084 match(ConI);
4085
4086 op_cost(0);
4087 format %{ %}
4088 interface(CONST_INTER);
4089 %}
4090
4091 operand immI_65535()
4092 %{
4093 predicate(n->get_int() == 65535);
4094 match(ConI);
4095
4096 op_cost(0);
4097 format %{ %}
4098 interface(CONST_INTER);
4099 %}
4100
4101 operand immI_positive()
4102 %{
4103 predicate(n->get_int() > 0);
4104 match(ConI);
4105
4106 op_cost(0);
4107 format %{ %}
4108 interface(CONST_INTER);
4109 %}
4110
4111 // BoolTest condition for signed compare
4112 operand immI_cmp_cond()
4113 %{
4114 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4115 match(ConI);
4116
4117 op_cost(0);
4118 format %{ %}
4119 interface(CONST_INTER);
4120 %}
4121
4122 // BoolTest condition for unsigned compare
4123 operand immI_cmpU_cond()
4124 %{
4125 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4126 match(ConI);
4127
4128 op_cost(0);
4129 format %{ %}
4130 interface(CONST_INTER);
4131 %}
4132
4133 operand immL_255()
4134 %{
4135 predicate(n->get_long() == 255L);
4136 match(ConL);
4137
4138 op_cost(0);
4139 format %{ %}
4140 interface(CONST_INTER);
4141 %}
4142
4143 operand immL_65535()
4144 %{
4145 predicate(n->get_long() == 65535L);
4146 match(ConL);
4147
4148 op_cost(0);
4149 format %{ %}
4150 interface(CONST_INTER);
4151 %}
4152
4153 operand immL_4294967295()
4154 %{
4155 predicate(n->get_long() == 4294967295L);
4156 match(ConL);
4157
4158 op_cost(0);
4159 format %{ %}
4160 interface(CONST_INTER);
4161 %}
4162
4163 operand immL_bitmask()
4164 %{
4165 predicate((n->get_long() != 0)
4166 && ((n->get_long() & 0xc000000000000000l) == 0)
4167 && is_power_of_2(n->get_long() + 1));
4168 match(ConL);
4169
4170 op_cost(0);
4171 format %{ %}
4172 interface(CONST_INTER);
4173 %}
4174
4175 operand immI_bitmask()
4176 %{
4177 predicate((n->get_int() != 0)
4178 && ((n->get_int() & 0xc0000000) == 0)
4179 && is_power_of_2(n->get_int() + 1));
4180 match(ConI);
4181
4182 op_cost(0);
4183 format %{ %}
4184 interface(CONST_INTER);
4185 %}
4186
4187 operand immL_positive_bitmaskI()
4188 %{
4189 predicate((n->get_long() != 0)
4190 && ((julong)n->get_long() < 0x80000000ULL)
4191 && is_power_of_2(n->get_long() + 1));
4192 match(ConL);
4193
4194 op_cost(0);
4195 format %{ %}
4196 interface(CONST_INTER);
4197 %}
4198
4199 // Scale values for scaled offset addressing modes (up to long but not quad)
4200 operand immIScale()
4201 %{
4202 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4203 match(ConI);
4204
4205 op_cost(0);
4206 format %{ %}
4207 interface(CONST_INTER);
4208 %}
4209
4210 // 5 bit signed integer
4211 operand immI5()
4212 %{
4213 predicate(Assembler::is_simm(n->get_int(), 5));
4214 match(ConI);
4215
4216 op_cost(0);
4217 format %{ %}
4218 interface(CONST_INTER);
4219 %}
4220
4221 // 7 bit unsigned integer
4222 operand immIU7()
4223 %{
4224 predicate(Assembler::is_uimm(n->get_int(), 7));
4225 match(ConI);
4226
4227 op_cost(0);
4228 format %{ %}
4229 interface(CONST_INTER);
4230 %}
4231
4232 // Offset for scaled or unscaled immediate loads and stores
4233 operand immIOffset()
4234 %{
4235 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4236 match(ConI);
4237
4238 op_cost(0);
4239 format %{ %}
4240 interface(CONST_INTER);
4241 %}
4242
4243 operand immIOffset1()
4244 %{
4245 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4246 match(ConI);
4247
4248 op_cost(0);
4249 format %{ %}
4250 interface(CONST_INTER);
4251 %}
4252
4253 operand immIOffset2()
4254 %{
4255 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4256 match(ConI);
4257
4258 op_cost(0);
4259 format %{ %}
4260 interface(CONST_INTER);
4261 %}
4262
4263 operand immIOffset4()
4264 %{
4265 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4266 match(ConI);
4267
4268 op_cost(0);
4269 format %{ %}
4270 interface(CONST_INTER);
4271 %}
4272
4273 operand immIOffset8()
4274 %{
4275 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4276 match(ConI);
4277
4278 op_cost(0);
4279 format %{ %}
4280 interface(CONST_INTER);
4281 %}
4282
4283 operand immIOffset16()
4284 %{
4285 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4286 match(ConI);
4287
4288 op_cost(0);
4289 format %{ %}
4290 interface(CONST_INTER);
4291 %}
4292
4293 operand immLOffset()
4294 %{
4295 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4296 match(ConL);
4297
4298 op_cost(0);
4299 format %{ %}
4300 interface(CONST_INTER);
4301 %}
4302
4303 operand immLoffset1()
4304 %{
4305 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4306 match(ConL);
4307
4308 op_cost(0);
4309 format %{ %}
4310 interface(CONST_INTER);
4311 %}
4312
4313 operand immLoffset2()
4314 %{
4315 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4316 match(ConL);
4317
4318 op_cost(0);
4319 format %{ %}
4320 interface(CONST_INTER);
4321 %}
4322
4323 operand immLoffset4()
4324 %{
4325 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4326 match(ConL);
4327
4328 op_cost(0);
4329 format %{ %}
4330 interface(CONST_INTER);
4331 %}
4332
4333 operand immLoffset8()
4334 %{
4335 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4336 match(ConL);
4337
4338 op_cost(0);
4339 format %{ %}
4340 interface(CONST_INTER);
4341 %}
4342
4343 operand immLoffset16()
4344 %{
4345 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4346 match(ConL);
4347
4348 op_cost(0);
4349 format %{ %}
4350 interface(CONST_INTER);
4351 %}
4352
4353 // 5 bit signed long integer
4354 operand immL5()
4355 %{
4356 predicate(Assembler::is_simm(n->get_long(), 5));
4357 match(ConL);
4358
4359 op_cost(0);
4360 format %{ %}
4361 interface(CONST_INTER);
4362 %}
4363
4364 // 7 bit unsigned long integer
4365 operand immLU7()
4366 %{
4367 predicate(Assembler::is_uimm(n->get_long(), 7));
4368 match(ConL);
4369
4370 op_cost(0);
4371 format %{ %}
4372 interface(CONST_INTER);
4373 %}
4374
4375 // 8 bit signed value.
4376 operand immI8()
4377 %{
4378 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4379 match(ConI);
4380
4381 op_cost(0);
4382 format %{ %}
4383 interface(CONST_INTER);
4384 %}
4385
4386 // 8 bit signed value (simm8), or #simm8 LSL 8.
4387 operand immI8_shift8()
4388 %{
4389 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4390 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4391 match(ConI);
4392
4393 op_cost(0);
4394 format %{ %}
4395 interface(CONST_INTER);
4396 %}
4397
4398 // 8 bit signed value (simm8), or #simm8 LSL 8.
4399 operand immL8_shift8()
4400 %{
4401 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4402 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4403 match(ConL);
4404
4405 op_cost(0);
4406 format %{ %}
4407 interface(CONST_INTER);
4408 %}
4409
4410 // 8 bit integer valid for vector add sub immediate
4411 operand immBAddSubV()
4412 %{
4413 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4414 match(ConI);
4415
4416 op_cost(0);
4417 format %{ %}
4418 interface(CONST_INTER);
4419 %}
4420
4421 // 32 bit integer valid for add sub immediate
4422 operand immIAddSub()
4423 %{
4424 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4425 match(ConI);
4426 op_cost(0);
4427 format %{ %}
4428 interface(CONST_INTER);
4429 %}
4430
4431 // 32 bit integer valid for vector add sub immediate
4432 operand immIAddSubV()
4433 %{
4434 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4435 match(ConI);
4436
4437 op_cost(0);
4438 format %{ %}
4439 interface(CONST_INTER);
4440 %}
4441
4442 // 32 bit unsigned integer valid for logical immediate
4443
4444 operand immBLog()
4445 %{
4446 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4447 match(ConI);
4448
4449 op_cost(0);
4450 format %{ %}
4451 interface(CONST_INTER);
4452 %}
4453
4454 operand immSLog()
4455 %{
4456 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4457 match(ConI);
4458
4459 op_cost(0);
4460 format %{ %}
4461 interface(CONST_INTER);
4462 %}
4463
4464 operand immILog()
4465 %{
4466 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4467 match(ConI);
4468
4469 op_cost(0);
4470 format %{ %}
4471 interface(CONST_INTER);
4472 %}
4473
4474 // Integer operands 64 bit
4475 // 64 bit immediate
4476 operand immL()
4477 %{
4478 match(ConL);
4479
4480 op_cost(0);
4481 format %{ %}
4482 interface(CONST_INTER);
4483 %}
4484
4485 // 64 bit zero
4486 operand immL0()
4487 %{
4488 predicate(n->get_long() == 0);
4489 match(ConL);
4490
4491 op_cost(0);
4492 format %{ %}
4493 interface(CONST_INTER);
4494 %}
4495
4496 // 64 bit unit decrement
4497 operand immL_M1()
4498 %{
4499 predicate(n->get_long() == -1);
4500 match(ConL);
4501
4502 op_cost(0);
4503 format %{ %}
4504 interface(CONST_INTER);
4505 %}
4506
4507 // 64 bit integer valid for add sub immediate
4508 operand immLAddSub()
4509 %{
4510 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4511 match(ConL);
4512 op_cost(0);
4513 format %{ %}
4514 interface(CONST_INTER);
4515 %}
4516
4517 // 64 bit integer valid for addv subv immediate
4518 operand immLAddSubV()
4519 %{
4520 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4521 match(ConL);
4522
4523 op_cost(0);
4524 format %{ %}
4525 interface(CONST_INTER);
4526 %}
4527
4528 // 64 bit integer valid for logical immediate
4529 operand immLLog()
4530 %{
4531 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4532 match(ConL);
4533 op_cost(0);
4534 format %{ %}
4535 interface(CONST_INTER);
4536 %}
4537
4538 // Long Immediate: low 32-bit mask
4539 operand immL_32bits()
4540 %{
4541 predicate(n->get_long() == 0xFFFFFFFFL);
4542 match(ConL);
4543 op_cost(0);
4544 format %{ %}
4545 interface(CONST_INTER);
4546 %}
4547
4548 // Pointer operands
4549 // Pointer Immediate
4550 operand immP()
4551 %{
4552 match(ConP);
4553
4554 op_cost(0);
4555 format %{ %}
4556 interface(CONST_INTER);
4557 %}
4558
4559 // nullptr Pointer Immediate
4560 operand immP0()
4561 %{
4562 predicate(n->get_ptr() == 0);
4563 match(ConP);
4564
4565 op_cost(0);
4566 format %{ %}
4567 interface(CONST_INTER);
4568 %}
4569
4570 // Pointer Immediate One
4571 // this is used in object initialization (initial object header)
4572 operand immP_1()
4573 %{
4574 predicate(n->get_ptr() == 1);
4575 match(ConP);
4576
4577 op_cost(0);
4578 format %{ %}
4579 interface(CONST_INTER);
4580 %}
4581
4582 // Card Table Byte Map Base
4583 operand immByteMapBase()
4584 %{
4585 // Get base of card map
4586 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4587 SHENANDOAHGC_ONLY(!BarrierSet::barrier_set()->is_a(BarrierSet::ShenandoahBarrierSet) &&)
4588 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4589 match(ConP);
4590
4591 op_cost(0);
4592 format %{ %}
4593 interface(CONST_INTER);
4594 %}
4595
4596 // Float and Double operands
4597 // Double Immediate
4598 operand immD()
4599 %{
4600 match(ConD);
4601 op_cost(0);
4602 format %{ %}
4603 interface(CONST_INTER);
4604 %}
4605
4606 // Double Immediate: +0.0d
4607 operand immD0()
4608 %{
4609 predicate(jlong_cast(n->getd()) == 0);
4610 match(ConD);
4611
4612 op_cost(0);
4613 format %{ %}
4614 interface(CONST_INTER);
4615 %}
4616
4617 // constant 'double +0.0'.
4618 operand immDPacked()
4619 %{
4620 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4621 match(ConD);
4622 op_cost(0);
4623 format %{ %}
4624 interface(CONST_INTER);
4625 %}
4626
4627 // Float Immediate
4628 operand immF()
4629 %{
4630 match(ConF);
4631 op_cost(0);
4632 format %{ %}
4633 interface(CONST_INTER);
4634 %}
4635
4636 // Float Immediate: +0.0f.
4637 operand immF0()
4638 %{
4639 predicate(jint_cast(n->getf()) == 0);
4640 match(ConF);
4641
4642 op_cost(0);
4643 format %{ %}
4644 interface(CONST_INTER);
4645 %}
4646
4647 // Half Float (FP16) Immediate
4648 operand immH()
4649 %{
4650 match(ConH);
4651 op_cost(0);
4652 format %{ %}
4653 interface(CONST_INTER);
4654 %}
4655
4656 //
4657 operand immFPacked()
4658 %{
4659 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4660 match(ConF);
4661 op_cost(0);
4662 format %{ %}
4663 interface(CONST_INTER);
4664 %}
4665
4666 // Narrow pointer operands
4667 // Narrow Pointer Immediate
4668 operand immN()
4669 %{
4670 match(ConN);
4671
4672 op_cost(0);
4673 format %{ %}
4674 interface(CONST_INTER);
4675 %}
4676
4677 // Narrow nullptr Pointer Immediate
4678 operand immN0()
4679 %{
4680 predicate(n->get_narrowcon() == 0);
4681 match(ConN);
4682
4683 op_cost(0);
4684 format %{ %}
4685 interface(CONST_INTER);
4686 %}
4687
4688 operand immNKlass()
4689 %{
4690 match(ConNKlass);
4691
4692 op_cost(0);
4693 format %{ %}
4694 interface(CONST_INTER);
4695 %}
4696
4697 // Integer 32 bit Register Operands
4698 // Integer 32 bitRegister (excludes SP)
4699 operand iRegI()
4700 %{
4701 constraint(ALLOC_IN_RC(any_reg32));
4702 match(RegI);
4703 match(iRegINoSp);
4704 op_cost(0);
4705 format %{ %}
4706 interface(REG_INTER);
4707 %}
4708
4709 // Integer 32 bit Register not Special
4710 operand iRegINoSp()
4711 %{
4712 constraint(ALLOC_IN_RC(no_special_reg32));
4713 match(RegI);
4714 op_cost(0);
4715 format %{ %}
4716 interface(REG_INTER);
4717 %}
4718
4719 // Integer 64 bit Register Operands
4720 // Integer 64 bit Register (includes SP)
4721 operand iRegL()
4722 %{
4723 constraint(ALLOC_IN_RC(any_reg));
4724 match(RegL);
4725 match(iRegLNoSp);
4726 op_cost(0);
4727 format %{ %}
4728 interface(REG_INTER);
4729 %}
4730
4731 // Integer 64 bit Register not Special
4732 operand iRegLNoSp()
4733 %{
4734 constraint(ALLOC_IN_RC(no_special_reg));
4735 match(RegL);
4736 match(iRegL_R0);
4737 format %{ %}
4738 interface(REG_INTER);
4739 %}
4740
4741 // Pointer Register Operands
4742 // Pointer Register
4743 operand iRegP()
4744 %{
4745 constraint(ALLOC_IN_RC(ptr_reg));
4746 match(RegP);
4747 match(iRegPNoSp);
4748 match(iRegP_R0);
4749 //match(iRegP_R2);
4750 //match(iRegP_R4);
4751 match(iRegP_R5);
4752 match(thread_RegP);
4753 op_cost(0);
4754 format %{ %}
4755 interface(REG_INTER);
4756 %}
4757
4758 // Pointer 64 bit Register not Special
4759 operand iRegPNoSp()
4760 %{
4761 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4762 match(RegP);
4763 // match(iRegP);
4764 // match(iRegP_R0);
4765 // match(iRegP_R2);
4766 // match(iRegP_R4);
4767 // match(iRegP_R5);
4768 // match(thread_RegP);
4769 op_cost(0);
4770 format %{ %}
4771 interface(REG_INTER);
4772 %}
4773
4774 // This operand is not allowed to use rfp even if
4775 // rfp is not used to hold the frame pointer.
4776 operand iRegPNoSpNoRfp()
4777 %{
4778 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4779 match(RegP);
4780 match(iRegPNoSp);
4781 op_cost(0);
4782 format %{ %}
4783 interface(REG_INTER);
4784 %}
4785
4786 // Pointer 64 bit Register R0 only
4787 operand iRegP_R0()
4788 %{
4789 constraint(ALLOC_IN_RC(r0_reg));
4790 match(RegP);
4791 // match(iRegP);
4792 match(iRegPNoSp);
4793 op_cost(0);
4794 format %{ %}
4795 interface(REG_INTER);
4796 %}
4797
4798 // Pointer 64 bit Register R1 only
4799 operand iRegP_R1()
4800 %{
4801 constraint(ALLOC_IN_RC(r1_reg));
4802 match(RegP);
4803 // match(iRegP);
4804 match(iRegPNoSp);
4805 op_cost(0);
4806 format %{ %}
4807 interface(REG_INTER);
4808 %}
4809
4810 // Pointer 64 bit Register R2 only
4811 operand iRegP_R2()
4812 %{
4813 constraint(ALLOC_IN_RC(r2_reg));
4814 match(RegP);
4815 // match(iRegP);
4816 match(iRegPNoSp);
4817 op_cost(0);
4818 format %{ %}
4819 interface(REG_INTER);
4820 %}
4821
4822 // Pointer 64 bit Register R3 only
4823 operand iRegP_R3()
4824 %{
4825 constraint(ALLOC_IN_RC(r3_reg));
4826 match(RegP);
4827 // match(iRegP);
4828 match(iRegPNoSp);
4829 op_cost(0);
4830 format %{ %}
4831 interface(REG_INTER);
4832 %}
4833
4834 // Pointer 64 bit Register R4 only
4835 operand iRegP_R4()
4836 %{
4837 constraint(ALLOC_IN_RC(r4_reg));
4838 match(RegP);
4839 // match(iRegP);
4840 match(iRegPNoSp);
4841 op_cost(0);
4842 format %{ %}
4843 interface(REG_INTER);
4844 %}
4845
4846 // Pointer 64 bit Register R5 only
4847 operand iRegP_R5()
4848 %{
4849 constraint(ALLOC_IN_RC(r5_reg));
4850 match(RegP);
4851 // match(iRegP);
4852 match(iRegPNoSp);
4853 op_cost(0);
4854 format %{ %}
4855 interface(REG_INTER);
4856 %}
4857
4858 // Pointer 64 bit Register R10 only
4859 operand iRegP_R10()
4860 %{
4861 constraint(ALLOC_IN_RC(r10_reg));
4862 match(RegP);
4863 // match(iRegP);
4864 match(iRegPNoSp);
4865 op_cost(0);
4866 format %{ %}
4867 interface(REG_INTER);
4868 %}
4869
4870 // Long 64 bit Register R0 only
4871 operand iRegL_R0()
4872 %{
4873 constraint(ALLOC_IN_RC(r0_reg));
4874 match(RegL);
4875 match(iRegLNoSp);
4876 op_cost(0);
4877 format %{ %}
4878 interface(REG_INTER);
4879 %}
4880
4881 // Long 64 bit Register R11 only
4882 operand iRegL_R11()
4883 %{
4884 constraint(ALLOC_IN_RC(r11_reg));
4885 match(RegL);
4886 match(iRegLNoSp);
4887 op_cost(0);
4888 format %{ %}
4889 interface(REG_INTER);
4890 %}
4891
4892 // Register R0 only
4893 operand iRegI_R0()
4894 %{
4895 constraint(ALLOC_IN_RC(int_r0_reg));
4896 match(RegI);
4897 match(iRegINoSp);
4898 op_cost(0);
4899 format %{ %}
4900 interface(REG_INTER);
4901 %}
4902
4903 // Register R2 only
4904 operand iRegI_R2()
4905 %{
4906 constraint(ALLOC_IN_RC(int_r2_reg));
4907 match(RegI);
4908 match(iRegINoSp);
4909 op_cost(0);
4910 format %{ %}
4911 interface(REG_INTER);
4912 %}
4913
4914 // Register R3 only
4915 operand iRegI_R3()
4916 %{
4917 constraint(ALLOC_IN_RC(int_r3_reg));
4918 match(RegI);
4919 match(iRegINoSp);
4920 op_cost(0);
4921 format %{ %}
4922 interface(REG_INTER);
4923 %}
4924
4925
4926 // Register R4 only
4927 operand iRegI_R4()
4928 %{
4929 constraint(ALLOC_IN_RC(int_r4_reg));
4930 match(RegI);
4931 match(iRegINoSp);
4932 op_cost(0);
4933 format %{ %}
4934 interface(REG_INTER);
4935 %}
4936
4937
4938 // Pointer Register Operands
4939 // Narrow Pointer Register
4940 operand iRegN()
4941 %{
4942 constraint(ALLOC_IN_RC(any_reg32));
4943 match(RegN);
4944 match(iRegNNoSp);
4945 op_cost(0);
4946 format %{ %}
4947 interface(REG_INTER);
4948 %}
4949
4950 // Integer 64 bit Register not Special
4951 operand iRegNNoSp()
4952 %{
4953 constraint(ALLOC_IN_RC(no_special_reg32));
4954 match(RegN);
4955 op_cost(0);
4956 format %{ %}
4957 interface(REG_INTER);
4958 %}
4959
4960 // Float Register
4961 // Float register operands
4962 operand vRegF()
4963 %{
4964 constraint(ALLOC_IN_RC(float_reg));
4965 match(RegF);
4966
4967 op_cost(0);
4968 format %{ %}
4969 interface(REG_INTER);
4970 %}
4971
4972 // Double Register
4973 // Double register operands
4974 operand vRegD()
4975 %{
4976 constraint(ALLOC_IN_RC(double_reg));
4977 match(RegD);
4978
4979 op_cost(0);
4980 format %{ %}
4981 interface(REG_INTER);
4982 %}
4983
4984 // Generic vector class. This will be used for
4985 // all vector operands, including NEON and SVE.
4986 operand vReg()
4987 %{
4988 constraint(ALLOC_IN_RC(dynamic));
4989 match(VecA);
4990 match(VecD);
4991 match(VecX);
4992
4993 op_cost(0);
4994 format %{ %}
4995 interface(REG_INTER);
4996 %}
4997
4998 operand vecA()
4999 %{
5000 constraint(ALLOC_IN_RC(vectora_reg));
5001 match(VecA);
5002
5003 op_cost(0);
5004 format %{ %}
5005 interface(REG_INTER);
5006 %}
5007
5008 operand vecD()
5009 %{
5010 constraint(ALLOC_IN_RC(vectord_reg));
5011 match(VecD);
5012
5013 op_cost(0);
5014 format %{ %}
5015 interface(REG_INTER);
5016 %}
5017
5018 operand vecX()
5019 %{
5020 constraint(ALLOC_IN_RC(vectorx_reg));
5021 match(VecX);
5022
5023 op_cost(0);
5024 format %{ %}
5025 interface(REG_INTER);
5026 %}
5027
5028 operand vRegD_V0()
5029 %{
5030 constraint(ALLOC_IN_RC(v0_reg));
5031 match(RegD);
5032 op_cost(0);
5033 format %{ %}
5034 interface(REG_INTER);
5035 %}
5036
5037 operand vRegD_V1()
5038 %{
5039 constraint(ALLOC_IN_RC(v1_reg));
5040 match(RegD);
5041 op_cost(0);
5042 format %{ %}
5043 interface(REG_INTER);
5044 %}
5045
5046 operand vRegD_V2()
5047 %{
5048 constraint(ALLOC_IN_RC(v2_reg));
5049 match(RegD);
5050 op_cost(0);
5051 format %{ %}
5052 interface(REG_INTER);
5053 %}
5054
5055 operand vRegD_V3()
5056 %{
5057 constraint(ALLOC_IN_RC(v3_reg));
5058 match(RegD);
5059 op_cost(0);
5060 format %{ %}
5061 interface(REG_INTER);
5062 %}
5063
5064 operand vRegD_V4()
5065 %{
5066 constraint(ALLOC_IN_RC(v4_reg));
5067 match(RegD);
5068 op_cost(0);
5069 format %{ %}
5070 interface(REG_INTER);
5071 %}
5072
5073 operand vRegD_V5()
5074 %{
5075 constraint(ALLOC_IN_RC(v5_reg));
5076 match(RegD);
5077 op_cost(0);
5078 format %{ %}
5079 interface(REG_INTER);
5080 %}
5081
5082 operand vRegD_V6()
5083 %{
5084 constraint(ALLOC_IN_RC(v6_reg));
5085 match(RegD);
5086 op_cost(0);
5087 format %{ %}
5088 interface(REG_INTER);
5089 %}
5090
5091 operand vRegD_V7()
5092 %{
5093 constraint(ALLOC_IN_RC(v7_reg));
5094 match(RegD);
5095 op_cost(0);
5096 format %{ %}
5097 interface(REG_INTER);
5098 %}
5099
5100 operand vRegD_V12()
5101 %{
5102 constraint(ALLOC_IN_RC(v12_reg));
5103 match(RegD);
5104 op_cost(0);
5105 format %{ %}
5106 interface(REG_INTER);
5107 %}
5108
5109 operand vRegD_V13()
5110 %{
5111 constraint(ALLOC_IN_RC(v13_reg));
5112 match(RegD);
5113 op_cost(0);
5114 format %{ %}
5115 interface(REG_INTER);
5116 %}
5117
5118 operand pReg()
5119 %{
5120 constraint(ALLOC_IN_RC(pr_reg));
5121 match(RegVectMask);
5122 match(pRegGov);
5123 op_cost(0);
5124 format %{ %}
5125 interface(REG_INTER);
5126 %}
5127
5128 operand pRegGov()
5129 %{
5130 constraint(ALLOC_IN_RC(gov_pr));
5131 match(RegVectMask);
5132 match(pReg);
5133 op_cost(0);
5134 format %{ %}
5135 interface(REG_INTER);
5136 %}
5137
5138 operand pRegGov_P0()
5139 %{
5140 constraint(ALLOC_IN_RC(p0_reg));
5141 match(RegVectMask);
5142 op_cost(0);
5143 format %{ %}
5144 interface(REG_INTER);
5145 %}
5146
5147 operand pRegGov_P1()
5148 %{
5149 constraint(ALLOC_IN_RC(p1_reg));
5150 match(RegVectMask);
5151 op_cost(0);
5152 format %{ %}
5153 interface(REG_INTER);
5154 %}
5155
5156 // Flags register, used as output of signed compare instructions
5157
5158 // note that on AArch64 we also use this register as the output for
5159 // for floating point compare instructions (CmpF CmpD). this ensures
5160 // that ordered inequality tests use GT, GE, LT or LE none of which
5161 // pass through cases where the result is unordered i.e. one or both
5162 // inputs to the compare is a NaN. this means that the ideal code can
5163 // replace e.g. a GT with an LE and not end up capturing the NaN case
5164 // (where the comparison should always fail). EQ and NE tests are
5165 // always generated in ideal code so that unordered folds into the NE
5166 // case, matching the behaviour of AArch64 NE.
5167 //
5168 // This differs from x86 where the outputs of FP compares use a
5169 // special FP flags registers and where compares based on this
5170 // register are distinguished into ordered inequalities (cmpOpUCF) and
5171 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5172 // to explicitly handle the unordered case in branches. x86 also has
5173 // to include extra CMoveX rules to accept a cmpOpUCF input.
5174
5175 operand rFlagsReg()
5176 %{
5177 constraint(ALLOC_IN_RC(int_flags));
5178 match(RegFlags);
5179
5180 op_cost(0);
5181 format %{ "RFLAGS" %}
5182 interface(REG_INTER);
5183 %}
5184
5185 // Flags register, used as output of unsigned compare instructions
5186 operand rFlagsRegU()
5187 %{
5188 constraint(ALLOC_IN_RC(int_flags));
5189 match(RegFlags);
5190
5191 op_cost(0);
5192 format %{ "RFLAGSU" %}
5193 interface(REG_INTER);
5194 %}
5195
5196 // Special Registers
5197
5198 // Method Register
5199 operand inline_cache_RegP(iRegP reg)
5200 %{
5201 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5202 match(reg);
5203 match(iRegPNoSp);
5204 op_cost(0);
5205 format %{ %}
5206 interface(REG_INTER);
5207 %}
5208
5209 // Thread Register
5210 operand thread_RegP(iRegP reg)
5211 %{
5212 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5213 match(reg);
5214 op_cost(0);
5215 format %{ %}
5216 interface(REG_INTER);
5217 %}
5218
5219 //----------Memory Operands----------------------------------------------------
5220
5221 operand indirect(iRegP reg)
5222 %{
5223 constraint(ALLOC_IN_RC(ptr_reg));
5224 match(reg);
5225 op_cost(0);
5226 format %{ "[$reg]" %}
5227 interface(MEMORY_INTER) %{
5228 base($reg);
5229 index(0xffffffff);
5230 scale(0x0);
5231 disp(0x0);
5232 %}
5233 %}
5234
5235 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5236 %{
5237 constraint(ALLOC_IN_RC(ptr_reg));
5238 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5239 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5240 op_cost(0);
5241 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5242 interface(MEMORY_INTER) %{
5243 base($reg);
5244 index($ireg);
5245 scale($scale);
5246 disp(0x0);
5247 %}
5248 %}
5249
5250 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5251 %{
5252 constraint(ALLOC_IN_RC(ptr_reg));
5253 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5254 match(AddP reg (LShiftL lreg scale));
5255 op_cost(0);
5256 format %{ "$reg, $lreg lsl($scale)" %}
5257 interface(MEMORY_INTER) %{
5258 base($reg);
5259 index($lreg);
5260 scale($scale);
5261 disp(0x0);
5262 %}
5263 %}
5264
5265 operand indIndexI2L(iRegP reg, iRegI ireg)
5266 %{
5267 constraint(ALLOC_IN_RC(ptr_reg));
5268 match(AddP reg (ConvI2L ireg));
5269 op_cost(0);
5270 format %{ "$reg, $ireg, 0, I2L" %}
5271 interface(MEMORY_INTER) %{
5272 base($reg);
5273 index($ireg);
5274 scale(0x0);
5275 disp(0x0);
5276 %}
5277 %}
5278
5279 operand indIndex(iRegP reg, iRegL lreg)
5280 %{
5281 constraint(ALLOC_IN_RC(ptr_reg));
5282 match(AddP reg lreg);
5283 op_cost(0);
5284 format %{ "$reg, $lreg" %}
5285 interface(MEMORY_INTER) %{
5286 base($reg);
5287 index($lreg);
5288 scale(0x0);
5289 disp(0x0);
5290 %}
5291 %}
5292
5293 operand indOffI1(iRegP reg, immIOffset1 off)
5294 %{
5295 constraint(ALLOC_IN_RC(ptr_reg));
5296 match(AddP reg off);
5297 op_cost(0);
5298 format %{ "[$reg, $off]" %}
5299 interface(MEMORY_INTER) %{
5300 base($reg);
5301 index(0xffffffff);
5302 scale(0x0);
5303 disp($off);
5304 %}
5305 %}
5306
5307 operand indOffI2(iRegP reg, immIOffset2 off)
5308 %{
5309 constraint(ALLOC_IN_RC(ptr_reg));
5310 match(AddP reg off);
5311 op_cost(0);
5312 format %{ "[$reg, $off]" %}
5313 interface(MEMORY_INTER) %{
5314 base($reg);
5315 index(0xffffffff);
5316 scale(0x0);
5317 disp($off);
5318 %}
5319 %}
5320
5321 operand indOffI4(iRegP reg, immIOffset4 off)
5322 %{
5323 constraint(ALLOC_IN_RC(ptr_reg));
5324 match(AddP reg off);
5325 op_cost(0);
5326 format %{ "[$reg, $off]" %}
5327 interface(MEMORY_INTER) %{
5328 base($reg);
5329 index(0xffffffff);
5330 scale(0x0);
5331 disp($off);
5332 %}
5333 %}
5334
5335 operand indOffI8(iRegP reg, immIOffset8 off)
5336 %{
5337 constraint(ALLOC_IN_RC(ptr_reg));
5338 match(AddP reg off);
5339 op_cost(0);
5340 format %{ "[$reg, $off]" %}
5341 interface(MEMORY_INTER) %{
5342 base($reg);
5343 index(0xffffffff);
5344 scale(0x0);
5345 disp($off);
5346 %}
5347 %}
5348
5349 operand indOffI16(iRegP reg, immIOffset16 off)
5350 %{
5351 constraint(ALLOC_IN_RC(ptr_reg));
5352 match(AddP reg off);
5353 op_cost(0);
5354 format %{ "[$reg, $off]" %}
5355 interface(MEMORY_INTER) %{
5356 base($reg);
5357 index(0xffffffff);
5358 scale(0x0);
5359 disp($off);
5360 %}
5361 %}
5362
5363 operand indOffL1(iRegP reg, immLoffset1 off)
5364 %{
5365 constraint(ALLOC_IN_RC(ptr_reg));
5366 match(AddP reg off);
5367 op_cost(0);
5368 format %{ "[$reg, $off]" %}
5369 interface(MEMORY_INTER) %{
5370 base($reg);
5371 index(0xffffffff);
5372 scale(0x0);
5373 disp($off);
5374 %}
5375 %}
5376
5377 operand indOffL2(iRegP reg, immLoffset2 off)
5378 %{
5379 constraint(ALLOC_IN_RC(ptr_reg));
5380 match(AddP reg off);
5381 op_cost(0);
5382 format %{ "[$reg, $off]" %}
5383 interface(MEMORY_INTER) %{
5384 base($reg);
5385 index(0xffffffff);
5386 scale(0x0);
5387 disp($off);
5388 %}
5389 %}
5390
5391 operand indOffL4(iRegP reg, immLoffset4 off)
5392 %{
5393 constraint(ALLOC_IN_RC(ptr_reg));
5394 match(AddP reg off);
5395 op_cost(0);
5396 format %{ "[$reg, $off]" %}
5397 interface(MEMORY_INTER) %{
5398 base($reg);
5399 index(0xffffffff);
5400 scale(0x0);
5401 disp($off);
5402 %}
5403 %}
5404
5405 operand indOffL8(iRegP reg, immLoffset8 off)
5406 %{
5407 constraint(ALLOC_IN_RC(ptr_reg));
5408 match(AddP reg off);
5409 op_cost(0);
5410 format %{ "[$reg, $off]" %}
5411 interface(MEMORY_INTER) %{
5412 base($reg);
5413 index(0xffffffff);
5414 scale(0x0);
5415 disp($off);
5416 %}
5417 %}
5418
5419 operand indOffL16(iRegP reg, immLoffset16 off)
5420 %{
5421 constraint(ALLOC_IN_RC(ptr_reg));
5422 match(AddP reg off);
5423 op_cost(0);
5424 format %{ "[$reg, $off]" %}
5425 interface(MEMORY_INTER) %{
5426 base($reg);
5427 index(0xffffffff);
5428 scale(0x0);
5429 disp($off);
5430 %}
5431 %}
5432
5433 operand indirectX2P(iRegL reg)
5434 %{
5435 constraint(ALLOC_IN_RC(ptr_reg));
5436 match(CastX2P reg);
5437 op_cost(0);
5438 format %{ "[$reg]\t# long -> ptr" %}
5439 interface(MEMORY_INTER) %{
5440 base($reg);
5441 index(0xffffffff);
5442 scale(0x0);
5443 disp(0x0);
5444 %}
5445 %}
5446
5447 operand indOffX2P(iRegL reg, immLOffset off)
5448 %{
5449 constraint(ALLOC_IN_RC(ptr_reg));
5450 match(AddP (CastX2P reg) off);
5451 op_cost(0);
5452 format %{ "[$reg, $off]\t# long -> ptr" %}
5453 interface(MEMORY_INTER) %{
5454 base($reg);
5455 index(0xffffffff);
5456 scale(0x0);
5457 disp($off);
5458 %}
5459 %}
5460
5461 operand indirectN(iRegN reg)
5462 %{
5463 predicate(CompressedOops::shift() == 0);
5464 constraint(ALLOC_IN_RC(ptr_reg));
5465 match(DecodeN reg);
5466 op_cost(0);
5467 format %{ "[$reg]\t# narrow" %}
5468 interface(MEMORY_INTER) %{
5469 base($reg);
5470 index(0xffffffff);
5471 scale(0x0);
5472 disp(0x0);
5473 %}
5474 %}
5475
5476 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5477 %{
5478 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5479 constraint(ALLOC_IN_RC(ptr_reg));
5480 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5481 op_cost(0);
5482 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5483 interface(MEMORY_INTER) %{
5484 base($reg);
5485 index($ireg);
5486 scale($scale);
5487 disp(0x0);
5488 %}
5489 %}
5490
5491 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5492 %{
5493 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5494 constraint(ALLOC_IN_RC(ptr_reg));
5495 match(AddP (DecodeN reg) (LShiftL lreg scale));
5496 op_cost(0);
5497 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5498 interface(MEMORY_INTER) %{
5499 base($reg);
5500 index($lreg);
5501 scale($scale);
5502 disp(0x0);
5503 %}
5504 %}
5505
5506 operand indIndexI2LN(iRegN reg, iRegI ireg)
5507 %{
5508 predicate(CompressedOops::shift() == 0);
5509 constraint(ALLOC_IN_RC(ptr_reg));
5510 match(AddP (DecodeN reg) (ConvI2L ireg));
5511 op_cost(0);
5512 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5513 interface(MEMORY_INTER) %{
5514 base($reg);
5515 index($ireg);
5516 scale(0x0);
5517 disp(0x0);
5518 %}
5519 %}
5520
5521 operand indIndexN(iRegN reg, iRegL lreg)
5522 %{
5523 predicate(CompressedOops::shift() == 0);
5524 constraint(ALLOC_IN_RC(ptr_reg));
5525 match(AddP (DecodeN reg) lreg);
5526 op_cost(0);
5527 format %{ "$reg, $lreg\t# narrow" %}
5528 interface(MEMORY_INTER) %{
5529 base($reg);
5530 index($lreg);
5531 scale(0x0);
5532 disp(0x0);
5533 %}
5534 %}
5535
5536 operand indOffIN(iRegN reg, immIOffset off)
5537 %{
5538 predicate(CompressedOops::shift() == 0);
5539 constraint(ALLOC_IN_RC(ptr_reg));
5540 match(AddP (DecodeN reg) off);
5541 op_cost(0);
5542 format %{ "[$reg, $off]\t# narrow" %}
5543 interface(MEMORY_INTER) %{
5544 base($reg);
5545 index(0xffffffff);
5546 scale(0x0);
5547 disp($off);
5548 %}
5549 %}
5550
5551 operand indOffLN(iRegN reg, immLOffset off)
5552 %{
5553 predicate(CompressedOops::shift() == 0);
5554 constraint(ALLOC_IN_RC(ptr_reg));
5555 match(AddP (DecodeN reg) off);
5556 op_cost(0);
5557 format %{ "[$reg, $off]\t# narrow" %}
5558 interface(MEMORY_INTER) %{
5559 base($reg);
5560 index(0xffffffff);
5561 scale(0x0);
5562 disp($off);
5563 %}
5564 %}
5565
5566
5567 //----------Special Memory Operands--------------------------------------------
5568 // Stack Slot Operand - This operand is used for loading and storing temporary
5569 // values on the stack where a match requires a value to
5570 // flow through memory.
5571 operand stackSlotP(sRegP reg)
5572 %{
5573 constraint(ALLOC_IN_RC(stack_slots));
5574 op_cost(100);
5575 // No match rule because this operand is only generated in matching
5576 // match(RegP);
5577 format %{ "[$reg]" %}
5578 interface(MEMORY_INTER) %{
5579 base(0x1e); // RSP
5580 index(0x0); // No Index
5581 scale(0x0); // No Scale
5582 disp($reg); // Stack Offset
5583 %}
5584 %}
5585
5586 operand stackSlotI(sRegI reg)
5587 %{
5588 constraint(ALLOC_IN_RC(stack_slots));
5589 // No match rule because this operand is only generated in matching
5590 // match(RegI);
5591 format %{ "[$reg]" %}
5592 interface(MEMORY_INTER) %{
5593 base(0x1e); // RSP
5594 index(0x0); // No Index
5595 scale(0x0); // No Scale
5596 disp($reg); // Stack Offset
5597 %}
5598 %}
5599
5600 operand stackSlotF(sRegF reg)
5601 %{
5602 constraint(ALLOC_IN_RC(stack_slots));
5603 // No match rule because this operand is only generated in matching
5604 // match(RegF);
5605 format %{ "[$reg]" %}
5606 interface(MEMORY_INTER) %{
5607 base(0x1e); // RSP
5608 index(0x0); // No Index
5609 scale(0x0); // No Scale
5610 disp($reg); // Stack Offset
5611 %}
5612 %}
5613
5614 operand stackSlotD(sRegD reg)
5615 %{
5616 constraint(ALLOC_IN_RC(stack_slots));
5617 // No match rule because this operand is only generated in matching
5618 // match(RegD);
5619 format %{ "[$reg]" %}
5620 interface(MEMORY_INTER) %{
5621 base(0x1e); // RSP
5622 index(0x0); // No Index
5623 scale(0x0); // No Scale
5624 disp($reg); // Stack Offset
5625 %}
5626 %}
5627
5628 operand stackSlotL(sRegL reg)
5629 %{
5630 constraint(ALLOC_IN_RC(stack_slots));
5631 // No match rule because this operand is only generated in matching
5632 // match(RegL);
5633 format %{ "[$reg]" %}
5634 interface(MEMORY_INTER) %{
5635 base(0x1e); // RSP
5636 index(0x0); // No Index
5637 scale(0x0); // No Scale
5638 disp($reg); // Stack Offset
5639 %}
5640 %}
5641
5642 // Operands for expressing Control Flow
5643 // NOTE: Label is a predefined operand which should not be redefined in
5644 // the AD file. It is generically handled within the ADLC.
5645
5646 //----------Conditional Branch Operands----------------------------------------
5647 // Comparison Op - This is the operation of the comparison, and is limited to
5648 // the following set of codes:
5649 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5650 //
5651 // Other attributes of the comparison, such as unsignedness, are specified
5652 // by the comparison instruction that sets a condition code flags register.
5653 // That result is represented by a flags operand whose subtype is appropriate
5654 // to the unsignedness (etc.) of the comparison.
5655 //
5656 // Later, the instruction which matches both the Comparison Op (a Bool) and
5657 // the flags (produced by the Cmp) specifies the coding of the comparison op
5658 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5659
5660 // used for signed integral comparisons and fp comparisons
5661
5662 operand cmpOp()
5663 %{
5664 match(Bool);
5665
5666 format %{ "" %}
5667 interface(COND_INTER) %{
5668 equal(0x0, "eq");
5669 not_equal(0x1, "ne");
5670 less(0xb, "lt");
5671 greater_equal(0xa, "ge");
5672 less_equal(0xd, "le");
5673 greater(0xc, "gt");
5674 overflow(0x6, "vs");
5675 no_overflow(0x7, "vc");
5676 %}
5677 %}
5678
5679 // used for unsigned integral comparisons
5680
5681 operand cmpOpU()
5682 %{
5683 match(Bool);
5684
5685 format %{ "" %}
5686 interface(COND_INTER) %{
5687 equal(0x0, "eq");
5688 not_equal(0x1, "ne");
5689 less(0x3, "lo");
5690 greater_equal(0x2, "hs");
5691 less_equal(0x9, "ls");
5692 greater(0x8, "hi");
5693 overflow(0x6, "vs");
5694 no_overflow(0x7, "vc");
5695 %}
5696 %}
5697
5698 // used for certain integral comparisons which can be
5699 // converted to cbxx or tbxx instructions
5700
5701 operand cmpOpEqNe()
5702 %{
5703 match(Bool);
5704 op_cost(0);
5705 predicate(n->as_Bool()->_test._test == BoolTest::ne
5706 || n->as_Bool()->_test._test == BoolTest::eq);
5707
5708 format %{ "" %}
5709 interface(COND_INTER) %{
5710 equal(0x0, "eq");
5711 not_equal(0x1, "ne");
5712 less(0xb, "lt");
5713 greater_equal(0xa, "ge");
5714 less_equal(0xd, "le");
5715 greater(0xc, "gt");
5716 overflow(0x6, "vs");
5717 no_overflow(0x7, "vc");
5718 %}
5719 %}
5720
5721 // used for certain integral comparisons which can be
5722 // converted to cbxx or tbxx instructions
5723
5724 operand cmpOpLtGe()
5725 %{
5726 match(Bool);
5727 op_cost(0);
5728
5729 predicate(n->as_Bool()->_test._test == BoolTest::lt
5730 || n->as_Bool()->_test._test == BoolTest::ge);
5731
5732 format %{ "" %}
5733 interface(COND_INTER) %{
5734 equal(0x0, "eq");
5735 not_equal(0x1, "ne");
5736 less(0xb, "lt");
5737 greater_equal(0xa, "ge");
5738 less_equal(0xd, "le");
5739 greater(0xc, "gt");
5740 overflow(0x6, "vs");
5741 no_overflow(0x7, "vc");
5742 %}
5743 %}
5744
5745 // used for certain unsigned integral comparisons which can be
5746 // converted to cbxx or tbxx instructions
5747
5748 operand cmpOpUEqNeLeGt()
5749 %{
5750 match(Bool);
5751 op_cost(0);
5752
5753 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5754 n->as_Bool()->_test._test == BoolTest::ne ||
5755 n->as_Bool()->_test._test == BoolTest::le ||
5756 n->as_Bool()->_test._test == BoolTest::gt);
5757
5758 format %{ "" %}
5759 interface(COND_INTER) %{
5760 equal(0x0, "eq");
5761 not_equal(0x1, "ne");
5762 less(0x3, "lo");
5763 greater_equal(0x2, "hs");
5764 less_equal(0x9, "ls");
5765 greater(0x8, "hi");
5766 overflow(0x6, "vs");
5767 no_overflow(0x7, "vc");
5768 %}
5769 %}
5770
5771 // Special operand allowing long args to int ops to be truncated for free
5772
5773 operand iRegL2I(iRegL reg) %{
5774
5775 op_cost(0);
5776
5777 match(ConvL2I reg);
5778
5779 format %{ "l2i($reg)" %}
5780
5781 interface(REG_INTER)
5782 %}
5783
5784 operand iRegL2P(iRegL reg) %{
5785
5786 op_cost(0);
5787
5788 match(CastX2P reg);
5789
5790 format %{ "l2p($reg)" %}
5791
5792 interface(REG_INTER)
5793 %}
5794
5795 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5796 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5797 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5798 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5799
5800 //----------OPERAND CLASSES----------------------------------------------------
5801 // Operand Classes are groups of operands that are used as to simplify
5802 // instruction definitions by not requiring the AD writer to specify
5803 // separate instructions for every form of operand when the
5804 // instruction accepts multiple operand types with the same basic
5805 // encoding and format. The classic case of this is memory operands.
5806
5807 // memory is used to define read/write location for load/store
5808 // instruction defs. we can turn a memory op into an Address
5809
5810 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5811 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5812
5813 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5814 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5815
5816 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5817 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5818
5819 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5820 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5821
5822 // All of the memory operands. For the pipeline description.
5823 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5824 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5825 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5826
5827
5828 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5829 // operations. it allows the src to be either an iRegI or a (ConvL2I
5830 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5831 // can be elided because the 32-bit instruction will just employ the
5832 // lower 32 bits anyway.
5833 //
5834 // n.b. this does not elide all L2I conversions. if the truncated
5835 // value is consumed by more than one operation then the ConvL2I
5836 // cannot be bundled into the consuming nodes so an l2i gets planted
5837 // (actually a movw $dst $src) and the downstream instructions consume
5838 // the result of the l2i as an iRegI input. That's a shame since the
5839 // movw is actually redundant but its not too costly.
5840
5841 opclass iRegIorL2I(iRegI, iRegL2I);
5842 opclass iRegPorL2P(iRegP, iRegL2P);
5843
5844 //----------PIPELINE-----------------------------------------------------------
5845 // Rules which define the behavior of the target architectures pipeline.
5846
5847 // For specific pipelines, eg A53, define the stages of that pipeline
5848 //pipe_desc(ISS, EX1, EX2, WR);
5849 #define ISS S0
5850 #define EX1 S1
5851 #define EX2 S2
5852 #define WR S3
5853
5854 // Integer ALU reg operation
5855 pipeline %{
5856
5857 attributes %{
5858 // ARM instructions are of fixed length
5859 fixed_size_instructions; // Fixed size instructions TODO does
5860 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5861 // ARM instructions come in 32-bit word units
5862 instruction_unit_size = 4; // An instruction is 4 bytes long
5863 instruction_fetch_unit_size = 64; // The processor fetches one line
5864 instruction_fetch_units = 1; // of 64 bytes
5865
5866 // List of nop instructions
5867 nops( MachNop );
5868 %}
5869
5870 // We don't use an actual pipeline model so don't care about resources
5871 // or description. we do use pipeline classes to introduce fixed
5872 // latencies
5873
5874 //----------RESOURCES----------------------------------------------------------
5875 // Resources are the functional units available to the machine
5876
5877 resources( INS0, INS1, INS01 = INS0 | INS1,
5878 ALU0, ALU1, ALU = ALU0 | ALU1,
5879 MAC,
5880 DIV,
5881 BRANCH,
5882 LDST,
5883 NEON_FP);
5884
5885 //----------PIPELINE DESCRIPTION-----------------------------------------------
5886 // Pipeline Description specifies the stages in the machine's pipeline
5887
5888 // Define the pipeline as a generic 6 stage pipeline
5889 pipe_desc(S0, S1, S2, S3, S4, S5);
5890
5891 //----------PIPELINE CLASSES---------------------------------------------------
5892 // Pipeline Classes describe the stages in which input and output are
5893 // referenced by the hardware pipeline.
5894
5895 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5896 %{
5897 single_instruction;
5898 src1 : S1(read);
5899 src2 : S2(read);
5900 dst : S5(write);
5901 INS01 : ISS;
5902 NEON_FP : S5;
5903 %}
5904
5905 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5906 %{
5907 single_instruction;
5908 src1 : S1(read);
5909 src2 : S2(read);
5910 dst : S5(write);
5911 INS01 : ISS;
5912 NEON_FP : S5;
5913 %}
5914
5915 pipe_class fp_uop_s(vRegF dst, vRegF src)
5916 %{
5917 single_instruction;
5918 src : S1(read);
5919 dst : S5(write);
5920 INS01 : ISS;
5921 NEON_FP : S5;
5922 %}
5923
5924 pipe_class fp_uop_d(vRegD dst, vRegD src)
5925 %{
5926 single_instruction;
5927 src : S1(read);
5928 dst : S5(write);
5929 INS01 : ISS;
5930 NEON_FP : S5;
5931 %}
5932
5933 pipe_class fp_d2f(vRegF dst, vRegD src)
5934 %{
5935 single_instruction;
5936 src : S1(read);
5937 dst : S5(write);
5938 INS01 : ISS;
5939 NEON_FP : S5;
5940 %}
5941
5942 pipe_class fp_f2d(vRegD dst, vRegF src)
5943 %{
5944 single_instruction;
5945 src : S1(read);
5946 dst : S5(write);
5947 INS01 : ISS;
5948 NEON_FP : S5;
5949 %}
5950
5951 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5952 %{
5953 single_instruction;
5954 src : S1(read);
5955 dst : S5(write);
5956 INS01 : ISS;
5957 NEON_FP : S5;
5958 %}
5959
5960 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5961 %{
5962 single_instruction;
5963 src : S1(read);
5964 dst : S5(write);
5965 INS01 : ISS;
5966 NEON_FP : S5;
5967 %}
5968
5969 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5970 %{
5971 single_instruction;
5972 src : S1(read);
5973 dst : S5(write);
5974 INS01 : ISS;
5975 NEON_FP : S5;
5976 %}
5977
5978 pipe_class fp_l2f(vRegF dst, iRegL src)
5979 %{
5980 single_instruction;
5981 src : S1(read);
5982 dst : S5(write);
5983 INS01 : ISS;
5984 NEON_FP : S5;
5985 %}
5986
5987 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5988 %{
5989 single_instruction;
5990 src : S1(read);
5991 dst : S5(write);
5992 INS01 : ISS;
5993 NEON_FP : S5;
5994 %}
5995
5996 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5997 %{
5998 single_instruction;
5999 src : S1(read);
6000 dst : S5(write);
6001 INS01 : ISS;
6002 NEON_FP : S5;
6003 %}
6004
6005 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6006 %{
6007 single_instruction;
6008 src : S1(read);
6009 dst : S5(write);
6010 INS01 : ISS;
6011 NEON_FP : S5;
6012 %}
6013
6014 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6015 %{
6016 single_instruction;
6017 src : S1(read);
6018 dst : S5(write);
6019 INS01 : ISS;
6020 NEON_FP : S5;
6021 %}
6022
6023 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6024 %{
6025 single_instruction;
6026 src1 : S1(read);
6027 src2 : S2(read);
6028 dst : S5(write);
6029 INS0 : ISS;
6030 NEON_FP : S5;
6031 %}
6032
6033 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6034 %{
6035 single_instruction;
6036 src1 : S1(read);
6037 src2 : S2(read);
6038 dst : S5(write);
6039 INS0 : ISS;
6040 NEON_FP : S5;
6041 %}
6042
6043 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6044 %{
6045 single_instruction;
6046 cr : S1(read);
6047 src1 : S1(read);
6048 src2 : S1(read);
6049 dst : S3(write);
6050 INS01 : ISS;
6051 NEON_FP : S3;
6052 %}
6053
6054 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6055 %{
6056 single_instruction;
6057 cr : S1(read);
6058 src1 : S1(read);
6059 src2 : S1(read);
6060 dst : S3(write);
6061 INS01 : ISS;
6062 NEON_FP : S3;
6063 %}
6064
6065 pipe_class fp_imm_s(vRegF dst)
6066 %{
6067 single_instruction;
6068 dst : S3(write);
6069 INS01 : ISS;
6070 NEON_FP : S3;
6071 %}
6072
6073 pipe_class fp_imm_d(vRegD dst)
6074 %{
6075 single_instruction;
6076 dst : S3(write);
6077 INS01 : ISS;
6078 NEON_FP : S3;
6079 %}
6080
6081 pipe_class fp_load_constant_s(vRegF dst)
6082 %{
6083 single_instruction;
6084 dst : S4(write);
6085 INS01 : ISS;
6086 NEON_FP : S4;
6087 %}
6088
6089 pipe_class fp_load_constant_d(vRegD dst)
6090 %{
6091 single_instruction;
6092 dst : S4(write);
6093 INS01 : ISS;
6094 NEON_FP : S4;
6095 %}
6096
6097 //------- Integer ALU operations --------------------------
6098
6099 // Integer ALU reg-reg operation
6100 // Operands needed in EX1, result generated in EX2
6101 // Eg. ADD x0, x1, x2
6102 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6103 %{
6104 single_instruction;
6105 dst : EX2(write);
6106 src1 : EX1(read);
6107 src2 : EX1(read);
6108 INS01 : ISS; // Dual issue as instruction 0 or 1
6109 ALU : EX2;
6110 %}
6111
6112 // Integer ALU reg-reg operation with constant shift
6113 // Shifted register must be available in LATE_ISS instead of EX1
6114 // Eg. ADD x0, x1, x2, LSL #2
6115 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6116 %{
6117 single_instruction;
6118 dst : EX2(write);
6119 src1 : EX1(read);
6120 src2 : ISS(read);
6121 INS01 : ISS;
6122 ALU : EX2;
6123 %}
6124
6125 // Integer ALU reg operation with constant shift
6126 // Eg. LSL x0, x1, #shift
6127 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6128 %{
6129 single_instruction;
6130 dst : EX2(write);
6131 src1 : ISS(read);
6132 INS01 : ISS;
6133 ALU : EX2;
6134 %}
6135
6136 // Integer ALU reg-reg operation with variable shift
6137 // Both operands must be available in LATE_ISS instead of EX1
6138 // Result is available in EX1 instead of EX2
6139 // Eg. LSLV x0, x1, x2
6140 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6141 %{
6142 single_instruction;
6143 dst : EX1(write);
6144 src1 : ISS(read);
6145 src2 : ISS(read);
6146 INS01 : ISS;
6147 ALU : EX1;
6148 %}
6149
6150 // Integer ALU reg-reg operation with extract
6151 // As for _vshift above, but result generated in EX2
6152 // Eg. EXTR x0, x1, x2, #N
6153 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6154 %{
6155 single_instruction;
6156 dst : EX2(write);
6157 src1 : ISS(read);
6158 src2 : ISS(read);
6159 INS1 : ISS; // Can only dual issue as Instruction 1
6160 ALU : EX1;
6161 %}
6162
6163 // Integer ALU reg operation
6164 // Eg. NEG x0, x1
6165 pipe_class ialu_reg(iRegI dst, iRegI src)
6166 %{
6167 single_instruction;
6168 dst : EX2(write);
6169 src : EX1(read);
6170 INS01 : ISS;
6171 ALU : EX2;
6172 %}
6173
6174 // Integer ALU reg mmediate operation
6175 // Eg. ADD x0, x1, #N
6176 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6177 %{
6178 single_instruction;
6179 dst : EX2(write);
6180 src1 : EX1(read);
6181 INS01 : ISS;
6182 ALU : EX2;
6183 %}
6184
6185 // Integer ALU immediate operation (no source operands)
6186 // Eg. MOV x0, #N
6187 pipe_class ialu_imm(iRegI dst)
6188 %{
6189 single_instruction;
6190 dst : EX1(write);
6191 INS01 : ISS;
6192 ALU : EX1;
6193 %}
6194
6195 //------- Compare operation -------------------------------
6196
6197 // Compare reg-reg
6198 // Eg. CMP x0, x1
6199 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6200 %{
6201 single_instruction;
6202 // fixed_latency(16);
6203 cr : EX2(write);
6204 op1 : EX1(read);
6205 op2 : EX1(read);
6206 INS01 : ISS;
6207 ALU : EX2;
6208 %}
6209
6210 // Compare reg-reg
6211 // Eg. CMP x0, #N
6212 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6213 %{
6214 single_instruction;
6215 // fixed_latency(16);
6216 cr : EX2(write);
6217 op1 : EX1(read);
6218 INS01 : ISS;
6219 ALU : EX2;
6220 %}
6221
6222 //------- Conditional instructions ------------------------
6223
6224 // Conditional no operands
6225 // Eg. CSINC x0, zr, zr, <cond>
6226 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6227 %{
6228 single_instruction;
6229 cr : EX1(read);
6230 dst : EX2(write);
6231 INS01 : ISS;
6232 ALU : EX2;
6233 %}
6234
6235 // Conditional 2 operand
6236 // EG. CSEL X0, X1, X2, <cond>
6237 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6238 %{
6239 single_instruction;
6240 cr : EX1(read);
6241 src1 : EX1(read);
6242 src2 : EX1(read);
6243 dst : EX2(write);
6244 INS01 : ISS;
6245 ALU : EX2;
6246 %}
6247
6248 // Conditional 2 operand
6249 // EG. CSEL X0, X1, X2, <cond>
6250 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6251 %{
6252 single_instruction;
6253 cr : EX1(read);
6254 src : EX1(read);
6255 dst : EX2(write);
6256 INS01 : ISS;
6257 ALU : EX2;
6258 %}
6259
6260 //------- Multiply pipeline operations --------------------
6261
6262 // Multiply reg-reg
6263 // Eg. MUL w0, w1, w2
6264 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6265 %{
6266 single_instruction;
6267 dst : WR(write);
6268 src1 : ISS(read);
6269 src2 : ISS(read);
6270 INS01 : ISS;
6271 MAC : WR;
6272 %}
6273
6274 // Multiply accumulate
6275 // Eg. MADD w0, w1, w2, w3
6276 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6277 %{
6278 single_instruction;
6279 dst : WR(write);
6280 src1 : ISS(read);
6281 src2 : ISS(read);
6282 src3 : ISS(read);
6283 INS01 : ISS;
6284 MAC : WR;
6285 %}
6286
6287 // Eg. MUL w0, w1, w2
6288 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6289 %{
6290 single_instruction;
6291 fixed_latency(3); // Maximum latency for 64 bit mul
6292 dst : WR(write);
6293 src1 : ISS(read);
6294 src2 : ISS(read);
6295 INS01 : ISS;
6296 MAC : WR;
6297 %}
6298
6299 // Multiply accumulate
6300 // Eg. MADD w0, w1, w2, w3
6301 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6302 %{
6303 single_instruction;
6304 fixed_latency(3); // Maximum latency for 64 bit mul
6305 dst : WR(write);
6306 src1 : ISS(read);
6307 src2 : ISS(read);
6308 src3 : ISS(read);
6309 INS01 : ISS;
6310 MAC : WR;
6311 %}
6312
6313 //------- Divide pipeline operations --------------------
6314
6315 // Eg. SDIV w0, w1, w2
6316 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6317 %{
6318 single_instruction;
6319 fixed_latency(8); // Maximum latency for 32 bit divide
6320 dst : WR(write);
6321 src1 : ISS(read);
6322 src2 : ISS(read);
6323 INS0 : ISS; // Can only dual issue as instruction 0
6324 DIV : WR;
6325 %}
6326
6327 // Eg. SDIV x0, x1, x2
6328 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6329 %{
6330 single_instruction;
6331 fixed_latency(16); // Maximum latency for 64 bit divide
6332 dst : WR(write);
6333 src1 : ISS(read);
6334 src2 : ISS(read);
6335 INS0 : ISS; // Can only dual issue as instruction 0
6336 DIV : WR;
6337 %}
6338
6339 //------- Load pipeline operations ------------------------
6340
6341 // Load - prefetch
6342 // Eg. PFRM <mem>
6343 pipe_class iload_prefetch(memory mem)
6344 %{
6345 single_instruction;
6346 mem : ISS(read);
6347 INS01 : ISS;
6348 LDST : WR;
6349 %}
6350
6351 // Load - reg, mem
6352 // Eg. LDR x0, <mem>
6353 pipe_class iload_reg_mem(iRegI dst, memory mem)
6354 %{
6355 single_instruction;
6356 dst : WR(write);
6357 mem : ISS(read);
6358 INS01 : ISS;
6359 LDST : WR;
6360 %}
6361
6362 // Load - reg, reg
6363 // Eg. LDR x0, [sp, x1]
6364 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6365 %{
6366 single_instruction;
6367 dst : WR(write);
6368 src : ISS(read);
6369 INS01 : ISS;
6370 LDST : WR;
6371 %}
6372
6373 //------- Store pipeline operations -----------------------
6374
6375 // Store - zr, mem
6376 // Eg. STR zr, <mem>
6377 pipe_class istore_mem(memory mem)
6378 %{
6379 single_instruction;
6380 mem : ISS(read);
6381 INS01 : ISS;
6382 LDST : WR;
6383 %}
6384
6385 // Store - reg, mem
6386 // Eg. STR x0, <mem>
6387 pipe_class istore_reg_mem(iRegI src, memory mem)
6388 %{
6389 single_instruction;
6390 mem : ISS(read);
6391 src : EX2(read);
6392 INS01 : ISS;
6393 LDST : WR;
6394 %}
6395
6396 // Store - reg, reg
6397 // Eg. STR x0, [sp, x1]
6398 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6399 %{
6400 single_instruction;
6401 dst : ISS(read);
6402 src : EX2(read);
6403 INS01 : ISS;
6404 LDST : WR;
6405 %}
6406
6407 //------- Store pipeline operations -----------------------
6408
6409 // Branch
6410 pipe_class pipe_branch()
6411 %{
6412 single_instruction;
6413 INS01 : ISS;
6414 BRANCH : EX1;
6415 %}
6416
6417 // Conditional branch
6418 pipe_class pipe_branch_cond(rFlagsReg cr)
6419 %{
6420 single_instruction;
6421 cr : EX1(read);
6422 INS01 : ISS;
6423 BRANCH : EX1;
6424 %}
6425
6426 // Compare & Branch
6427 // EG. CBZ/CBNZ
6428 pipe_class pipe_cmp_branch(iRegI op1)
6429 %{
6430 single_instruction;
6431 op1 : EX1(read);
6432 INS01 : ISS;
6433 BRANCH : EX1;
6434 %}
6435
6436 //------- Synchronisation operations ----------------------
6437
6438 // Any operation requiring serialization.
6439 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6440 pipe_class pipe_serial()
6441 %{
6442 single_instruction;
6443 force_serialization;
6444 fixed_latency(16);
6445 INS01 : ISS(2); // Cannot dual issue with any other instruction
6446 LDST : WR;
6447 %}
6448
6449 // Generic big/slow expanded idiom - also serialized
6450 pipe_class pipe_slow()
6451 %{
6452 instruction_count(10);
6453 multiple_bundles;
6454 force_serialization;
6455 fixed_latency(16);
6456 INS01 : ISS(2); // Cannot dual issue with any other instruction
6457 LDST : WR;
6458 %}
6459
6460 // Empty pipeline class
6461 pipe_class pipe_class_empty()
6462 %{
6463 single_instruction;
6464 fixed_latency(0);
6465 %}
6466
6467 // Default pipeline class.
6468 pipe_class pipe_class_default()
6469 %{
6470 single_instruction;
6471 fixed_latency(2);
6472 %}
6473
6474 // Pipeline class for compares.
6475 pipe_class pipe_class_compare()
6476 %{
6477 single_instruction;
6478 fixed_latency(16);
6479 %}
6480
6481 // Pipeline class for memory operations.
6482 pipe_class pipe_class_memory()
6483 %{
6484 single_instruction;
6485 fixed_latency(16);
6486 %}
6487
6488 // Pipeline class for call.
6489 pipe_class pipe_class_call()
6490 %{
6491 single_instruction;
6492 fixed_latency(100);
6493 %}
6494
6495 // Define the class for the Nop node.
6496 define %{
6497 MachNop = pipe_class_empty;
6498 %}
6499
6500 %}
6501 //----------INSTRUCTIONS-------------------------------------------------------
6502 //
6503 // match -- States which machine-independent subtree may be replaced
6504 // by this instruction.
6505 // ins_cost -- The estimated cost of this instruction is used by instruction
6506 // selection to identify a minimum cost tree of machine
6507 // instructions that matches a tree of machine-independent
6508 // instructions.
6509 // format -- A string providing the disassembly for this instruction.
6510 // The value of an instruction's operand may be inserted
6511 // by referring to it with a '$' prefix.
6512 // opcode -- Three instruction opcodes may be provided. These are referred
6513 // to within an encode class as $primary, $secondary, and $tertiary
6514 // rrspectively. The primary opcode is commonly used to
6515 // indicate the type of machine instruction, while secondary
6516 // and tertiary are often used for prefix options or addressing
6517 // modes.
6518 // ins_encode -- A list of encode classes with parameters. The encode class
6519 // name must have been defined in an 'enc_class' specification
6520 // in the encode section of the architecture description.
6521
6522 // ============================================================================
6523 // Memory (Load/Store) Instructions
6524
6525 // Load Instructions
6526
6527 // Load Byte (8 bit signed)
6528 instruct loadB(iRegINoSp dst, memory1 mem)
6529 %{
6530 match(Set dst (LoadB mem));
6531 predicate(!needs_acquiring_load(n));
6532
6533 ins_cost(4 * INSN_COST);
6534 format %{ "ldrsbw $dst, $mem\t# byte" %}
6535
6536 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6537
6538 ins_pipe(iload_reg_mem);
6539 %}
6540
6541 // Load Byte (8 bit signed) into long
6542 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6543 %{
6544 match(Set dst (ConvI2L (LoadB mem)));
6545 predicate(!needs_acquiring_load(n->in(1)));
6546
6547 ins_cost(4 * INSN_COST);
6548 format %{ "ldrsb $dst, $mem\t# byte" %}
6549
6550 ins_encode(aarch64_enc_ldrsb(dst, mem));
6551
6552 ins_pipe(iload_reg_mem);
6553 %}
6554
6555 // Load Byte (8 bit unsigned)
6556 instruct loadUB(iRegINoSp dst, memory1 mem)
6557 %{
6558 match(Set dst (LoadUB mem));
6559 predicate(!needs_acquiring_load(n));
6560
6561 ins_cost(4 * INSN_COST);
6562 format %{ "ldrbw $dst, $mem\t# byte" %}
6563
6564 ins_encode(aarch64_enc_ldrb(dst, mem));
6565
6566 ins_pipe(iload_reg_mem);
6567 %}
6568
6569 // Load Byte (8 bit unsigned) into long
6570 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6571 %{
6572 match(Set dst (ConvI2L (LoadUB mem)));
6573 predicate(!needs_acquiring_load(n->in(1)));
6574
6575 ins_cost(4 * INSN_COST);
6576 format %{ "ldrb $dst, $mem\t# byte" %}
6577
6578 ins_encode(aarch64_enc_ldrb(dst, mem));
6579
6580 ins_pipe(iload_reg_mem);
6581 %}
6582
6583 // Load Short (16 bit signed)
6584 instruct loadS(iRegINoSp dst, memory2 mem)
6585 %{
6586 match(Set dst (LoadS mem));
6587 predicate(!needs_acquiring_load(n));
6588
6589 ins_cost(4 * INSN_COST);
6590 format %{ "ldrshw $dst, $mem\t# short" %}
6591
6592 ins_encode(aarch64_enc_ldrshw(dst, mem));
6593
6594 ins_pipe(iload_reg_mem);
6595 %}
6596
6597 // Load Short (16 bit signed) into long
6598 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6599 %{
6600 match(Set dst (ConvI2L (LoadS mem)));
6601 predicate(!needs_acquiring_load(n->in(1)));
6602
6603 ins_cost(4 * INSN_COST);
6604 format %{ "ldrsh $dst, $mem\t# short" %}
6605
6606 ins_encode(aarch64_enc_ldrsh(dst, mem));
6607
6608 ins_pipe(iload_reg_mem);
6609 %}
6610
6611 // Load Char (16 bit unsigned)
6612 instruct loadUS(iRegINoSp dst, memory2 mem)
6613 %{
6614 match(Set dst (LoadUS mem));
6615 predicate(!needs_acquiring_load(n));
6616
6617 ins_cost(4 * INSN_COST);
6618 format %{ "ldrh $dst, $mem\t# short" %}
6619
6620 ins_encode(aarch64_enc_ldrh(dst, mem));
6621
6622 ins_pipe(iload_reg_mem);
6623 %}
6624
6625 // Load Short/Char (16 bit unsigned) into long
6626 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6627 %{
6628 match(Set dst (ConvI2L (LoadUS mem)));
6629 predicate(!needs_acquiring_load(n->in(1)));
6630
6631 ins_cost(4 * INSN_COST);
6632 format %{ "ldrh $dst, $mem\t# short" %}
6633
6634 ins_encode(aarch64_enc_ldrh(dst, mem));
6635
6636 ins_pipe(iload_reg_mem);
6637 %}
6638
6639 // Load Integer (32 bit signed)
6640 instruct loadI(iRegINoSp dst, memory4 mem)
6641 %{
6642 match(Set dst (LoadI mem));
6643 predicate(!needs_acquiring_load(n));
6644
6645 ins_cost(4 * INSN_COST);
6646 format %{ "ldrw $dst, $mem\t# int" %}
6647
6648 ins_encode(aarch64_enc_ldrw(dst, mem));
6649
6650 ins_pipe(iload_reg_mem);
6651 %}
6652
6653 // Load Integer (32 bit signed) into long
6654 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6655 %{
6656 match(Set dst (ConvI2L (LoadI mem)));
6657 predicate(!needs_acquiring_load(n->in(1)));
6658
6659 ins_cost(4 * INSN_COST);
6660 format %{ "ldrsw $dst, $mem\t# int" %}
6661
6662 ins_encode(aarch64_enc_ldrsw(dst, mem));
6663
6664 ins_pipe(iload_reg_mem);
6665 %}
6666
6667 // Load Integer (32 bit unsigned) into long
6668 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6669 %{
6670 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6671 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6672
6673 ins_cost(4 * INSN_COST);
6674 format %{ "ldrw $dst, $mem\t# int" %}
6675
6676 ins_encode(aarch64_enc_ldrw(dst, mem));
6677
6678 ins_pipe(iload_reg_mem);
6679 %}
6680
6681 // Load Long (64 bit signed)
6682 instruct loadL(iRegLNoSp dst, memory8 mem)
6683 %{
6684 match(Set dst (LoadL mem));
6685 predicate(!needs_acquiring_load(n));
6686
6687 ins_cost(4 * INSN_COST);
6688 format %{ "ldr $dst, $mem\t# int" %}
6689
6690 ins_encode(aarch64_enc_ldr(dst, mem));
6691
6692 ins_pipe(iload_reg_mem);
6693 %}
6694
6695 // Load Range
6696 instruct loadRange(iRegINoSp dst, memory4 mem)
6697 %{
6698 match(Set dst (LoadRange mem));
6699
6700 ins_cost(4 * INSN_COST);
6701 format %{ "ldrw $dst, $mem\t# range" %}
6702
6703 ins_encode(aarch64_enc_ldrw(dst, mem));
6704
6705 ins_pipe(iload_reg_mem);
6706 %}
6707
6708 // Load Pointer
6709 instruct loadP(iRegPNoSp dst, memory8 mem)
6710 %{
6711 match(Set dst (LoadP mem));
6712 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6713
6714 ins_cost(4 * INSN_COST);
6715 format %{ "ldr $dst, $mem\t# ptr" %}
6716
6717 ins_encode(aarch64_enc_ldr(dst, mem));
6718
6719 ins_pipe(iload_reg_mem);
6720 %}
6721
6722 // Load Compressed Pointer
6723 instruct loadN(iRegNNoSp dst, memory4 mem)
6724 %{
6725 match(Set dst (LoadN mem));
6726 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6727
6728 ins_cost(4 * INSN_COST);
6729 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6730
6731 ins_encode(aarch64_enc_ldrw(dst, mem));
6732
6733 ins_pipe(iload_reg_mem);
6734 %}
6735
6736 // Load Klass Pointer
6737 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6738 %{
6739 match(Set dst (LoadKlass mem));
6740 predicate(!needs_acquiring_load(n));
6741
6742 ins_cost(4 * INSN_COST);
6743 format %{ "ldr $dst, $mem\t# class" %}
6744
6745 ins_encode(aarch64_enc_ldr(dst, mem));
6746
6747 ins_pipe(iload_reg_mem);
6748 %}
6749
6750 // Load Narrow Klass Pointer
6751 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6752 %{
6753 match(Set dst (LoadNKlass mem));
6754 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6755
6756 ins_cost(4 * INSN_COST);
6757 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6758
6759 ins_encode(aarch64_enc_ldrw(dst, mem));
6760
6761 ins_pipe(iload_reg_mem);
6762 %}
6763
6764 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6765 %{
6766 match(Set dst (LoadNKlass mem));
6767 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6768
6769 ins_cost(4 * INSN_COST);
6770 format %{
6771 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6772 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6773 %}
6774 ins_encode %{
6775 // inlined aarch64_enc_ldrw
6776 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6777 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6778 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6779 %}
6780 ins_pipe(iload_reg_mem);
6781 %}
6782
6783 // Load Float
6784 instruct loadF(vRegF dst, memory4 mem)
6785 %{
6786 match(Set dst (LoadF mem));
6787 predicate(!needs_acquiring_load(n));
6788
6789 ins_cost(4 * INSN_COST);
6790 format %{ "ldrs $dst, $mem\t# float" %}
6791
6792 ins_encode( aarch64_enc_ldrs(dst, mem) );
6793
6794 ins_pipe(pipe_class_memory);
6795 %}
6796
6797 // Load Double
6798 instruct loadD(vRegD dst, memory8 mem)
6799 %{
6800 match(Set dst (LoadD mem));
6801 predicate(!needs_acquiring_load(n));
6802
6803 ins_cost(4 * INSN_COST);
6804 format %{ "ldrd $dst, $mem\t# double" %}
6805
6806 ins_encode( aarch64_enc_ldrd(dst, mem) );
6807
6808 ins_pipe(pipe_class_memory);
6809 %}
6810
6811
6812 // Load Int Constant
6813 instruct loadConI(iRegINoSp dst, immI src)
6814 %{
6815 match(Set dst src);
6816
6817 ins_cost(INSN_COST);
6818 format %{ "mov $dst, $src\t# int" %}
6819
6820 ins_encode( aarch64_enc_movw_imm(dst, src) );
6821
6822 ins_pipe(ialu_imm);
6823 %}
6824
6825 // Load Long Constant
6826 instruct loadConL(iRegLNoSp dst, immL src)
6827 %{
6828 match(Set dst src);
6829
6830 ins_cost(INSN_COST);
6831 format %{ "mov $dst, $src\t# long" %}
6832
6833 ins_encode( aarch64_enc_mov_imm(dst, src) );
6834
6835 ins_pipe(ialu_imm);
6836 %}
6837
6838 // Load Pointer Constant
6839
6840 instruct loadConP(iRegPNoSp dst, immP con)
6841 %{
6842 match(Set dst con);
6843
6844 ins_cost(INSN_COST * 4);
6845 format %{
6846 "mov $dst, $con\t# ptr"
6847 %}
6848
6849 ins_encode(aarch64_enc_mov_p(dst, con));
6850
6851 ins_pipe(ialu_imm);
6852 %}
6853
6854 // Load Null Pointer Constant
6855
6856 instruct loadConP0(iRegPNoSp dst, immP0 con)
6857 %{
6858 match(Set dst con);
6859
6860 ins_cost(INSN_COST);
6861 format %{ "mov $dst, $con\t# nullptr ptr" %}
6862
6863 ins_encode(aarch64_enc_mov_p0(dst, con));
6864
6865 ins_pipe(ialu_imm);
6866 %}
6867
6868 // Load Pointer Constant One
6869
6870 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6871 %{
6872 match(Set dst con);
6873
6874 ins_cost(INSN_COST);
6875 format %{ "mov $dst, $con\t# nullptr ptr" %}
6876
6877 ins_encode(aarch64_enc_mov_p1(dst, con));
6878
6879 ins_pipe(ialu_imm);
6880 %}
6881
6882 // Load Byte Map Base Constant
6883
6884 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6885 %{
6886 match(Set dst con);
6887
6888 ins_cost(INSN_COST);
6889 format %{ "adr $dst, $con\t# Byte Map Base" %}
6890
6891 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6892
6893 ins_pipe(ialu_imm);
6894 %}
6895
6896 // Load Narrow Pointer Constant
6897
6898 instruct loadConN(iRegNNoSp dst, immN con)
6899 %{
6900 match(Set dst con);
6901
6902 ins_cost(INSN_COST * 4);
6903 format %{ "mov $dst, $con\t# compressed ptr" %}
6904
6905 ins_encode(aarch64_enc_mov_n(dst, con));
6906
6907 ins_pipe(ialu_imm);
6908 %}
6909
6910 // Load Narrow Null Pointer Constant
6911
6912 instruct loadConN0(iRegNNoSp dst, immN0 con)
6913 %{
6914 match(Set dst con);
6915
6916 ins_cost(INSN_COST);
6917 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6918
6919 ins_encode(aarch64_enc_mov_n0(dst, con));
6920
6921 ins_pipe(ialu_imm);
6922 %}
6923
6924 // Load Narrow Klass Constant
6925
6926 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6927 %{
6928 match(Set dst con);
6929
6930 ins_cost(INSN_COST);
6931 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6932
6933 ins_encode(aarch64_enc_mov_nk(dst, con));
6934
6935 ins_pipe(ialu_imm);
6936 %}
6937
6938 // Load Packed Float Constant
6939
6940 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6941 match(Set dst con);
6942 ins_cost(INSN_COST * 4);
6943 format %{ "fmovs $dst, $con"%}
6944 ins_encode %{
6945 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6946 %}
6947
6948 ins_pipe(fp_imm_s);
6949 %}
6950
6951 // Load Float Constant
6952
6953 instruct loadConF(vRegF dst, immF con) %{
6954 match(Set dst con);
6955
6956 ins_cost(INSN_COST * 4);
6957
6958 format %{
6959 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6960 %}
6961
6962 ins_encode %{
6963 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6964 %}
6965
6966 ins_pipe(fp_load_constant_s);
6967 %}
6968
6969 // Load Packed Double Constant
6970
6971 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6972 match(Set dst con);
6973 ins_cost(INSN_COST);
6974 format %{ "fmovd $dst, $con"%}
6975 ins_encode %{
6976 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6977 %}
6978
6979 ins_pipe(fp_imm_d);
6980 %}
6981
6982 // Load Double Constant
6983
6984 instruct loadConD(vRegD dst, immD con) %{
6985 match(Set dst con);
6986
6987 ins_cost(INSN_COST * 5);
6988 format %{
6989 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6990 %}
6991
6992 ins_encode %{
6993 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6994 %}
6995
6996 ins_pipe(fp_load_constant_d);
6997 %}
6998
6999 // Load Half Float Constant
7000 // The "ldr" instruction loads a 32-bit word from the constant pool into a
7001 // 32-bit register but only the bottom half will be populated and the top
7002 // 16 bits are zero.
7003 instruct loadConH(vRegF dst, immH con) %{
7004 match(Set dst con);
7005 format %{
7006 "ldrs $dst, [$constantaddress]\t# load from constant table: half float=$con\n\t"
7007 %}
7008 ins_encode %{
7009 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7010 %}
7011 ins_pipe(fp_load_constant_s);
7012 %}
7013
7014 // Store Instructions
7015
7016 // Store Byte
7017 instruct storeB(iRegIorL2I src, memory1 mem)
7018 %{
7019 match(Set mem (StoreB mem src));
7020 predicate(!needs_releasing_store(n));
7021
7022 ins_cost(INSN_COST);
7023 format %{ "strb $src, $mem\t# byte" %}
7024
7025 ins_encode(aarch64_enc_strb(src, mem));
7026
7027 ins_pipe(istore_reg_mem);
7028 %}
7029
7030
7031 instruct storeimmB0(immI0 zero, memory1 mem)
7032 %{
7033 match(Set mem (StoreB mem zero));
7034 predicate(!needs_releasing_store(n));
7035
7036 ins_cost(INSN_COST);
7037 format %{ "strb rscractch2, $mem\t# byte" %}
7038
7039 ins_encode(aarch64_enc_strb0(mem));
7040
7041 ins_pipe(istore_mem);
7042 %}
7043
7044 // Store Char/Short
7045 instruct storeC(iRegIorL2I src, memory2 mem)
7046 %{
7047 match(Set mem (StoreC mem src));
7048 predicate(!needs_releasing_store(n));
7049
7050 ins_cost(INSN_COST);
7051 format %{ "strh $src, $mem\t# short" %}
7052
7053 ins_encode(aarch64_enc_strh(src, mem));
7054
7055 ins_pipe(istore_reg_mem);
7056 %}
7057
7058 instruct storeimmC0(immI0 zero, memory2 mem)
7059 %{
7060 match(Set mem (StoreC mem zero));
7061 predicate(!needs_releasing_store(n));
7062
7063 ins_cost(INSN_COST);
7064 format %{ "strh zr, $mem\t# short" %}
7065
7066 ins_encode(aarch64_enc_strh0(mem));
7067
7068 ins_pipe(istore_mem);
7069 %}
7070
7071 // Store Integer
7072
7073 instruct storeI(iRegIorL2I src, memory4 mem)
7074 %{
7075 match(Set mem(StoreI mem src));
7076 predicate(!needs_releasing_store(n));
7077
7078 ins_cost(INSN_COST);
7079 format %{ "strw $src, $mem\t# int" %}
7080
7081 ins_encode(aarch64_enc_strw(src, mem));
7082
7083 ins_pipe(istore_reg_mem);
7084 %}
7085
7086 instruct storeimmI0(immI0 zero, memory4 mem)
7087 %{
7088 match(Set mem(StoreI mem zero));
7089 predicate(!needs_releasing_store(n));
7090
7091 ins_cost(INSN_COST);
7092 format %{ "strw zr, $mem\t# int" %}
7093
7094 ins_encode(aarch64_enc_strw0(mem));
7095
7096 ins_pipe(istore_mem);
7097 %}
7098
7099 // Store Long (64 bit signed)
7100 instruct storeL(iRegL src, memory8 mem)
7101 %{
7102 match(Set mem (StoreL mem src));
7103 predicate(!needs_releasing_store(n));
7104
7105 ins_cost(INSN_COST);
7106 format %{ "str $src, $mem\t# int" %}
7107
7108 ins_encode(aarch64_enc_str(src, mem));
7109
7110 ins_pipe(istore_reg_mem);
7111 %}
7112
7113 // Store Long (64 bit signed)
7114 instruct storeimmL0(immL0 zero, memory8 mem)
7115 %{
7116 match(Set mem (StoreL mem zero));
7117 predicate(!needs_releasing_store(n));
7118
7119 ins_cost(INSN_COST);
7120 format %{ "str zr, $mem\t# int" %}
7121
7122 ins_encode(aarch64_enc_str0(mem));
7123
7124 ins_pipe(istore_mem);
7125 %}
7126
7127 // Store Pointer
7128 instruct storeP(iRegP src, memory8 mem)
7129 %{
7130 match(Set mem (StoreP mem src));
7131 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7132
7133 ins_cost(INSN_COST);
7134 format %{ "str $src, $mem\t# ptr" %}
7135
7136 ins_encode(aarch64_enc_str(src, mem));
7137
7138 ins_pipe(istore_reg_mem);
7139 %}
7140
7141 // Store Pointer
7142 instruct storeimmP0(immP0 zero, memory8 mem)
7143 %{
7144 match(Set mem (StoreP mem zero));
7145 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7146
7147 ins_cost(INSN_COST);
7148 format %{ "str zr, $mem\t# ptr" %}
7149
7150 ins_encode(aarch64_enc_str0(mem));
7151
7152 ins_pipe(istore_mem);
7153 %}
7154
7155 // Store Compressed Pointer
7156 instruct storeN(iRegN src, memory4 mem)
7157 %{
7158 match(Set mem (StoreN mem src));
7159 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7160
7161 ins_cost(INSN_COST);
7162 format %{ "strw $src, $mem\t# compressed ptr" %}
7163
7164 ins_encode(aarch64_enc_strw(src, mem));
7165
7166 ins_pipe(istore_reg_mem);
7167 %}
7168
7169 instruct storeImmN0(immN0 zero, memory4 mem)
7170 %{
7171 match(Set mem (StoreN mem zero));
7172 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7173
7174 ins_cost(INSN_COST);
7175 format %{ "strw zr, $mem\t# compressed ptr" %}
7176
7177 ins_encode(aarch64_enc_strw0(mem));
7178
7179 ins_pipe(istore_mem);
7180 %}
7181
7182 // Store Float
7183 instruct storeF(vRegF src, memory4 mem)
7184 %{
7185 match(Set mem (StoreF mem src));
7186 predicate(!needs_releasing_store(n));
7187
7188 ins_cost(INSN_COST);
7189 format %{ "strs $src, $mem\t# float" %}
7190
7191 ins_encode( aarch64_enc_strs(src, mem) );
7192
7193 ins_pipe(pipe_class_memory);
7194 %}
7195
7196 // TODO
7197 // implement storeImmF0 and storeFImmPacked
7198
7199 // Store Double
7200 instruct storeD(vRegD src, memory8 mem)
7201 %{
7202 match(Set mem (StoreD mem src));
7203 predicate(!needs_releasing_store(n));
7204
7205 ins_cost(INSN_COST);
7206 format %{ "strd $src, $mem\t# double" %}
7207
7208 ins_encode( aarch64_enc_strd(src, mem) );
7209
7210 ins_pipe(pipe_class_memory);
7211 %}
7212
7213 // Store Compressed Klass Pointer
7214 instruct storeNKlass(iRegN src, memory4 mem)
7215 %{
7216 predicate(!needs_releasing_store(n));
7217 match(Set mem (StoreNKlass mem src));
7218
7219 ins_cost(INSN_COST);
7220 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7221
7222 ins_encode(aarch64_enc_strw(src, mem));
7223
7224 ins_pipe(istore_reg_mem);
7225 %}
7226
7227 // TODO
7228 // implement storeImmD0 and storeDImmPacked
7229
7230 // prefetch instructions
7231 // Must be safe to execute with invalid address (cannot fault).
7232
7233 instruct prefetchalloc( memory8 mem ) %{
7234 match(PrefetchAllocation mem);
7235
7236 ins_cost(INSN_COST);
7237 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7238
7239 ins_encode( aarch64_enc_prefetchw(mem) );
7240
7241 ins_pipe(iload_prefetch);
7242 %}
7243
7244 // ---------------- volatile loads and stores ----------------
7245
7246 // Load Byte (8 bit signed)
7247 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7248 %{
7249 match(Set dst (LoadB mem));
7250
7251 ins_cost(VOLATILE_REF_COST);
7252 format %{ "ldarsb $dst, $mem\t# byte" %}
7253
7254 ins_encode(aarch64_enc_ldarsb(dst, mem));
7255
7256 ins_pipe(pipe_serial);
7257 %}
7258
7259 // Load Byte (8 bit signed) into long
7260 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7261 %{
7262 match(Set dst (ConvI2L (LoadB mem)));
7263
7264 ins_cost(VOLATILE_REF_COST);
7265 format %{ "ldarsb $dst, $mem\t# byte" %}
7266
7267 ins_encode(aarch64_enc_ldarsb(dst, mem));
7268
7269 ins_pipe(pipe_serial);
7270 %}
7271
7272 // Load Byte (8 bit unsigned)
7273 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7274 %{
7275 match(Set dst (LoadUB mem));
7276
7277 ins_cost(VOLATILE_REF_COST);
7278 format %{ "ldarb $dst, $mem\t# byte" %}
7279
7280 ins_encode(aarch64_enc_ldarb(dst, mem));
7281
7282 ins_pipe(pipe_serial);
7283 %}
7284
7285 // Load Byte (8 bit unsigned) into long
7286 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7287 %{
7288 match(Set dst (ConvI2L (LoadUB mem)));
7289
7290 ins_cost(VOLATILE_REF_COST);
7291 format %{ "ldarb $dst, $mem\t# byte" %}
7292
7293 ins_encode(aarch64_enc_ldarb(dst, mem));
7294
7295 ins_pipe(pipe_serial);
7296 %}
7297
7298 // Load Short (16 bit signed)
7299 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7300 %{
7301 match(Set dst (LoadS mem));
7302
7303 ins_cost(VOLATILE_REF_COST);
7304 format %{ "ldarshw $dst, $mem\t# short" %}
7305
7306 ins_encode(aarch64_enc_ldarshw(dst, mem));
7307
7308 ins_pipe(pipe_serial);
7309 %}
7310
7311 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7312 %{
7313 match(Set dst (LoadUS mem));
7314
7315 ins_cost(VOLATILE_REF_COST);
7316 format %{ "ldarhw $dst, $mem\t# short" %}
7317
7318 ins_encode(aarch64_enc_ldarhw(dst, mem));
7319
7320 ins_pipe(pipe_serial);
7321 %}
7322
7323 // Load Short/Char (16 bit unsigned) into long
7324 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7325 %{
7326 match(Set dst (ConvI2L (LoadUS mem)));
7327
7328 ins_cost(VOLATILE_REF_COST);
7329 format %{ "ldarh $dst, $mem\t# short" %}
7330
7331 ins_encode(aarch64_enc_ldarh(dst, mem));
7332
7333 ins_pipe(pipe_serial);
7334 %}
7335
7336 // Load Short/Char (16 bit signed) into long
7337 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7338 %{
7339 match(Set dst (ConvI2L (LoadS mem)));
7340
7341 ins_cost(VOLATILE_REF_COST);
7342 format %{ "ldarh $dst, $mem\t# short" %}
7343
7344 ins_encode(aarch64_enc_ldarsh(dst, mem));
7345
7346 ins_pipe(pipe_serial);
7347 %}
7348
7349 // Load Integer (32 bit signed)
7350 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7351 %{
7352 match(Set dst (LoadI mem));
7353
7354 ins_cost(VOLATILE_REF_COST);
7355 format %{ "ldarw $dst, $mem\t# int" %}
7356
7357 ins_encode(aarch64_enc_ldarw(dst, mem));
7358
7359 ins_pipe(pipe_serial);
7360 %}
7361
7362 // Load Integer (32 bit unsigned) into long
7363 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7364 %{
7365 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7366
7367 ins_cost(VOLATILE_REF_COST);
7368 format %{ "ldarw $dst, $mem\t# int" %}
7369
7370 ins_encode(aarch64_enc_ldarw(dst, mem));
7371
7372 ins_pipe(pipe_serial);
7373 %}
7374
7375 // Load Long (64 bit signed)
7376 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7377 %{
7378 match(Set dst (LoadL mem));
7379
7380 ins_cost(VOLATILE_REF_COST);
7381 format %{ "ldar $dst, $mem\t# int" %}
7382
7383 ins_encode(aarch64_enc_ldar(dst, mem));
7384
7385 ins_pipe(pipe_serial);
7386 %}
7387
7388 // Load Pointer
7389 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7390 %{
7391 match(Set dst (LoadP mem));
7392 predicate(n->as_Load()->barrier_data() == 0);
7393
7394 ins_cost(VOLATILE_REF_COST);
7395 format %{ "ldar $dst, $mem\t# ptr" %}
7396
7397 ins_encode(aarch64_enc_ldar(dst, mem));
7398
7399 ins_pipe(pipe_serial);
7400 %}
7401
7402 // Load Compressed Pointer
7403 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7404 %{
7405 match(Set dst (LoadN mem));
7406 predicate(n->as_Load()->barrier_data() == 0);
7407
7408 ins_cost(VOLATILE_REF_COST);
7409 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7410
7411 ins_encode(aarch64_enc_ldarw(dst, mem));
7412
7413 ins_pipe(pipe_serial);
7414 %}
7415
7416 // Load Float
7417 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7418 %{
7419 match(Set dst (LoadF mem));
7420
7421 ins_cost(VOLATILE_REF_COST);
7422 format %{ "ldars $dst, $mem\t# float" %}
7423
7424 ins_encode( aarch64_enc_fldars(dst, mem) );
7425
7426 ins_pipe(pipe_serial);
7427 %}
7428
7429 // Load Double
7430 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7431 %{
7432 match(Set dst (LoadD mem));
7433
7434 ins_cost(VOLATILE_REF_COST);
7435 format %{ "ldard $dst, $mem\t# double" %}
7436
7437 ins_encode( aarch64_enc_fldard(dst, mem) );
7438
7439 ins_pipe(pipe_serial);
7440 %}
7441
7442 // Store Byte
7443 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7444 %{
7445 match(Set mem (StoreB mem src));
7446
7447 ins_cost(VOLATILE_REF_COST);
7448 format %{ "stlrb $src, $mem\t# byte" %}
7449
7450 ins_encode(aarch64_enc_stlrb(src, mem));
7451
7452 ins_pipe(pipe_class_memory);
7453 %}
7454
7455 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7456 %{
7457 match(Set mem (StoreB mem zero));
7458
7459 ins_cost(VOLATILE_REF_COST);
7460 format %{ "stlrb zr, $mem\t# byte" %}
7461
7462 ins_encode(aarch64_enc_stlrb0(mem));
7463
7464 ins_pipe(pipe_class_memory);
7465 %}
7466
7467 // Store Char/Short
7468 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7469 %{
7470 match(Set mem (StoreC mem src));
7471
7472 ins_cost(VOLATILE_REF_COST);
7473 format %{ "stlrh $src, $mem\t# short" %}
7474
7475 ins_encode(aarch64_enc_stlrh(src, mem));
7476
7477 ins_pipe(pipe_class_memory);
7478 %}
7479
7480 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7481 %{
7482 match(Set mem (StoreC mem zero));
7483
7484 ins_cost(VOLATILE_REF_COST);
7485 format %{ "stlrh zr, $mem\t# short" %}
7486
7487 ins_encode(aarch64_enc_stlrh0(mem));
7488
7489 ins_pipe(pipe_class_memory);
7490 %}
7491
7492 // Store Integer
7493
7494 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7495 %{
7496 match(Set mem(StoreI mem src));
7497
7498 ins_cost(VOLATILE_REF_COST);
7499 format %{ "stlrw $src, $mem\t# int" %}
7500
7501 ins_encode(aarch64_enc_stlrw(src, mem));
7502
7503 ins_pipe(pipe_class_memory);
7504 %}
7505
7506 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7507 %{
7508 match(Set mem(StoreI mem zero));
7509
7510 ins_cost(VOLATILE_REF_COST);
7511 format %{ "stlrw zr, $mem\t# int" %}
7512
7513 ins_encode(aarch64_enc_stlrw0(mem));
7514
7515 ins_pipe(pipe_class_memory);
7516 %}
7517
7518 // Store Long (64 bit signed)
7519 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7520 %{
7521 match(Set mem (StoreL mem src));
7522
7523 ins_cost(VOLATILE_REF_COST);
7524 format %{ "stlr $src, $mem\t# int" %}
7525
7526 ins_encode(aarch64_enc_stlr(src, mem));
7527
7528 ins_pipe(pipe_class_memory);
7529 %}
7530
7531 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7532 %{
7533 match(Set mem (StoreL mem zero));
7534
7535 ins_cost(VOLATILE_REF_COST);
7536 format %{ "stlr zr, $mem\t# int" %}
7537
7538 ins_encode(aarch64_enc_stlr0(mem));
7539
7540 ins_pipe(pipe_class_memory);
7541 %}
7542
7543 // Store Pointer
7544 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7545 %{
7546 match(Set mem (StoreP mem src));
7547 predicate(n->as_Store()->barrier_data() == 0);
7548
7549 ins_cost(VOLATILE_REF_COST);
7550 format %{ "stlr $src, $mem\t# ptr" %}
7551
7552 ins_encode(aarch64_enc_stlr(src, mem));
7553
7554 ins_pipe(pipe_class_memory);
7555 %}
7556
7557 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7558 %{
7559 match(Set mem (StoreP mem zero));
7560 predicate(n->as_Store()->barrier_data() == 0);
7561
7562 ins_cost(VOLATILE_REF_COST);
7563 format %{ "stlr zr, $mem\t# ptr" %}
7564
7565 ins_encode(aarch64_enc_stlr0(mem));
7566
7567 ins_pipe(pipe_class_memory);
7568 %}
7569
7570 // Store Compressed Pointer
7571 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7572 %{
7573 match(Set mem (StoreN mem src));
7574 predicate(n->as_Store()->barrier_data() == 0);
7575
7576 ins_cost(VOLATILE_REF_COST);
7577 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7578
7579 ins_encode(aarch64_enc_stlrw(src, mem));
7580
7581 ins_pipe(pipe_class_memory);
7582 %}
7583
7584 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7585 %{
7586 match(Set mem (StoreN mem zero));
7587 predicate(n->as_Store()->barrier_data() == 0);
7588
7589 ins_cost(VOLATILE_REF_COST);
7590 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7591
7592 ins_encode(aarch64_enc_stlrw0(mem));
7593
7594 ins_pipe(pipe_class_memory);
7595 %}
7596
7597 // Store Float
7598 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7599 %{
7600 match(Set mem (StoreF mem src));
7601
7602 ins_cost(VOLATILE_REF_COST);
7603 format %{ "stlrs $src, $mem\t# float" %}
7604
7605 ins_encode( aarch64_enc_fstlrs(src, mem) );
7606
7607 ins_pipe(pipe_class_memory);
7608 %}
7609
7610 // TODO
7611 // implement storeImmF0 and storeFImmPacked
7612
7613 // Store Double
7614 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7615 %{
7616 match(Set mem (StoreD mem src));
7617
7618 ins_cost(VOLATILE_REF_COST);
7619 format %{ "stlrd $src, $mem\t# double" %}
7620
7621 ins_encode( aarch64_enc_fstlrd(src, mem) );
7622
7623 ins_pipe(pipe_class_memory);
7624 %}
7625
7626 // ---------------- end of volatile loads and stores ----------------
7627
7628 instruct cacheWB(indirect addr)
7629 %{
7630 predicate(VM_Version::supports_data_cache_line_flush());
7631 match(CacheWB addr);
7632
7633 ins_cost(100);
7634 format %{"cache wb $addr" %}
7635 ins_encode %{
7636 assert($addr->index_position() < 0, "should be");
7637 assert($addr$$disp == 0, "should be");
7638 __ cache_wb(Address($addr$$base$$Register, 0));
7639 %}
7640 ins_pipe(pipe_slow); // XXX
7641 %}
7642
7643 instruct cacheWBPreSync()
7644 %{
7645 predicate(VM_Version::supports_data_cache_line_flush());
7646 match(CacheWBPreSync);
7647
7648 ins_cost(100);
7649 format %{"cache wb presync" %}
7650 ins_encode %{
7651 __ cache_wbsync(true);
7652 %}
7653 ins_pipe(pipe_slow); // XXX
7654 %}
7655
7656 instruct cacheWBPostSync()
7657 %{
7658 predicate(VM_Version::supports_data_cache_line_flush());
7659 match(CacheWBPostSync);
7660
7661 ins_cost(100);
7662 format %{"cache wb postsync" %}
7663 ins_encode %{
7664 __ cache_wbsync(false);
7665 %}
7666 ins_pipe(pipe_slow); // XXX
7667 %}
7668
7669 // ============================================================================
7670 // BSWAP Instructions
7671
7672 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7673 match(Set dst (ReverseBytesI src));
7674
7675 ins_cost(INSN_COST);
7676 format %{ "revw $dst, $src" %}
7677
7678 ins_encode %{
7679 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7680 %}
7681
7682 ins_pipe(ialu_reg);
7683 %}
7684
7685 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7686 match(Set dst (ReverseBytesL src));
7687
7688 ins_cost(INSN_COST);
7689 format %{ "rev $dst, $src" %}
7690
7691 ins_encode %{
7692 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7693 %}
7694
7695 ins_pipe(ialu_reg);
7696 %}
7697
7698 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7699 match(Set dst (ReverseBytesUS src));
7700
7701 ins_cost(INSN_COST);
7702 format %{ "rev16w $dst, $src" %}
7703
7704 ins_encode %{
7705 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7706 %}
7707
7708 ins_pipe(ialu_reg);
7709 %}
7710
7711 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7712 match(Set dst (ReverseBytesS src));
7713
7714 ins_cost(INSN_COST);
7715 format %{ "rev16w $dst, $src\n\t"
7716 "sbfmw $dst, $dst, #0, #15" %}
7717
7718 ins_encode %{
7719 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7720 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7721 %}
7722
7723 ins_pipe(ialu_reg);
7724 %}
7725
7726 // ============================================================================
7727 // Zero Count Instructions
7728
7729 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7730 match(Set dst (CountLeadingZerosI src));
7731
7732 ins_cost(INSN_COST);
7733 format %{ "clzw $dst, $src" %}
7734 ins_encode %{
7735 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7736 %}
7737
7738 ins_pipe(ialu_reg);
7739 %}
7740
7741 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7742 match(Set dst (CountLeadingZerosL src));
7743
7744 ins_cost(INSN_COST);
7745 format %{ "clz $dst, $src" %}
7746 ins_encode %{
7747 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7748 %}
7749
7750 ins_pipe(ialu_reg);
7751 %}
7752
7753 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7754 match(Set dst (CountTrailingZerosI src));
7755
7756 ins_cost(INSN_COST * 2);
7757 format %{ "rbitw $dst, $src\n\t"
7758 "clzw $dst, $dst" %}
7759 ins_encode %{
7760 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7761 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7762 %}
7763
7764 ins_pipe(ialu_reg);
7765 %}
7766
7767 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7768 match(Set dst (CountTrailingZerosL src));
7769
7770 ins_cost(INSN_COST * 2);
7771 format %{ "rbit $dst, $src\n\t"
7772 "clz $dst, $dst" %}
7773 ins_encode %{
7774 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7775 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7776 %}
7777
7778 ins_pipe(ialu_reg);
7779 %}
7780
7781 //---------- Population Count Instructions -------------------------------------
7782 //
7783
7784 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7785 match(Set dst (PopCountI src));
7786 effect(TEMP tmp);
7787 ins_cost(INSN_COST * 13);
7788
7789 format %{ "movw $src, $src\n\t"
7790 "mov $tmp, $src\t# vector (1D)\n\t"
7791 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7792 "addv $tmp, $tmp\t# vector (8B)\n\t"
7793 "mov $dst, $tmp\t# vector (1D)" %}
7794 ins_encode %{
7795 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7796 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7797 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7798 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7799 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7800 %}
7801
7802 ins_pipe(pipe_class_default);
7803 %}
7804
7805 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7806 match(Set dst (PopCountI (LoadI mem)));
7807 effect(TEMP tmp);
7808 ins_cost(INSN_COST * 13);
7809
7810 format %{ "ldrs $tmp, $mem\n\t"
7811 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7812 "addv $tmp, $tmp\t# vector (8B)\n\t"
7813 "mov $dst, $tmp\t# vector (1D)" %}
7814 ins_encode %{
7815 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7816 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7817 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7818 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7819 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7820 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7821 %}
7822
7823 ins_pipe(pipe_class_default);
7824 %}
7825
7826 // Note: Long.bitCount(long) returns an int.
7827 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7828 match(Set dst (PopCountL src));
7829 effect(TEMP tmp);
7830 ins_cost(INSN_COST * 13);
7831
7832 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7833 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7834 "addv $tmp, $tmp\t# vector (8B)\n\t"
7835 "mov $dst, $tmp\t# vector (1D)" %}
7836 ins_encode %{
7837 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7838 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7839 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7840 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7841 %}
7842
7843 ins_pipe(pipe_class_default);
7844 %}
7845
7846 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7847 match(Set dst (PopCountL (LoadL mem)));
7848 effect(TEMP tmp);
7849 ins_cost(INSN_COST * 13);
7850
7851 format %{ "ldrd $tmp, $mem\n\t"
7852 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7853 "addv $tmp, $tmp\t# vector (8B)\n\t"
7854 "mov $dst, $tmp\t# vector (1D)" %}
7855 ins_encode %{
7856 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7857 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7858 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7859 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7860 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7861 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7862 %}
7863
7864 ins_pipe(pipe_class_default);
7865 %}
7866
7867 // ============================================================================
7868 // VerifyVectorAlignment Instruction
7869
7870 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7871 match(Set addr (VerifyVectorAlignment addr mask));
7872 effect(KILL cr);
7873 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7874 ins_encode %{
7875 Label Lskip;
7876 // check if masked bits of addr are zero
7877 __ tst($addr$$Register, $mask$$constant);
7878 __ br(Assembler::EQ, Lskip);
7879 __ stop("verify_vector_alignment found a misaligned vector memory access");
7880 __ bind(Lskip);
7881 %}
7882 ins_pipe(pipe_slow);
7883 %}
7884
7885 // ============================================================================
7886 // MemBar Instruction
7887
7888 instruct load_fence() %{
7889 match(LoadFence);
7890 ins_cost(VOLATILE_REF_COST);
7891
7892 format %{ "load_fence" %}
7893
7894 ins_encode %{
7895 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7896 %}
7897 ins_pipe(pipe_serial);
7898 %}
7899
7900 instruct unnecessary_membar_acquire() %{
7901 predicate(unnecessary_acquire(n));
7902 match(MemBarAcquire);
7903 ins_cost(0);
7904
7905 format %{ "membar_acquire (elided)" %}
7906
7907 ins_encode %{
7908 __ block_comment("membar_acquire (elided)");
7909 %}
7910
7911 ins_pipe(pipe_class_empty);
7912 %}
7913
7914 instruct membar_acquire() %{
7915 match(MemBarAcquire);
7916 ins_cost(VOLATILE_REF_COST);
7917
7918 format %{ "membar_acquire\n\t"
7919 "dmb ishld" %}
7920
7921 ins_encode %{
7922 __ block_comment("membar_acquire");
7923 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7924 %}
7925
7926 ins_pipe(pipe_serial);
7927 %}
7928
7929
7930 instruct membar_acquire_lock() %{
7931 match(MemBarAcquireLock);
7932 ins_cost(VOLATILE_REF_COST);
7933
7934 format %{ "membar_acquire_lock (elided)" %}
7935
7936 ins_encode %{
7937 __ block_comment("membar_acquire_lock (elided)");
7938 %}
7939
7940 ins_pipe(pipe_serial);
7941 %}
7942
7943 instruct store_fence() %{
7944 match(StoreFence);
7945 ins_cost(VOLATILE_REF_COST);
7946
7947 format %{ "store_fence" %}
7948
7949 ins_encode %{
7950 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7951 %}
7952 ins_pipe(pipe_serial);
7953 %}
7954
7955 instruct unnecessary_membar_release() %{
7956 predicate(unnecessary_release(n));
7957 match(MemBarRelease);
7958 ins_cost(0);
7959
7960 format %{ "membar_release (elided)" %}
7961
7962 ins_encode %{
7963 __ block_comment("membar_release (elided)");
7964 %}
7965 ins_pipe(pipe_serial);
7966 %}
7967
7968 instruct membar_release() %{
7969 match(MemBarRelease);
7970 ins_cost(VOLATILE_REF_COST);
7971
7972 format %{ "membar_release\n\t"
7973 "dmb ishst\n\tdmb ishld" %}
7974
7975 ins_encode %{
7976 __ block_comment("membar_release");
7977 // These will be merged if AlwaysMergeDMB is enabled.
7978 __ membar(Assembler::StoreStore);
7979 __ membar(Assembler::LoadStore);
7980 %}
7981 ins_pipe(pipe_serial);
7982 %}
7983
7984 instruct membar_storestore() %{
7985 match(MemBarStoreStore);
7986 match(StoreStoreFence);
7987 ins_cost(VOLATILE_REF_COST);
7988
7989 format %{ "MEMBAR-store-store" %}
7990
7991 ins_encode %{
7992 __ membar(Assembler::StoreStore);
7993 %}
7994 ins_pipe(pipe_serial);
7995 %}
7996
7997 instruct membar_release_lock() %{
7998 match(MemBarReleaseLock);
7999 ins_cost(VOLATILE_REF_COST);
8000
8001 format %{ "membar_release_lock (elided)" %}
8002
8003 ins_encode %{
8004 __ block_comment("membar_release_lock (elided)");
8005 %}
8006
8007 ins_pipe(pipe_serial);
8008 %}
8009
8010 instruct unnecessary_membar_volatile() %{
8011 predicate(unnecessary_volatile(n));
8012 match(MemBarVolatile);
8013 ins_cost(0);
8014
8015 format %{ "membar_volatile (elided)" %}
8016
8017 ins_encode %{
8018 __ block_comment("membar_volatile (elided)");
8019 %}
8020
8021 ins_pipe(pipe_serial);
8022 %}
8023
8024 instruct membar_volatile() %{
8025 match(MemBarVolatile);
8026 ins_cost(VOLATILE_REF_COST*100);
8027
8028 format %{ "membar_volatile\n\t"
8029 "dmb ish"%}
8030
8031 ins_encode %{
8032 __ block_comment("membar_volatile");
8033 __ membar(Assembler::StoreLoad);
8034 %}
8035
8036 ins_pipe(pipe_serial);
8037 %}
8038
8039 // ============================================================================
8040 // Cast/Convert Instructions
8041
8042 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8043 match(Set dst (CastX2P src));
8044
8045 ins_cost(INSN_COST);
8046 format %{ "mov $dst, $src\t# long -> ptr" %}
8047
8048 ins_encode %{
8049 if ($dst$$reg != $src$$reg) {
8050 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8051 }
8052 %}
8053
8054 ins_pipe(ialu_reg);
8055 %}
8056
8057 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8058 match(Set dst (CastI2N src));
8059
8060 ins_cost(INSN_COST);
8061 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8062
8063 ins_encode %{
8064 if ($dst$$reg != $src$$reg) {
8065 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8066 }
8067 %}
8068
8069 ins_pipe(ialu_reg);
8070 %}
8071
8072 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8073 match(Set dst (CastP2X src));
8074
8075 ins_cost(INSN_COST);
8076 format %{ "mov $dst, $src\t# ptr -> long" %}
8077
8078 ins_encode %{
8079 if ($dst$$reg != $src$$reg) {
8080 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8081 }
8082 %}
8083
8084 ins_pipe(ialu_reg);
8085 %}
8086
8087 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8088 match(Set dst (CastP2X src));
8089
8090 ins_cost(INSN_COST);
8091 format %{ "mov $dst, $src\t# ptr -> long" %}
8092
8093 ins_encode %{
8094 if ($dst$$reg != $src$$reg) {
8095 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8096 }
8097 %}
8098
8099 ins_pipe(ialu_reg);
8100 %}
8101
8102 // Convert oop into int for vectors alignment masking
8103 instruct convP2I(iRegINoSp dst, iRegP src) %{
8104 match(Set dst (ConvL2I (CastP2X src)));
8105
8106 ins_cost(INSN_COST);
8107 format %{ "movw $dst, $src\t# ptr -> int" %}
8108 ins_encode %{
8109 __ movw($dst$$Register, $src$$Register);
8110 %}
8111
8112 ins_pipe(ialu_reg);
8113 %}
8114
8115 // Convert compressed oop into int for vectors alignment masking
8116 // in case of 32bit oops (heap < 4Gb).
8117 instruct convN2I(iRegINoSp dst, iRegN src)
8118 %{
8119 predicate(CompressedOops::shift() == 0);
8120 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8121
8122 ins_cost(INSN_COST);
8123 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8124 ins_encode %{
8125 __ movw($dst$$Register, $src$$Register);
8126 %}
8127
8128 ins_pipe(ialu_reg);
8129 %}
8130
8131
8132 // Convert oop pointer into compressed form
8133 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8134 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8135 match(Set dst (EncodeP src));
8136 effect(KILL cr);
8137 ins_cost(INSN_COST * 3);
8138 format %{ "encode_heap_oop $dst, $src" %}
8139 ins_encode %{
8140 Register s = $src$$Register;
8141 Register d = $dst$$Register;
8142 __ encode_heap_oop(d, s);
8143 %}
8144 ins_pipe(ialu_reg);
8145 %}
8146
8147 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8148 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8149 match(Set dst (EncodeP src));
8150 ins_cost(INSN_COST * 3);
8151 format %{ "encode_heap_oop_not_null $dst, $src" %}
8152 ins_encode %{
8153 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8154 %}
8155 ins_pipe(ialu_reg);
8156 %}
8157
8158 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8159 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8160 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8161 match(Set dst (DecodeN src));
8162 ins_cost(INSN_COST * 3);
8163 format %{ "decode_heap_oop $dst, $src" %}
8164 ins_encode %{
8165 Register s = $src$$Register;
8166 Register d = $dst$$Register;
8167 __ decode_heap_oop(d, s);
8168 %}
8169 ins_pipe(ialu_reg);
8170 %}
8171
8172 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8173 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8174 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8175 match(Set dst (DecodeN src));
8176 ins_cost(INSN_COST * 3);
8177 format %{ "decode_heap_oop_not_null $dst, $src" %}
8178 ins_encode %{
8179 Register s = $src$$Register;
8180 Register d = $dst$$Register;
8181 __ decode_heap_oop_not_null(d, s);
8182 %}
8183 ins_pipe(ialu_reg);
8184 %}
8185
8186 // n.b. AArch64 implementations of encode_klass_not_null and
8187 // decode_klass_not_null do not modify the flags register so, unlike
8188 // Intel, we don't kill CR as a side effect here
8189
8190 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8191 match(Set dst (EncodePKlass src));
8192
8193 ins_cost(INSN_COST * 3);
8194 format %{ "encode_klass_not_null $dst,$src" %}
8195
8196 ins_encode %{
8197 Register src_reg = as_Register($src$$reg);
8198 Register dst_reg = as_Register($dst$$reg);
8199 __ encode_klass_not_null(dst_reg, src_reg);
8200 %}
8201
8202 ins_pipe(ialu_reg);
8203 %}
8204
8205 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8206 match(Set dst (DecodeNKlass src));
8207
8208 ins_cost(INSN_COST * 3);
8209 format %{ "decode_klass_not_null $dst,$src" %}
8210
8211 ins_encode %{
8212 Register src_reg = as_Register($src$$reg);
8213 Register dst_reg = as_Register($dst$$reg);
8214 if (dst_reg != src_reg) {
8215 __ decode_klass_not_null(dst_reg, src_reg);
8216 } else {
8217 __ decode_klass_not_null(dst_reg);
8218 }
8219 %}
8220
8221 ins_pipe(ialu_reg);
8222 %}
8223
8224 instruct checkCastPP(iRegPNoSp dst)
8225 %{
8226 match(Set dst (CheckCastPP dst));
8227
8228 size(0);
8229 format %{ "# checkcastPP of $dst" %}
8230 ins_encode(/* empty encoding */);
8231 ins_pipe(pipe_class_empty);
8232 %}
8233
8234 instruct castPP(iRegPNoSp dst)
8235 %{
8236 match(Set dst (CastPP dst));
8237
8238 size(0);
8239 format %{ "# castPP of $dst" %}
8240 ins_encode(/* empty encoding */);
8241 ins_pipe(pipe_class_empty);
8242 %}
8243
8244 instruct castII(iRegI dst)
8245 %{
8246 predicate(VerifyConstraintCasts == 0);
8247 match(Set dst (CastII dst));
8248
8249 size(0);
8250 format %{ "# castII of $dst" %}
8251 ins_encode(/* empty encoding */);
8252 ins_cost(0);
8253 ins_pipe(pipe_class_empty);
8254 %}
8255
8256 instruct castII_checked(iRegI dst, rFlagsReg cr)
8257 %{
8258 predicate(VerifyConstraintCasts > 0);
8259 match(Set dst (CastII dst));
8260 effect(KILL cr);
8261
8262 format %{ "# castII_checked of $dst" %}
8263 ins_encode %{
8264 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8265 %}
8266 ins_pipe(pipe_slow);
8267 %}
8268
8269 instruct castLL(iRegL dst)
8270 %{
8271 predicate(VerifyConstraintCasts == 0);
8272 match(Set dst (CastLL dst));
8273
8274 size(0);
8275 format %{ "# castLL of $dst" %}
8276 ins_encode(/* empty encoding */);
8277 ins_cost(0);
8278 ins_pipe(pipe_class_empty);
8279 %}
8280
8281 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8282 %{
8283 predicate(VerifyConstraintCasts > 0);
8284 match(Set dst (CastLL dst));
8285 effect(KILL cr);
8286
8287 format %{ "# castLL_checked of $dst" %}
8288 ins_encode %{
8289 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8290 %}
8291 ins_pipe(pipe_slow);
8292 %}
8293
8294 instruct castFF(vRegF dst)
8295 %{
8296 match(Set dst (CastFF dst));
8297
8298 size(0);
8299 format %{ "# castFF of $dst" %}
8300 ins_encode(/* empty encoding */);
8301 ins_cost(0);
8302 ins_pipe(pipe_class_empty);
8303 %}
8304
8305 instruct castDD(vRegD dst)
8306 %{
8307 match(Set dst (CastDD dst));
8308
8309 size(0);
8310 format %{ "# castDD of $dst" %}
8311 ins_encode(/* empty encoding */);
8312 ins_cost(0);
8313 ins_pipe(pipe_class_empty);
8314 %}
8315
8316 instruct castVV(vReg dst)
8317 %{
8318 match(Set dst (CastVV dst));
8319
8320 size(0);
8321 format %{ "# castVV of $dst" %}
8322 ins_encode(/* empty encoding */);
8323 ins_cost(0);
8324 ins_pipe(pipe_class_empty);
8325 %}
8326
8327 instruct castVVMask(pRegGov dst)
8328 %{
8329 match(Set dst (CastVV dst));
8330
8331 size(0);
8332 format %{ "# castVV of $dst" %}
8333 ins_encode(/* empty encoding */);
8334 ins_cost(0);
8335 ins_pipe(pipe_class_empty);
8336 %}
8337
8338 // ============================================================================
8339 // Atomic operation instructions
8340 //
8341
8342 // standard CompareAndSwapX when we are using barriers
8343 // these have higher priority than the rules selected by a predicate
8344
8345 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8346 // can't match them
8347
8348 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8349
8350 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8351 ins_cost(2 * VOLATILE_REF_COST);
8352
8353 effect(KILL cr);
8354
8355 format %{
8356 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8357 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8358 %}
8359
8360 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8361 aarch64_enc_cset_eq(res));
8362
8363 ins_pipe(pipe_slow);
8364 %}
8365
8366 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8367
8368 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8369 ins_cost(2 * VOLATILE_REF_COST);
8370
8371 effect(KILL cr);
8372
8373 format %{
8374 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8375 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8376 %}
8377
8378 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8379 aarch64_enc_cset_eq(res));
8380
8381 ins_pipe(pipe_slow);
8382 %}
8383
8384 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8385
8386 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8387 ins_cost(2 * VOLATILE_REF_COST);
8388
8389 effect(KILL cr);
8390
8391 format %{
8392 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8393 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8394 %}
8395
8396 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8397 aarch64_enc_cset_eq(res));
8398
8399 ins_pipe(pipe_slow);
8400 %}
8401
8402 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8403
8404 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8405 ins_cost(2 * VOLATILE_REF_COST);
8406
8407 effect(KILL cr);
8408
8409 format %{
8410 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8411 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8412 %}
8413
8414 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8415 aarch64_enc_cset_eq(res));
8416
8417 ins_pipe(pipe_slow);
8418 %}
8419
8420 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8421
8422 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8423 predicate(n->as_LoadStore()->barrier_data() == 0);
8424 ins_cost(2 * VOLATILE_REF_COST);
8425
8426 effect(KILL cr);
8427
8428 format %{
8429 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8430 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8431 %}
8432
8433 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8434 aarch64_enc_cset_eq(res));
8435
8436 ins_pipe(pipe_slow);
8437 %}
8438
8439 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8440
8441 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8442 predicate(n->as_LoadStore()->barrier_data() == 0);
8443 ins_cost(2 * VOLATILE_REF_COST);
8444
8445 effect(KILL cr);
8446
8447 format %{
8448 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8449 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8450 %}
8451
8452 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8453 aarch64_enc_cset_eq(res));
8454
8455 ins_pipe(pipe_slow);
8456 %}
8457
8458 // alternative CompareAndSwapX when we are eliding barriers
8459
8460 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8461
8462 predicate(needs_acquiring_load_exclusive(n));
8463 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8464 ins_cost(VOLATILE_REF_COST);
8465
8466 effect(KILL cr);
8467
8468 format %{
8469 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8470 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8471 %}
8472
8473 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8474 aarch64_enc_cset_eq(res));
8475
8476 ins_pipe(pipe_slow);
8477 %}
8478
8479 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8480
8481 predicate(needs_acquiring_load_exclusive(n));
8482 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8483 ins_cost(VOLATILE_REF_COST);
8484
8485 effect(KILL cr);
8486
8487 format %{
8488 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8489 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8490 %}
8491
8492 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8493 aarch64_enc_cset_eq(res));
8494
8495 ins_pipe(pipe_slow);
8496 %}
8497
8498 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8499
8500 predicate(needs_acquiring_load_exclusive(n));
8501 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8502 ins_cost(VOLATILE_REF_COST);
8503
8504 effect(KILL cr);
8505
8506 format %{
8507 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8508 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8509 %}
8510
8511 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8512 aarch64_enc_cset_eq(res));
8513
8514 ins_pipe(pipe_slow);
8515 %}
8516
8517 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8518
8519 predicate(needs_acquiring_load_exclusive(n));
8520 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8521 ins_cost(VOLATILE_REF_COST);
8522
8523 effect(KILL cr);
8524
8525 format %{
8526 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8527 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8528 %}
8529
8530 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8531 aarch64_enc_cset_eq(res));
8532
8533 ins_pipe(pipe_slow);
8534 %}
8535
8536 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8537
8538 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8539 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8540 ins_cost(VOLATILE_REF_COST);
8541
8542 effect(KILL cr);
8543
8544 format %{
8545 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8546 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8547 %}
8548
8549 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8550 aarch64_enc_cset_eq(res));
8551
8552 ins_pipe(pipe_slow);
8553 %}
8554
8555 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8556
8557 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8558 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8559 ins_cost(VOLATILE_REF_COST);
8560
8561 effect(KILL cr);
8562
8563 format %{
8564 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8565 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8566 %}
8567
8568 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8569 aarch64_enc_cset_eq(res));
8570
8571 ins_pipe(pipe_slow);
8572 %}
8573
8574
8575 // ---------------------------------------------------------------------
8576
8577 // BEGIN This section of the file is automatically generated. Do not edit --------------
8578
8579 // Sundry CAS operations. Note that release is always true,
8580 // regardless of the memory ordering of the CAS. This is because we
8581 // need the volatile case to be sequentially consistent but there is
8582 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8583 // can't check the type of memory ordering here, so we always emit a
8584 // STLXR.
8585
8586 // This section is generated from cas.m4
8587
8588
8589 // This pattern is generated automatically from cas.m4.
8590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8591 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8592 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8593 ins_cost(2 * VOLATILE_REF_COST);
8594 effect(TEMP_DEF res, KILL cr);
8595 format %{
8596 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8597 %}
8598 ins_encode %{
8599 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8600 Assembler::byte, /*acquire*/ false, /*release*/ true,
8601 /*weak*/ false, $res$$Register);
8602 __ sxtbw($res$$Register, $res$$Register);
8603 %}
8604 ins_pipe(pipe_slow);
8605 %}
8606
8607 // This pattern is generated automatically from cas.m4.
8608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8609 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8610 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8611 ins_cost(2 * VOLATILE_REF_COST);
8612 effect(TEMP_DEF res, KILL cr);
8613 format %{
8614 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8615 %}
8616 ins_encode %{
8617 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8618 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8619 /*weak*/ false, $res$$Register);
8620 __ sxthw($res$$Register, $res$$Register);
8621 %}
8622 ins_pipe(pipe_slow);
8623 %}
8624
8625 // This pattern is generated automatically from cas.m4.
8626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8627 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8628 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8629 ins_cost(2 * VOLATILE_REF_COST);
8630 effect(TEMP_DEF res, KILL cr);
8631 format %{
8632 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8633 %}
8634 ins_encode %{
8635 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8636 Assembler::word, /*acquire*/ false, /*release*/ true,
8637 /*weak*/ false, $res$$Register);
8638 %}
8639 ins_pipe(pipe_slow);
8640 %}
8641
8642 // This pattern is generated automatically from cas.m4.
8643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8644 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8645 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8646 ins_cost(2 * VOLATILE_REF_COST);
8647 effect(TEMP_DEF res, KILL cr);
8648 format %{
8649 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8650 %}
8651 ins_encode %{
8652 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8653 Assembler::xword, /*acquire*/ false, /*release*/ true,
8654 /*weak*/ false, $res$$Register);
8655 %}
8656 ins_pipe(pipe_slow);
8657 %}
8658
8659 // This pattern is generated automatically from cas.m4.
8660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8661 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8662 predicate(n->as_LoadStore()->barrier_data() == 0);
8663 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8664 ins_cost(2 * VOLATILE_REF_COST);
8665 effect(TEMP_DEF res, KILL cr);
8666 format %{
8667 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8668 %}
8669 ins_encode %{
8670 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8671 Assembler::word, /*acquire*/ false, /*release*/ true,
8672 /*weak*/ false, $res$$Register);
8673 %}
8674 ins_pipe(pipe_slow);
8675 %}
8676
8677 // This pattern is generated automatically from cas.m4.
8678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8679 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8680 predicate(n->as_LoadStore()->barrier_data() == 0);
8681 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8682 ins_cost(2 * VOLATILE_REF_COST);
8683 effect(TEMP_DEF res, KILL cr);
8684 format %{
8685 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8686 %}
8687 ins_encode %{
8688 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8689 Assembler::xword, /*acquire*/ false, /*release*/ true,
8690 /*weak*/ false, $res$$Register);
8691 %}
8692 ins_pipe(pipe_slow);
8693 %}
8694
8695 // This pattern is generated automatically from cas.m4.
8696 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8697 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8698 predicate(needs_acquiring_load_exclusive(n));
8699 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8700 ins_cost(VOLATILE_REF_COST);
8701 effect(TEMP_DEF res, KILL cr);
8702 format %{
8703 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8704 %}
8705 ins_encode %{
8706 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8707 Assembler::byte, /*acquire*/ true, /*release*/ true,
8708 /*weak*/ false, $res$$Register);
8709 __ sxtbw($res$$Register, $res$$Register);
8710 %}
8711 ins_pipe(pipe_slow);
8712 %}
8713
8714 // This pattern is generated automatically from cas.m4.
8715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8716 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8717 predicate(needs_acquiring_load_exclusive(n));
8718 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8719 ins_cost(VOLATILE_REF_COST);
8720 effect(TEMP_DEF res, KILL cr);
8721 format %{
8722 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8723 %}
8724 ins_encode %{
8725 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8726 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8727 /*weak*/ false, $res$$Register);
8728 __ sxthw($res$$Register, $res$$Register);
8729 %}
8730 ins_pipe(pipe_slow);
8731 %}
8732
8733 // This pattern is generated automatically from cas.m4.
8734 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8735 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8736 predicate(needs_acquiring_load_exclusive(n));
8737 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8738 ins_cost(VOLATILE_REF_COST);
8739 effect(TEMP_DEF res, KILL cr);
8740 format %{
8741 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8742 %}
8743 ins_encode %{
8744 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8745 Assembler::word, /*acquire*/ true, /*release*/ true,
8746 /*weak*/ false, $res$$Register);
8747 %}
8748 ins_pipe(pipe_slow);
8749 %}
8750
8751 // This pattern is generated automatically from cas.m4.
8752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8753 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8754 predicate(needs_acquiring_load_exclusive(n));
8755 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8756 ins_cost(VOLATILE_REF_COST);
8757 effect(TEMP_DEF res, KILL cr);
8758 format %{
8759 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8760 %}
8761 ins_encode %{
8762 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8763 Assembler::xword, /*acquire*/ true, /*release*/ true,
8764 /*weak*/ false, $res$$Register);
8765 %}
8766 ins_pipe(pipe_slow);
8767 %}
8768
8769 // This pattern is generated automatically from cas.m4.
8770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8771 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8772 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8773 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8774 ins_cost(VOLATILE_REF_COST);
8775 effect(TEMP_DEF res, KILL cr);
8776 format %{
8777 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8778 %}
8779 ins_encode %{
8780 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8781 Assembler::word, /*acquire*/ true, /*release*/ true,
8782 /*weak*/ false, $res$$Register);
8783 %}
8784 ins_pipe(pipe_slow);
8785 %}
8786
8787 // This pattern is generated automatically from cas.m4.
8788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8789 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8790 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8791 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8792 ins_cost(VOLATILE_REF_COST);
8793 effect(TEMP_DEF res, KILL cr);
8794 format %{
8795 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8796 %}
8797 ins_encode %{
8798 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8799 Assembler::xword, /*acquire*/ true, /*release*/ true,
8800 /*weak*/ false, $res$$Register);
8801 %}
8802 ins_pipe(pipe_slow);
8803 %}
8804
8805 // This pattern is generated automatically from cas.m4.
8806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8807 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8808 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8809 ins_cost(2 * VOLATILE_REF_COST);
8810 effect(KILL cr);
8811 format %{
8812 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8813 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8814 %}
8815 ins_encode %{
8816 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8817 Assembler::byte, /*acquire*/ false, /*release*/ true,
8818 /*weak*/ true, noreg);
8819 __ csetw($res$$Register, Assembler::EQ);
8820 %}
8821 ins_pipe(pipe_slow);
8822 %}
8823
8824 // This pattern is generated automatically from cas.m4.
8825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8826 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8827 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8828 ins_cost(2 * VOLATILE_REF_COST);
8829 effect(KILL cr);
8830 format %{
8831 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8832 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8833 %}
8834 ins_encode %{
8835 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8836 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8837 /*weak*/ true, noreg);
8838 __ csetw($res$$Register, Assembler::EQ);
8839 %}
8840 ins_pipe(pipe_slow);
8841 %}
8842
8843 // This pattern is generated automatically from cas.m4.
8844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8845 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8846 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8847 ins_cost(2 * VOLATILE_REF_COST);
8848 effect(KILL cr);
8849 format %{
8850 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8851 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8852 %}
8853 ins_encode %{
8854 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8855 Assembler::word, /*acquire*/ false, /*release*/ true,
8856 /*weak*/ true, noreg);
8857 __ csetw($res$$Register, Assembler::EQ);
8858 %}
8859 ins_pipe(pipe_slow);
8860 %}
8861
8862 // This pattern is generated automatically from cas.m4.
8863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8864 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8865 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8866 ins_cost(2 * VOLATILE_REF_COST);
8867 effect(KILL cr);
8868 format %{
8869 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8870 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8871 %}
8872 ins_encode %{
8873 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8874 Assembler::xword, /*acquire*/ false, /*release*/ true,
8875 /*weak*/ true, noreg);
8876 __ csetw($res$$Register, Assembler::EQ);
8877 %}
8878 ins_pipe(pipe_slow);
8879 %}
8880
8881 // This pattern is generated automatically from cas.m4.
8882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8883 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8884 predicate(n->as_LoadStore()->barrier_data() == 0);
8885 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8886 ins_cost(2 * VOLATILE_REF_COST);
8887 effect(KILL cr);
8888 format %{
8889 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8890 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8891 %}
8892 ins_encode %{
8893 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8894 Assembler::word, /*acquire*/ false, /*release*/ true,
8895 /*weak*/ true, noreg);
8896 __ csetw($res$$Register, Assembler::EQ);
8897 %}
8898 ins_pipe(pipe_slow);
8899 %}
8900
8901 // This pattern is generated automatically from cas.m4.
8902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8903 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8904 predicate(n->as_LoadStore()->barrier_data() == 0);
8905 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8906 ins_cost(2 * VOLATILE_REF_COST);
8907 effect(KILL cr);
8908 format %{
8909 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8910 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8911 %}
8912 ins_encode %{
8913 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8914 Assembler::xword, /*acquire*/ false, /*release*/ true,
8915 /*weak*/ true, noreg);
8916 __ csetw($res$$Register, Assembler::EQ);
8917 %}
8918 ins_pipe(pipe_slow);
8919 %}
8920
8921 // This pattern is generated automatically from cas.m4.
8922 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8923 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8924 predicate(needs_acquiring_load_exclusive(n));
8925 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8926 ins_cost(VOLATILE_REF_COST);
8927 effect(KILL cr);
8928 format %{
8929 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8930 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8931 %}
8932 ins_encode %{
8933 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8934 Assembler::byte, /*acquire*/ true, /*release*/ true,
8935 /*weak*/ true, noreg);
8936 __ csetw($res$$Register, Assembler::EQ);
8937 %}
8938 ins_pipe(pipe_slow);
8939 %}
8940
8941 // This pattern is generated automatically from cas.m4.
8942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8943 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8944 predicate(needs_acquiring_load_exclusive(n));
8945 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8946 ins_cost(VOLATILE_REF_COST);
8947 effect(KILL cr);
8948 format %{
8949 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8950 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8951 %}
8952 ins_encode %{
8953 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8954 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8955 /*weak*/ true, noreg);
8956 __ csetw($res$$Register, Assembler::EQ);
8957 %}
8958 ins_pipe(pipe_slow);
8959 %}
8960
8961 // This pattern is generated automatically from cas.m4.
8962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8963 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8964 predicate(needs_acquiring_load_exclusive(n));
8965 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8966 ins_cost(VOLATILE_REF_COST);
8967 effect(KILL cr);
8968 format %{
8969 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8970 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8971 %}
8972 ins_encode %{
8973 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8974 Assembler::word, /*acquire*/ true, /*release*/ true,
8975 /*weak*/ true, noreg);
8976 __ csetw($res$$Register, Assembler::EQ);
8977 %}
8978 ins_pipe(pipe_slow);
8979 %}
8980
8981 // This pattern is generated automatically from cas.m4.
8982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8983 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8984 predicate(needs_acquiring_load_exclusive(n));
8985 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8986 ins_cost(VOLATILE_REF_COST);
8987 effect(KILL cr);
8988 format %{
8989 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8990 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8991 %}
8992 ins_encode %{
8993 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8994 Assembler::xword, /*acquire*/ true, /*release*/ true,
8995 /*weak*/ true, noreg);
8996 __ csetw($res$$Register, Assembler::EQ);
8997 %}
8998 ins_pipe(pipe_slow);
8999 %}
9000
9001 // This pattern is generated automatically from cas.m4.
9002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9003 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9004 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9005 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9006 ins_cost(VOLATILE_REF_COST);
9007 effect(KILL cr);
9008 format %{
9009 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9010 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9011 %}
9012 ins_encode %{
9013 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9014 Assembler::word, /*acquire*/ true, /*release*/ true,
9015 /*weak*/ true, noreg);
9016 __ csetw($res$$Register, Assembler::EQ);
9017 %}
9018 ins_pipe(pipe_slow);
9019 %}
9020
9021 // This pattern is generated automatically from cas.m4.
9022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9023 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9024 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9025 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9026 ins_cost(VOLATILE_REF_COST);
9027 effect(KILL cr);
9028 format %{
9029 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9030 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9031 %}
9032 ins_encode %{
9033 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9034 Assembler::xword, /*acquire*/ true, /*release*/ true,
9035 /*weak*/ true, noreg);
9036 __ csetw($res$$Register, Assembler::EQ);
9037 %}
9038 ins_pipe(pipe_slow);
9039 %}
9040
9041 // END This section of the file is automatically generated. Do not edit --------------
9042 // ---------------------------------------------------------------------
9043
9044 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9045 match(Set prev (GetAndSetI mem newv));
9046 ins_cost(2 * VOLATILE_REF_COST);
9047 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9048 ins_encode %{
9049 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9050 %}
9051 ins_pipe(pipe_serial);
9052 %}
9053
9054 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9055 match(Set prev (GetAndSetL mem newv));
9056 ins_cost(2 * VOLATILE_REF_COST);
9057 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9058 ins_encode %{
9059 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9060 %}
9061 ins_pipe(pipe_serial);
9062 %}
9063
9064 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9065 predicate(n->as_LoadStore()->barrier_data() == 0);
9066 match(Set prev (GetAndSetN mem newv));
9067 ins_cost(2 * VOLATILE_REF_COST);
9068 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9069 ins_encode %{
9070 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9071 %}
9072 ins_pipe(pipe_serial);
9073 %}
9074
9075 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9076 predicate(n->as_LoadStore()->barrier_data() == 0);
9077 match(Set prev (GetAndSetP mem newv));
9078 ins_cost(2 * VOLATILE_REF_COST);
9079 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9080 ins_encode %{
9081 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9082 %}
9083 ins_pipe(pipe_serial);
9084 %}
9085
9086 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9087 predicate(needs_acquiring_load_exclusive(n));
9088 match(Set prev (GetAndSetI mem newv));
9089 ins_cost(VOLATILE_REF_COST);
9090 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9091 ins_encode %{
9092 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9093 %}
9094 ins_pipe(pipe_serial);
9095 %}
9096
9097 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9098 predicate(needs_acquiring_load_exclusive(n));
9099 match(Set prev (GetAndSetL mem newv));
9100 ins_cost(VOLATILE_REF_COST);
9101 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9102 ins_encode %{
9103 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9104 %}
9105 ins_pipe(pipe_serial);
9106 %}
9107
9108 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9109 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9110 match(Set prev (GetAndSetN mem newv));
9111 ins_cost(VOLATILE_REF_COST);
9112 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9113 ins_encode %{
9114 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9115 %}
9116 ins_pipe(pipe_serial);
9117 %}
9118
9119 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9120 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9121 match(Set prev (GetAndSetP mem newv));
9122 ins_cost(VOLATILE_REF_COST);
9123 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9124 ins_encode %{
9125 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9126 %}
9127 ins_pipe(pipe_serial);
9128 %}
9129
9130
9131 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9132 match(Set newval (GetAndAddL mem incr));
9133 ins_cost(2 * VOLATILE_REF_COST + 1);
9134 format %{ "get_and_addL $newval, [$mem], $incr" %}
9135 ins_encode %{
9136 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9137 %}
9138 ins_pipe(pipe_serial);
9139 %}
9140
9141 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9142 predicate(n->as_LoadStore()->result_not_used());
9143 match(Set dummy (GetAndAddL mem incr));
9144 ins_cost(2 * VOLATILE_REF_COST);
9145 format %{ "get_and_addL [$mem], $incr" %}
9146 ins_encode %{
9147 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9148 %}
9149 ins_pipe(pipe_serial);
9150 %}
9151
9152 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9153 match(Set newval (GetAndAddL mem incr));
9154 ins_cost(2 * VOLATILE_REF_COST + 1);
9155 format %{ "get_and_addL $newval, [$mem], $incr" %}
9156 ins_encode %{
9157 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9158 %}
9159 ins_pipe(pipe_serial);
9160 %}
9161
9162 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9163 predicate(n->as_LoadStore()->result_not_used());
9164 match(Set dummy (GetAndAddL mem incr));
9165 ins_cost(2 * VOLATILE_REF_COST);
9166 format %{ "get_and_addL [$mem], $incr" %}
9167 ins_encode %{
9168 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9169 %}
9170 ins_pipe(pipe_serial);
9171 %}
9172
9173 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9174 match(Set newval (GetAndAddI mem incr));
9175 ins_cost(2 * VOLATILE_REF_COST + 1);
9176 format %{ "get_and_addI $newval, [$mem], $incr" %}
9177 ins_encode %{
9178 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9179 %}
9180 ins_pipe(pipe_serial);
9181 %}
9182
9183 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9184 predicate(n->as_LoadStore()->result_not_used());
9185 match(Set dummy (GetAndAddI mem incr));
9186 ins_cost(2 * VOLATILE_REF_COST);
9187 format %{ "get_and_addI [$mem], $incr" %}
9188 ins_encode %{
9189 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9190 %}
9191 ins_pipe(pipe_serial);
9192 %}
9193
9194 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9195 match(Set newval (GetAndAddI mem incr));
9196 ins_cost(2 * VOLATILE_REF_COST + 1);
9197 format %{ "get_and_addI $newval, [$mem], $incr" %}
9198 ins_encode %{
9199 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9200 %}
9201 ins_pipe(pipe_serial);
9202 %}
9203
9204 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9205 predicate(n->as_LoadStore()->result_not_used());
9206 match(Set dummy (GetAndAddI mem incr));
9207 ins_cost(2 * VOLATILE_REF_COST);
9208 format %{ "get_and_addI [$mem], $incr" %}
9209 ins_encode %{
9210 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9211 %}
9212 ins_pipe(pipe_serial);
9213 %}
9214
9215 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9216 predicate(needs_acquiring_load_exclusive(n));
9217 match(Set newval (GetAndAddL mem incr));
9218 ins_cost(VOLATILE_REF_COST + 1);
9219 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9220 ins_encode %{
9221 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9222 %}
9223 ins_pipe(pipe_serial);
9224 %}
9225
9226 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9227 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9228 match(Set dummy (GetAndAddL mem incr));
9229 ins_cost(VOLATILE_REF_COST);
9230 format %{ "get_and_addL_acq [$mem], $incr" %}
9231 ins_encode %{
9232 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9233 %}
9234 ins_pipe(pipe_serial);
9235 %}
9236
9237 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9238 predicate(needs_acquiring_load_exclusive(n));
9239 match(Set newval (GetAndAddL mem incr));
9240 ins_cost(VOLATILE_REF_COST + 1);
9241 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9242 ins_encode %{
9243 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9244 %}
9245 ins_pipe(pipe_serial);
9246 %}
9247
9248 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9249 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9250 match(Set dummy (GetAndAddL mem incr));
9251 ins_cost(VOLATILE_REF_COST);
9252 format %{ "get_and_addL_acq [$mem], $incr" %}
9253 ins_encode %{
9254 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9255 %}
9256 ins_pipe(pipe_serial);
9257 %}
9258
9259 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9260 predicate(needs_acquiring_load_exclusive(n));
9261 match(Set newval (GetAndAddI mem incr));
9262 ins_cost(VOLATILE_REF_COST + 1);
9263 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9264 ins_encode %{
9265 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9266 %}
9267 ins_pipe(pipe_serial);
9268 %}
9269
9270 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9271 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9272 match(Set dummy (GetAndAddI mem incr));
9273 ins_cost(VOLATILE_REF_COST);
9274 format %{ "get_and_addI_acq [$mem], $incr" %}
9275 ins_encode %{
9276 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9277 %}
9278 ins_pipe(pipe_serial);
9279 %}
9280
9281 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9282 predicate(needs_acquiring_load_exclusive(n));
9283 match(Set newval (GetAndAddI mem incr));
9284 ins_cost(VOLATILE_REF_COST + 1);
9285 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9286 ins_encode %{
9287 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9288 %}
9289 ins_pipe(pipe_serial);
9290 %}
9291
9292 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9293 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9294 match(Set dummy (GetAndAddI mem incr));
9295 ins_cost(VOLATILE_REF_COST);
9296 format %{ "get_and_addI_acq [$mem], $incr" %}
9297 ins_encode %{
9298 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9299 %}
9300 ins_pipe(pipe_serial);
9301 %}
9302
9303 // Manifest a CmpU result in an integer register.
9304 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9305 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9306 %{
9307 match(Set dst (CmpU3 src1 src2));
9308 effect(KILL flags);
9309
9310 ins_cost(INSN_COST * 3);
9311 format %{
9312 "cmpw $src1, $src2\n\t"
9313 "csetw $dst, ne\n\t"
9314 "cnegw $dst, lo\t# CmpU3(reg)"
9315 %}
9316 ins_encode %{
9317 __ cmpw($src1$$Register, $src2$$Register);
9318 __ csetw($dst$$Register, Assembler::NE);
9319 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9320 %}
9321
9322 ins_pipe(pipe_class_default);
9323 %}
9324
9325 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9326 %{
9327 match(Set dst (CmpU3 src1 src2));
9328 effect(KILL flags);
9329
9330 ins_cost(INSN_COST * 3);
9331 format %{
9332 "subsw zr, $src1, $src2\n\t"
9333 "csetw $dst, ne\n\t"
9334 "cnegw $dst, lo\t# CmpU3(imm)"
9335 %}
9336 ins_encode %{
9337 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9338 __ csetw($dst$$Register, Assembler::NE);
9339 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9340 %}
9341
9342 ins_pipe(pipe_class_default);
9343 %}
9344
9345 // Manifest a CmpUL result in an integer register.
9346 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9347 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9348 %{
9349 match(Set dst (CmpUL3 src1 src2));
9350 effect(KILL flags);
9351
9352 ins_cost(INSN_COST * 3);
9353 format %{
9354 "cmp $src1, $src2\n\t"
9355 "csetw $dst, ne\n\t"
9356 "cnegw $dst, lo\t# CmpUL3(reg)"
9357 %}
9358 ins_encode %{
9359 __ cmp($src1$$Register, $src2$$Register);
9360 __ csetw($dst$$Register, Assembler::NE);
9361 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9362 %}
9363
9364 ins_pipe(pipe_class_default);
9365 %}
9366
9367 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9368 %{
9369 match(Set dst (CmpUL3 src1 src2));
9370 effect(KILL flags);
9371
9372 ins_cost(INSN_COST * 3);
9373 format %{
9374 "subs zr, $src1, $src2\n\t"
9375 "csetw $dst, ne\n\t"
9376 "cnegw $dst, lo\t# CmpUL3(imm)"
9377 %}
9378 ins_encode %{
9379 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9380 __ csetw($dst$$Register, Assembler::NE);
9381 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9382 %}
9383
9384 ins_pipe(pipe_class_default);
9385 %}
9386
9387 // Manifest a CmpL result in an integer register.
9388 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9389 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9390 %{
9391 match(Set dst (CmpL3 src1 src2));
9392 effect(KILL flags);
9393
9394 ins_cost(INSN_COST * 3);
9395 format %{
9396 "cmp $src1, $src2\n\t"
9397 "csetw $dst, ne\n\t"
9398 "cnegw $dst, lt\t# CmpL3(reg)"
9399 %}
9400 ins_encode %{
9401 __ cmp($src1$$Register, $src2$$Register);
9402 __ csetw($dst$$Register, Assembler::NE);
9403 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9404 %}
9405
9406 ins_pipe(pipe_class_default);
9407 %}
9408
9409 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9410 %{
9411 match(Set dst (CmpL3 src1 src2));
9412 effect(KILL flags);
9413
9414 ins_cost(INSN_COST * 3);
9415 format %{
9416 "subs zr, $src1, $src2\n\t"
9417 "csetw $dst, ne\n\t"
9418 "cnegw $dst, lt\t# CmpL3(imm)"
9419 %}
9420 ins_encode %{
9421 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9422 __ csetw($dst$$Register, Assembler::NE);
9423 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9424 %}
9425
9426 ins_pipe(pipe_class_default);
9427 %}
9428
9429 // ============================================================================
9430 // Conditional Move Instructions
9431
9432 // n.b. we have identical rules for both a signed compare op (cmpOp)
9433 // and an unsigned compare op (cmpOpU). it would be nice if we could
9434 // define an op class which merged both inputs and use it to type the
9435 // argument to a single rule. unfortunatelyt his fails because the
9436 // opclass does not live up to the COND_INTER interface of its
9437 // component operands. When the generic code tries to negate the
9438 // operand it ends up running the generci Machoper::negate method
9439 // which throws a ShouldNotHappen. So, we have to provide two flavours
9440 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9441
9442 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9443 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9444
9445 ins_cost(INSN_COST * 2);
9446 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9447
9448 ins_encode %{
9449 __ cselw(as_Register($dst$$reg),
9450 as_Register($src2$$reg),
9451 as_Register($src1$$reg),
9452 (Assembler::Condition)$cmp$$cmpcode);
9453 %}
9454
9455 ins_pipe(icond_reg_reg);
9456 %}
9457
9458 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9459 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9460
9461 ins_cost(INSN_COST * 2);
9462 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9463
9464 ins_encode %{
9465 __ cselw(as_Register($dst$$reg),
9466 as_Register($src2$$reg),
9467 as_Register($src1$$reg),
9468 (Assembler::Condition)$cmp$$cmpcode);
9469 %}
9470
9471 ins_pipe(icond_reg_reg);
9472 %}
9473
9474 // special cases where one arg is zero
9475
9476 // n.b. this is selected in preference to the rule above because it
9477 // avoids loading constant 0 into a source register
9478
9479 // TODO
9480 // we ought only to be able to cull one of these variants as the ideal
9481 // transforms ought always to order the zero consistently (to left/right?)
9482
9483 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9484 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9485
9486 ins_cost(INSN_COST * 2);
9487 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9488
9489 ins_encode %{
9490 __ cselw(as_Register($dst$$reg),
9491 as_Register($src$$reg),
9492 zr,
9493 (Assembler::Condition)$cmp$$cmpcode);
9494 %}
9495
9496 ins_pipe(icond_reg);
9497 %}
9498
9499 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9500 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9501
9502 ins_cost(INSN_COST * 2);
9503 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9504
9505 ins_encode %{
9506 __ cselw(as_Register($dst$$reg),
9507 as_Register($src$$reg),
9508 zr,
9509 (Assembler::Condition)$cmp$$cmpcode);
9510 %}
9511
9512 ins_pipe(icond_reg);
9513 %}
9514
9515 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9516 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9517
9518 ins_cost(INSN_COST * 2);
9519 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9520
9521 ins_encode %{
9522 __ cselw(as_Register($dst$$reg),
9523 zr,
9524 as_Register($src$$reg),
9525 (Assembler::Condition)$cmp$$cmpcode);
9526 %}
9527
9528 ins_pipe(icond_reg);
9529 %}
9530
9531 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9532 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9533
9534 ins_cost(INSN_COST * 2);
9535 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9536
9537 ins_encode %{
9538 __ cselw(as_Register($dst$$reg),
9539 zr,
9540 as_Register($src$$reg),
9541 (Assembler::Condition)$cmp$$cmpcode);
9542 %}
9543
9544 ins_pipe(icond_reg);
9545 %}
9546
9547 // special case for creating a boolean 0 or 1
9548
9549 // n.b. this is selected in preference to the rule above because it
9550 // avoids loading constants 0 and 1 into a source register
9551
9552 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9553 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9554
9555 ins_cost(INSN_COST * 2);
9556 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9557
9558 ins_encode %{
9559 // equivalently
9560 // cset(as_Register($dst$$reg),
9561 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9562 __ csincw(as_Register($dst$$reg),
9563 zr,
9564 zr,
9565 (Assembler::Condition)$cmp$$cmpcode);
9566 %}
9567
9568 ins_pipe(icond_none);
9569 %}
9570
9571 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9572 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9573
9574 ins_cost(INSN_COST * 2);
9575 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9576
9577 ins_encode %{
9578 // equivalently
9579 // cset(as_Register($dst$$reg),
9580 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9581 __ csincw(as_Register($dst$$reg),
9582 zr,
9583 zr,
9584 (Assembler::Condition)$cmp$$cmpcode);
9585 %}
9586
9587 ins_pipe(icond_none);
9588 %}
9589
9590 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9591 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9592
9593 ins_cost(INSN_COST * 2);
9594 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9595
9596 ins_encode %{
9597 __ csel(as_Register($dst$$reg),
9598 as_Register($src2$$reg),
9599 as_Register($src1$$reg),
9600 (Assembler::Condition)$cmp$$cmpcode);
9601 %}
9602
9603 ins_pipe(icond_reg_reg);
9604 %}
9605
9606 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9607 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9608
9609 ins_cost(INSN_COST * 2);
9610 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9611
9612 ins_encode %{
9613 __ csel(as_Register($dst$$reg),
9614 as_Register($src2$$reg),
9615 as_Register($src1$$reg),
9616 (Assembler::Condition)$cmp$$cmpcode);
9617 %}
9618
9619 ins_pipe(icond_reg_reg);
9620 %}
9621
9622 // special cases where one arg is zero
9623
9624 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9625 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9626
9627 ins_cost(INSN_COST * 2);
9628 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9629
9630 ins_encode %{
9631 __ csel(as_Register($dst$$reg),
9632 zr,
9633 as_Register($src$$reg),
9634 (Assembler::Condition)$cmp$$cmpcode);
9635 %}
9636
9637 ins_pipe(icond_reg);
9638 %}
9639
9640 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9641 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9642
9643 ins_cost(INSN_COST * 2);
9644 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9645
9646 ins_encode %{
9647 __ csel(as_Register($dst$$reg),
9648 zr,
9649 as_Register($src$$reg),
9650 (Assembler::Condition)$cmp$$cmpcode);
9651 %}
9652
9653 ins_pipe(icond_reg);
9654 %}
9655
9656 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9657 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9658
9659 ins_cost(INSN_COST * 2);
9660 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9661
9662 ins_encode %{
9663 __ csel(as_Register($dst$$reg),
9664 as_Register($src$$reg),
9665 zr,
9666 (Assembler::Condition)$cmp$$cmpcode);
9667 %}
9668
9669 ins_pipe(icond_reg);
9670 %}
9671
9672 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9673 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9674
9675 ins_cost(INSN_COST * 2);
9676 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9677
9678 ins_encode %{
9679 __ csel(as_Register($dst$$reg),
9680 as_Register($src$$reg),
9681 zr,
9682 (Assembler::Condition)$cmp$$cmpcode);
9683 %}
9684
9685 ins_pipe(icond_reg);
9686 %}
9687
9688 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9689 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9690
9691 ins_cost(INSN_COST * 2);
9692 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9693
9694 ins_encode %{
9695 __ csel(as_Register($dst$$reg),
9696 as_Register($src2$$reg),
9697 as_Register($src1$$reg),
9698 (Assembler::Condition)$cmp$$cmpcode);
9699 %}
9700
9701 ins_pipe(icond_reg_reg);
9702 %}
9703
9704 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9705 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9706
9707 ins_cost(INSN_COST * 2);
9708 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9709
9710 ins_encode %{
9711 __ csel(as_Register($dst$$reg),
9712 as_Register($src2$$reg),
9713 as_Register($src1$$reg),
9714 (Assembler::Condition)$cmp$$cmpcode);
9715 %}
9716
9717 ins_pipe(icond_reg_reg);
9718 %}
9719
9720 // special cases where one arg is zero
9721
9722 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9723 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9724
9725 ins_cost(INSN_COST * 2);
9726 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9727
9728 ins_encode %{
9729 __ csel(as_Register($dst$$reg),
9730 zr,
9731 as_Register($src$$reg),
9732 (Assembler::Condition)$cmp$$cmpcode);
9733 %}
9734
9735 ins_pipe(icond_reg);
9736 %}
9737
9738 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9739 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9740
9741 ins_cost(INSN_COST * 2);
9742 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9743
9744 ins_encode %{
9745 __ csel(as_Register($dst$$reg),
9746 zr,
9747 as_Register($src$$reg),
9748 (Assembler::Condition)$cmp$$cmpcode);
9749 %}
9750
9751 ins_pipe(icond_reg);
9752 %}
9753
9754 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9755 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9756
9757 ins_cost(INSN_COST * 2);
9758 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9759
9760 ins_encode %{
9761 __ csel(as_Register($dst$$reg),
9762 as_Register($src$$reg),
9763 zr,
9764 (Assembler::Condition)$cmp$$cmpcode);
9765 %}
9766
9767 ins_pipe(icond_reg);
9768 %}
9769
9770 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9771 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9772
9773 ins_cost(INSN_COST * 2);
9774 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9775
9776 ins_encode %{
9777 __ csel(as_Register($dst$$reg),
9778 as_Register($src$$reg),
9779 zr,
9780 (Assembler::Condition)$cmp$$cmpcode);
9781 %}
9782
9783 ins_pipe(icond_reg);
9784 %}
9785
9786 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9787 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9788
9789 ins_cost(INSN_COST * 2);
9790 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9791
9792 ins_encode %{
9793 __ cselw(as_Register($dst$$reg),
9794 as_Register($src2$$reg),
9795 as_Register($src1$$reg),
9796 (Assembler::Condition)$cmp$$cmpcode);
9797 %}
9798
9799 ins_pipe(icond_reg_reg);
9800 %}
9801
9802 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9803 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9804
9805 ins_cost(INSN_COST * 2);
9806 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9807
9808 ins_encode %{
9809 __ cselw(as_Register($dst$$reg),
9810 as_Register($src2$$reg),
9811 as_Register($src1$$reg),
9812 (Assembler::Condition)$cmp$$cmpcode);
9813 %}
9814
9815 ins_pipe(icond_reg_reg);
9816 %}
9817
9818 // special cases where one arg is zero
9819
9820 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9821 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9822
9823 ins_cost(INSN_COST * 2);
9824 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9825
9826 ins_encode %{
9827 __ cselw(as_Register($dst$$reg),
9828 zr,
9829 as_Register($src$$reg),
9830 (Assembler::Condition)$cmp$$cmpcode);
9831 %}
9832
9833 ins_pipe(icond_reg);
9834 %}
9835
9836 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9837 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9838
9839 ins_cost(INSN_COST * 2);
9840 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9841
9842 ins_encode %{
9843 __ cselw(as_Register($dst$$reg),
9844 zr,
9845 as_Register($src$$reg),
9846 (Assembler::Condition)$cmp$$cmpcode);
9847 %}
9848
9849 ins_pipe(icond_reg);
9850 %}
9851
9852 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9853 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9854
9855 ins_cost(INSN_COST * 2);
9856 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9857
9858 ins_encode %{
9859 __ cselw(as_Register($dst$$reg),
9860 as_Register($src$$reg),
9861 zr,
9862 (Assembler::Condition)$cmp$$cmpcode);
9863 %}
9864
9865 ins_pipe(icond_reg);
9866 %}
9867
9868 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9869 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9870
9871 ins_cost(INSN_COST * 2);
9872 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9873
9874 ins_encode %{
9875 __ cselw(as_Register($dst$$reg),
9876 as_Register($src$$reg),
9877 zr,
9878 (Assembler::Condition)$cmp$$cmpcode);
9879 %}
9880
9881 ins_pipe(icond_reg);
9882 %}
9883
9884 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9885 %{
9886 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9887
9888 ins_cost(INSN_COST * 3);
9889
9890 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9891 ins_encode %{
9892 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9893 __ fcsels(as_FloatRegister($dst$$reg),
9894 as_FloatRegister($src2$$reg),
9895 as_FloatRegister($src1$$reg),
9896 cond);
9897 %}
9898
9899 ins_pipe(fp_cond_reg_reg_s);
9900 %}
9901
9902 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9903 %{
9904 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9905
9906 ins_cost(INSN_COST * 3);
9907
9908 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9909 ins_encode %{
9910 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9911 __ fcsels(as_FloatRegister($dst$$reg),
9912 as_FloatRegister($src2$$reg),
9913 as_FloatRegister($src1$$reg),
9914 cond);
9915 %}
9916
9917 ins_pipe(fp_cond_reg_reg_s);
9918 %}
9919
9920 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9921 %{
9922 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9923
9924 ins_cost(INSN_COST * 3);
9925
9926 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9927 ins_encode %{
9928 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9929 __ fcseld(as_FloatRegister($dst$$reg),
9930 as_FloatRegister($src2$$reg),
9931 as_FloatRegister($src1$$reg),
9932 cond);
9933 %}
9934
9935 ins_pipe(fp_cond_reg_reg_d);
9936 %}
9937
9938 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9939 %{
9940 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9941
9942 ins_cost(INSN_COST * 3);
9943
9944 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9945 ins_encode %{
9946 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9947 __ fcseld(as_FloatRegister($dst$$reg),
9948 as_FloatRegister($src2$$reg),
9949 as_FloatRegister($src1$$reg),
9950 cond);
9951 %}
9952
9953 ins_pipe(fp_cond_reg_reg_d);
9954 %}
9955
9956 // ============================================================================
9957 // Arithmetic Instructions
9958 //
9959
9960 // Integer Addition
9961
9962 // TODO
9963 // these currently employ operations which do not set CR and hence are
9964 // not flagged as killing CR but we would like to isolate the cases
9965 // where we want to set flags from those where we don't. need to work
9966 // out how to do that.
9967
9968 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9969 match(Set dst (AddI src1 src2));
9970
9971 ins_cost(INSN_COST);
9972 format %{ "addw $dst, $src1, $src2" %}
9973
9974 ins_encode %{
9975 __ addw(as_Register($dst$$reg),
9976 as_Register($src1$$reg),
9977 as_Register($src2$$reg));
9978 %}
9979
9980 ins_pipe(ialu_reg_reg);
9981 %}
9982
9983 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9984 match(Set dst (AddI src1 src2));
9985
9986 ins_cost(INSN_COST);
9987 format %{ "addw $dst, $src1, $src2" %}
9988
9989 // use opcode to indicate that this is an add not a sub
9990 opcode(0x0);
9991
9992 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9993
9994 ins_pipe(ialu_reg_imm);
9995 %}
9996
9997 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9998 match(Set dst (AddI (ConvL2I src1) src2));
9999
10000 ins_cost(INSN_COST);
10001 format %{ "addw $dst, $src1, $src2" %}
10002
10003 // use opcode to indicate that this is an add not a sub
10004 opcode(0x0);
10005
10006 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10007
10008 ins_pipe(ialu_reg_imm);
10009 %}
10010
10011 // Pointer Addition
10012 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
10013 match(Set dst (AddP src1 src2));
10014
10015 ins_cost(INSN_COST);
10016 format %{ "add $dst, $src1, $src2\t# ptr" %}
10017
10018 ins_encode %{
10019 __ add(as_Register($dst$$reg),
10020 as_Register($src1$$reg),
10021 as_Register($src2$$reg));
10022 %}
10023
10024 ins_pipe(ialu_reg_reg);
10025 %}
10026
10027 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
10028 match(Set dst (AddP src1 (ConvI2L src2)));
10029
10030 ins_cost(1.9 * INSN_COST);
10031 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10032
10033 ins_encode %{
10034 __ add(as_Register($dst$$reg),
10035 as_Register($src1$$reg),
10036 as_Register($src2$$reg), ext::sxtw);
10037 %}
10038
10039 ins_pipe(ialu_reg_reg);
10040 %}
10041
10042 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10043 match(Set dst (AddP src1 (LShiftL src2 scale)));
10044
10045 ins_cost(1.9 * INSN_COST);
10046 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10047
10048 ins_encode %{
10049 __ lea(as_Register($dst$$reg),
10050 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10051 Address::lsl($scale$$constant)));
10052 %}
10053
10054 ins_pipe(ialu_reg_reg_shift);
10055 %}
10056
10057 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10058 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10059
10060 ins_cost(1.9 * INSN_COST);
10061 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10062
10063 ins_encode %{
10064 __ lea(as_Register($dst$$reg),
10065 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10066 Address::sxtw($scale$$constant)));
10067 %}
10068
10069 ins_pipe(ialu_reg_reg_shift);
10070 %}
10071
10072 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10073 match(Set dst (LShiftL (ConvI2L src) scale));
10074
10075 ins_cost(INSN_COST);
10076 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10077
10078 ins_encode %{
10079 __ sbfiz(as_Register($dst$$reg),
10080 as_Register($src$$reg),
10081 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10082 %}
10083
10084 ins_pipe(ialu_reg_shift);
10085 %}
10086
10087 // Pointer Immediate Addition
10088 // n.b. this needs to be more expensive than using an indirect memory
10089 // operand
10090 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10091 match(Set dst (AddP src1 src2));
10092
10093 ins_cost(INSN_COST);
10094 format %{ "add $dst, $src1, $src2\t# ptr" %}
10095
10096 // use opcode to indicate that this is an add not a sub
10097 opcode(0x0);
10098
10099 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10100
10101 ins_pipe(ialu_reg_imm);
10102 %}
10103
10104 // Long Addition
10105 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10106
10107 match(Set dst (AddL src1 src2));
10108
10109 ins_cost(INSN_COST);
10110 format %{ "add $dst, $src1, $src2" %}
10111
10112 ins_encode %{
10113 __ add(as_Register($dst$$reg),
10114 as_Register($src1$$reg),
10115 as_Register($src2$$reg));
10116 %}
10117
10118 ins_pipe(ialu_reg_reg);
10119 %}
10120
10121 // No constant pool entries requiredLong Immediate Addition.
10122 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10123 match(Set dst (AddL src1 src2));
10124
10125 ins_cost(INSN_COST);
10126 format %{ "add $dst, $src1, $src2" %}
10127
10128 // use opcode to indicate that this is an add not a sub
10129 opcode(0x0);
10130
10131 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10132
10133 ins_pipe(ialu_reg_imm);
10134 %}
10135
10136 // Integer Subtraction
10137 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10138 match(Set dst (SubI src1 src2));
10139
10140 ins_cost(INSN_COST);
10141 format %{ "subw $dst, $src1, $src2" %}
10142
10143 ins_encode %{
10144 __ subw(as_Register($dst$$reg),
10145 as_Register($src1$$reg),
10146 as_Register($src2$$reg));
10147 %}
10148
10149 ins_pipe(ialu_reg_reg);
10150 %}
10151
10152 // Immediate Subtraction
10153 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10154 match(Set dst (SubI src1 src2));
10155
10156 ins_cost(INSN_COST);
10157 format %{ "subw $dst, $src1, $src2" %}
10158
10159 // use opcode to indicate that this is a sub not an add
10160 opcode(0x1);
10161
10162 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10163
10164 ins_pipe(ialu_reg_imm);
10165 %}
10166
10167 // Long Subtraction
10168 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10169
10170 match(Set dst (SubL src1 src2));
10171
10172 ins_cost(INSN_COST);
10173 format %{ "sub $dst, $src1, $src2" %}
10174
10175 ins_encode %{
10176 __ sub(as_Register($dst$$reg),
10177 as_Register($src1$$reg),
10178 as_Register($src2$$reg));
10179 %}
10180
10181 ins_pipe(ialu_reg_reg);
10182 %}
10183
10184 // No constant pool entries requiredLong Immediate Subtraction.
10185 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10186 match(Set dst (SubL src1 src2));
10187
10188 ins_cost(INSN_COST);
10189 format %{ "sub$dst, $src1, $src2" %}
10190
10191 // use opcode to indicate that this is a sub not an add
10192 opcode(0x1);
10193
10194 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10195
10196 ins_pipe(ialu_reg_imm);
10197 %}
10198
10199 // Integer Negation (special case for sub)
10200
10201 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10202 match(Set dst (SubI zero src));
10203
10204 ins_cost(INSN_COST);
10205 format %{ "negw $dst, $src\t# int" %}
10206
10207 ins_encode %{
10208 __ negw(as_Register($dst$$reg),
10209 as_Register($src$$reg));
10210 %}
10211
10212 ins_pipe(ialu_reg);
10213 %}
10214
10215 // Long Negation
10216
10217 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10218 match(Set dst (SubL zero src));
10219
10220 ins_cost(INSN_COST);
10221 format %{ "neg $dst, $src\t# long" %}
10222
10223 ins_encode %{
10224 __ neg(as_Register($dst$$reg),
10225 as_Register($src$$reg));
10226 %}
10227
10228 ins_pipe(ialu_reg);
10229 %}
10230
10231 // Integer Multiply
10232
10233 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10234 match(Set dst (MulI src1 src2));
10235
10236 ins_cost(INSN_COST * 3);
10237 format %{ "mulw $dst, $src1, $src2" %}
10238
10239 ins_encode %{
10240 __ mulw(as_Register($dst$$reg),
10241 as_Register($src1$$reg),
10242 as_Register($src2$$reg));
10243 %}
10244
10245 ins_pipe(imul_reg_reg);
10246 %}
10247
10248 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10249 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10250
10251 ins_cost(INSN_COST * 3);
10252 format %{ "smull $dst, $src1, $src2" %}
10253
10254 ins_encode %{
10255 __ smull(as_Register($dst$$reg),
10256 as_Register($src1$$reg),
10257 as_Register($src2$$reg));
10258 %}
10259
10260 ins_pipe(imul_reg_reg);
10261 %}
10262
10263 // Long Multiply
10264
10265 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10266 match(Set dst (MulL src1 src2));
10267
10268 ins_cost(INSN_COST * 5);
10269 format %{ "mul $dst, $src1, $src2" %}
10270
10271 ins_encode %{
10272 __ mul(as_Register($dst$$reg),
10273 as_Register($src1$$reg),
10274 as_Register($src2$$reg));
10275 %}
10276
10277 ins_pipe(lmul_reg_reg);
10278 %}
10279
10280 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10281 %{
10282 match(Set dst (MulHiL src1 src2));
10283
10284 ins_cost(INSN_COST * 7);
10285 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10286
10287 ins_encode %{
10288 __ smulh(as_Register($dst$$reg),
10289 as_Register($src1$$reg),
10290 as_Register($src2$$reg));
10291 %}
10292
10293 ins_pipe(lmul_reg_reg);
10294 %}
10295
10296 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10297 %{
10298 match(Set dst (UMulHiL src1 src2));
10299
10300 ins_cost(INSN_COST * 7);
10301 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10302
10303 ins_encode %{
10304 __ umulh(as_Register($dst$$reg),
10305 as_Register($src1$$reg),
10306 as_Register($src2$$reg));
10307 %}
10308
10309 ins_pipe(lmul_reg_reg);
10310 %}
10311
10312 // Combined Integer Multiply & Add/Sub
10313
10314 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10315 match(Set dst (AddI src3 (MulI src1 src2)));
10316
10317 ins_cost(INSN_COST * 3);
10318 format %{ "madd $dst, $src1, $src2, $src3" %}
10319
10320 ins_encode %{
10321 __ maddw(as_Register($dst$$reg),
10322 as_Register($src1$$reg),
10323 as_Register($src2$$reg),
10324 as_Register($src3$$reg));
10325 %}
10326
10327 ins_pipe(imac_reg_reg);
10328 %}
10329
10330 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10331 match(Set dst (SubI src3 (MulI src1 src2)));
10332
10333 ins_cost(INSN_COST * 3);
10334 format %{ "msub $dst, $src1, $src2, $src3" %}
10335
10336 ins_encode %{
10337 __ msubw(as_Register($dst$$reg),
10338 as_Register($src1$$reg),
10339 as_Register($src2$$reg),
10340 as_Register($src3$$reg));
10341 %}
10342
10343 ins_pipe(imac_reg_reg);
10344 %}
10345
10346 // Combined Integer Multiply & Neg
10347
10348 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10349 match(Set dst (MulI (SubI zero src1) src2));
10350
10351 ins_cost(INSN_COST * 3);
10352 format %{ "mneg $dst, $src1, $src2" %}
10353
10354 ins_encode %{
10355 __ mnegw(as_Register($dst$$reg),
10356 as_Register($src1$$reg),
10357 as_Register($src2$$reg));
10358 %}
10359
10360 ins_pipe(imac_reg_reg);
10361 %}
10362
10363 // Combined Long Multiply & Add/Sub
10364
10365 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10366 match(Set dst (AddL src3 (MulL src1 src2)));
10367
10368 ins_cost(INSN_COST * 5);
10369 format %{ "madd $dst, $src1, $src2, $src3" %}
10370
10371 ins_encode %{
10372 __ madd(as_Register($dst$$reg),
10373 as_Register($src1$$reg),
10374 as_Register($src2$$reg),
10375 as_Register($src3$$reg));
10376 %}
10377
10378 ins_pipe(lmac_reg_reg);
10379 %}
10380
10381 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10382 match(Set dst (SubL src3 (MulL src1 src2)));
10383
10384 ins_cost(INSN_COST * 5);
10385 format %{ "msub $dst, $src1, $src2, $src3" %}
10386
10387 ins_encode %{
10388 __ msub(as_Register($dst$$reg),
10389 as_Register($src1$$reg),
10390 as_Register($src2$$reg),
10391 as_Register($src3$$reg));
10392 %}
10393
10394 ins_pipe(lmac_reg_reg);
10395 %}
10396
10397 // Combined Long Multiply & Neg
10398
10399 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10400 match(Set dst (MulL (SubL zero src1) src2));
10401
10402 ins_cost(INSN_COST * 5);
10403 format %{ "mneg $dst, $src1, $src2" %}
10404
10405 ins_encode %{
10406 __ mneg(as_Register($dst$$reg),
10407 as_Register($src1$$reg),
10408 as_Register($src2$$reg));
10409 %}
10410
10411 ins_pipe(lmac_reg_reg);
10412 %}
10413
10414 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10415
10416 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10417 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10418
10419 ins_cost(INSN_COST * 3);
10420 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10421
10422 ins_encode %{
10423 __ smaddl(as_Register($dst$$reg),
10424 as_Register($src1$$reg),
10425 as_Register($src2$$reg),
10426 as_Register($src3$$reg));
10427 %}
10428
10429 ins_pipe(imac_reg_reg);
10430 %}
10431
10432 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10433 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10434
10435 ins_cost(INSN_COST * 3);
10436 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10437
10438 ins_encode %{
10439 __ smsubl(as_Register($dst$$reg),
10440 as_Register($src1$$reg),
10441 as_Register($src2$$reg),
10442 as_Register($src3$$reg));
10443 %}
10444
10445 ins_pipe(imac_reg_reg);
10446 %}
10447
10448 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10449 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10450
10451 ins_cost(INSN_COST * 3);
10452 format %{ "smnegl $dst, $src1, $src2" %}
10453
10454 ins_encode %{
10455 __ smnegl(as_Register($dst$$reg),
10456 as_Register($src1$$reg),
10457 as_Register($src2$$reg));
10458 %}
10459
10460 ins_pipe(imac_reg_reg);
10461 %}
10462
10463 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10464
10465 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10466 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10467
10468 ins_cost(INSN_COST * 5);
10469 format %{ "mulw rscratch1, $src1, $src2\n\t"
10470 "maddw $dst, $src3, $src4, rscratch1" %}
10471
10472 ins_encode %{
10473 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10474 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10475
10476 ins_pipe(imac_reg_reg);
10477 %}
10478
10479 // Integer Divide
10480
10481 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10482 match(Set dst (DivI src1 src2));
10483
10484 ins_cost(INSN_COST * 19);
10485 format %{ "sdivw $dst, $src1, $src2" %}
10486
10487 ins_encode(aarch64_enc_divw(dst, src1, src2));
10488 ins_pipe(idiv_reg_reg);
10489 %}
10490
10491 // Long Divide
10492
10493 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10494 match(Set dst (DivL src1 src2));
10495
10496 ins_cost(INSN_COST * 35);
10497 format %{ "sdiv $dst, $src1, $src2" %}
10498
10499 ins_encode(aarch64_enc_div(dst, src1, src2));
10500 ins_pipe(ldiv_reg_reg);
10501 %}
10502
10503 // Integer Remainder
10504
10505 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10506 match(Set dst (ModI src1 src2));
10507
10508 ins_cost(INSN_COST * 22);
10509 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10510 "msubw $dst, rscratch1, $src2, $src1" %}
10511
10512 ins_encode(aarch64_enc_modw(dst, src1, src2));
10513 ins_pipe(idiv_reg_reg);
10514 %}
10515
10516 // Long Remainder
10517
10518 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10519 match(Set dst (ModL src1 src2));
10520
10521 ins_cost(INSN_COST * 38);
10522 format %{ "sdiv rscratch1, $src1, $src2\n"
10523 "msub $dst, rscratch1, $src2, $src1" %}
10524
10525 ins_encode(aarch64_enc_mod(dst, src1, src2));
10526 ins_pipe(ldiv_reg_reg);
10527 %}
10528
10529 // Unsigned Integer Divide
10530
10531 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10532 match(Set dst (UDivI src1 src2));
10533
10534 ins_cost(INSN_COST * 19);
10535 format %{ "udivw $dst, $src1, $src2" %}
10536
10537 ins_encode %{
10538 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10539 %}
10540
10541 ins_pipe(idiv_reg_reg);
10542 %}
10543
10544 // Unsigned Long Divide
10545
10546 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10547 match(Set dst (UDivL src1 src2));
10548
10549 ins_cost(INSN_COST * 35);
10550 format %{ "udiv $dst, $src1, $src2" %}
10551
10552 ins_encode %{
10553 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10554 %}
10555
10556 ins_pipe(ldiv_reg_reg);
10557 %}
10558
10559 // Unsigned Integer Remainder
10560
10561 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10562 match(Set dst (UModI src1 src2));
10563
10564 ins_cost(INSN_COST * 22);
10565 format %{ "udivw rscratch1, $src1, $src2\n\t"
10566 "msubw $dst, rscratch1, $src2, $src1" %}
10567
10568 ins_encode %{
10569 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10570 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10571 %}
10572
10573 ins_pipe(idiv_reg_reg);
10574 %}
10575
10576 // Unsigned Long Remainder
10577
10578 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10579 match(Set dst (UModL src1 src2));
10580
10581 ins_cost(INSN_COST * 38);
10582 format %{ "udiv rscratch1, $src1, $src2\n"
10583 "msub $dst, rscratch1, $src2, $src1" %}
10584
10585 ins_encode %{
10586 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10587 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10588 %}
10589
10590 ins_pipe(ldiv_reg_reg);
10591 %}
10592
10593 // Integer Shifts
10594
10595 // Shift Left Register
10596 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10597 match(Set dst (LShiftI src1 src2));
10598
10599 ins_cost(INSN_COST * 2);
10600 format %{ "lslvw $dst, $src1, $src2" %}
10601
10602 ins_encode %{
10603 __ lslvw(as_Register($dst$$reg),
10604 as_Register($src1$$reg),
10605 as_Register($src2$$reg));
10606 %}
10607
10608 ins_pipe(ialu_reg_reg_vshift);
10609 %}
10610
10611 // Shift Left Immediate
10612 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10613 match(Set dst (LShiftI src1 src2));
10614
10615 ins_cost(INSN_COST);
10616 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10617
10618 ins_encode %{
10619 __ lslw(as_Register($dst$$reg),
10620 as_Register($src1$$reg),
10621 $src2$$constant & 0x1f);
10622 %}
10623
10624 ins_pipe(ialu_reg_shift);
10625 %}
10626
10627 // Shift Right Logical Register
10628 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10629 match(Set dst (URShiftI src1 src2));
10630
10631 ins_cost(INSN_COST * 2);
10632 format %{ "lsrvw $dst, $src1, $src2" %}
10633
10634 ins_encode %{
10635 __ lsrvw(as_Register($dst$$reg),
10636 as_Register($src1$$reg),
10637 as_Register($src2$$reg));
10638 %}
10639
10640 ins_pipe(ialu_reg_reg_vshift);
10641 %}
10642
10643 // Shift Right Logical Immediate
10644 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10645 match(Set dst (URShiftI src1 src2));
10646
10647 ins_cost(INSN_COST);
10648 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10649
10650 ins_encode %{
10651 __ lsrw(as_Register($dst$$reg),
10652 as_Register($src1$$reg),
10653 $src2$$constant & 0x1f);
10654 %}
10655
10656 ins_pipe(ialu_reg_shift);
10657 %}
10658
10659 // Shift Right Arithmetic Register
10660 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10661 match(Set dst (RShiftI src1 src2));
10662
10663 ins_cost(INSN_COST * 2);
10664 format %{ "asrvw $dst, $src1, $src2" %}
10665
10666 ins_encode %{
10667 __ asrvw(as_Register($dst$$reg),
10668 as_Register($src1$$reg),
10669 as_Register($src2$$reg));
10670 %}
10671
10672 ins_pipe(ialu_reg_reg_vshift);
10673 %}
10674
10675 // Shift Right Arithmetic Immediate
10676 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10677 match(Set dst (RShiftI src1 src2));
10678
10679 ins_cost(INSN_COST);
10680 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10681
10682 ins_encode %{
10683 __ asrw(as_Register($dst$$reg),
10684 as_Register($src1$$reg),
10685 $src2$$constant & 0x1f);
10686 %}
10687
10688 ins_pipe(ialu_reg_shift);
10689 %}
10690
10691 // Combined Int Mask and Right Shift (using UBFM)
10692 // TODO
10693
10694 // Long Shifts
10695
10696 // Shift Left Register
10697 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10698 match(Set dst (LShiftL src1 src2));
10699
10700 ins_cost(INSN_COST * 2);
10701 format %{ "lslv $dst, $src1, $src2" %}
10702
10703 ins_encode %{
10704 __ lslv(as_Register($dst$$reg),
10705 as_Register($src1$$reg),
10706 as_Register($src2$$reg));
10707 %}
10708
10709 ins_pipe(ialu_reg_reg_vshift);
10710 %}
10711
10712 // Shift Left Immediate
10713 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10714 match(Set dst (LShiftL src1 src2));
10715
10716 ins_cost(INSN_COST);
10717 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10718
10719 ins_encode %{
10720 __ lsl(as_Register($dst$$reg),
10721 as_Register($src1$$reg),
10722 $src2$$constant & 0x3f);
10723 %}
10724
10725 ins_pipe(ialu_reg_shift);
10726 %}
10727
10728 // Shift Right Logical Register
10729 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10730 match(Set dst (URShiftL src1 src2));
10731
10732 ins_cost(INSN_COST * 2);
10733 format %{ "lsrv $dst, $src1, $src2" %}
10734
10735 ins_encode %{
10736 __ lsrv(as_Register($dst$$reg),
10737 as_Register($src1$$reg),
10738 as_Register($src2$$reg));
10739 %}
10740
10741 ins_pipe(ialu_reg_reg_vshift);
10742 %}
10743
10744 // Shift Right Logical Immediate
10745 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10746 match(Set dst (URShiftL src1 src2));
10747
10748 ins_cost(INSN_COST);
10749 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10750
10751 ins_encode %{
10752 __ lsr(as_Register($dst$$reg),
10753 as_Register($src1$$reg),
10754 $src2$$constant & 0x3f);
10755 %}
10756
10757 ins_pipe(ialu_reg_shift);
10758 %}
10759
10760 // A special-case pattern for card table stores.
10761 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10762 match(Set dst (URShiftL (CastP2X src1) src2));
10763
10764 ins_cost(INSN_COST);
10765 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10766
10767 ins_encode %{
10768 __ lsr(as_Register($dst$$reg),
10769 as_Register($src1$$reg),
10770 $src2$$constant & 0x3f);
10771 %}
10772
10773 ins_pipe(ialu_reg_shift);
10774 %}
10775
10776 // Shift Right Arithmetic Register
10777 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10778 match(Set dst (RShiftL src1 src2));
10779
10780 ins_cost(INSN_COST * 2);
10781 format %{ "asrv $dst, $src1, $src2" %}
10782
10783 ins_encode %{
10784 __ asrv(as_Register($dst$$reg),
10785 as_Register($src1$$reg),
10786 as_Register($src2$$reg));
10787 %}
10788
10789 ins_pipe(ialu_reg_reg_vshift);
10790 %}
10791
10792 // Shift Right Arithmetic Immediate
10793 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10794 match(Set dst (RShiftL src1 src2));
10795
10796 ins_cost(INSN_COST);
10797 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10798
10799 ins_encode %{
10800 __ asr(as_Register($dst$$reg),
10801 as_Register($src1$$reg),
10802 $src2$$constant & 0x3f);
10803 %}
10804
10805 ins_pipe(ialu_reg_shift);
10806 %}
10807
10808 // BEGIN This section of the file is automatically generated. Do not edit --------------
10809 // This section is generated from aarch64_ad.m4
10810
10811 // This pattern is automatically generated from aarch64_ad.m4
10812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10813 instruct regL_not_reg(iRegLNoSp dst,
10814 iRegL src1, immL_M1 m1,
10815 rFlagsReg cr) %{
10816 match(Set dst (XorL src1 m1));
10817 ins_cost(INSN_COST);
10818 format %{ "eon $dst, $src1, zr" %}
10819
10820 ins_encode %{
10821 __ eon(as_Register($dst$$reg),
10822 as_Register($src1$$reg),
10823 zr,
10824 Assembler::LSL, 0);
10825 %}
10826
10827 ins_pipe(ialu_reg);
10828 %}
10829
10830 // This pattern is automatically generated from aarch64_ad.m4
10831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10832 instruct regI_not_reg(iRegINoSp dst,
10833 iRegIorL2I src1, immI_M1 m1,
10834 rFlagsReg cr) %{
10835 match(Set dst (XorI src1 m1));
10836 ins_cost(INSN_COST);
10837 format %{ "eonw $dst, $src1, zr" %}
10838
10839 ins_encode %{
10840 __ eonw(as_Register($dst$$reg),
10841 as_Register($src1$$reg),
10842 zr,
10843 Assembler::LSL, 0);
10844 %}
10845
10846 ins_pipe(ialu_reg);
10847 %}
10848
10849 // This pattern is automatically generated from aarch64_ad.m4
10850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10851 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10852 immI0 zero, iRegIorL2I src1, immI src2) %{
10853 match(Set dst (SubI zero (URShiftI src1 src2)));
10854
10855 ins_cost(1.9 * INSN_COST);
10856 format %{ "negw $dst, $src1, LSR $src2" %}
10857
10858 ins_encode %{
10859 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10860 Assembler::LSR, $src2$$constant & 0x1f);
10861 %}
10862
10863 ins_pipe(ialu_reg_shift);
10864 %}
10865
10866 // This pattern is automatically generated from aarch64_ad.m4
10867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10868 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10869 immI0 zero, iRegIorL2I src1, immI src2) %{
10870 match(Set dst (SubI zero (RShiftI src1 src2)));
10871
10872 ins_cost(1.9 * INSN_COST);
10873 format %{ "negw $dst, $src1, ASR $src2" %}
10874
10875 ins_encode %{
10876 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10877 Assembler::ASR, $src2$$constant & 0x1f);
10878 %}
10879
10880 ins_pipe(ialu_reg_shift);
10881 %}
10882
10883 // This pattern is automatically generated from aarch64_ad.m4
10884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10885 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10886 immI0 zero, iRegIorL2I src1, immI src2) %{
10887 match(Set dst (SubI zero (LShiftI src1 src2)));
10888
10889 ins_cost(1.9 * INSN_COST);
10890 format %{ "negw $dst, $src1, LSL $src2" %}
10891
10892 ins_encode %{
10893 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10894 Assembler::LSL, $src2$$constant & 0x1f);
10895 %}
10896
10897 ins_pipe(ialu_reg_shift);
10898 %}
10899
10900 // This pattern is automatically generated from aarch64_ad.m4
10901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10902 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10903 immL0 zero, iRegL src1, immI src2) %{
10904 match(Set dst (SubL zero (URShiftL src1 src2)));
10905
10906 ins_cost(1.9 * INSN_COST);
10907 format %{ "neg $dst, $src1, LSR $src2" %}
10908
10909 ins_encode %{
10910 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10911 Assembler::LSR, $src2$$constant & 0x3f);
10912 %}
10913
10914 ins_pipe(ialu_reg_shift);
10915 %}
10916
10917 // This pattern is automatically generated from aarch64_ad.m4
10918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10919 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10920 immL0 zero, iRegL src1, immI src2) %{
10921 match(Set dst (SubL zero (RShiftL src1 src2)));
10922
10923 ins_cost(1.9 * INSN_COST);
10924 format %{ "neg $dst, $src1, ASR $src2" %}
10925
10926 ins_encode %{
10927 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10928 Assembler::ASR, $src2$$constant & 0x3f);
10929 %}
10930
10931 ins_pipe(ialu_reg_shift);
10932 %}
10933
10934 // This pattern is automatically generated from aarch64_ad.m4
10935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10936 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10937 immL0 zero, iRegL src1, immI src2) %{
10938 match(Set dst (SubL zero (LShiftL src1 src2)));
10939
10940 ins_cost(1.9 * INSN_COST);
10941 format %{ "neg $dst, $src1, LSL $src2" %}
10942
10943 ins_encode %{
10944 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10945 Assembler::LSL, $src2$$constant & 0x3f);
10946 %}
10947
10948 ins_pipe(ialu_reg_shift);
10949 %}
10950
10951 // This pattern is automatically generated from aarch64_ad.m4
10952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10953 instruct AndI_reg_not_reg(iRegINoSp dst,
10954 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10955 match(Set dst (AndI src1 (XorI src2 m1)));
10956 ins_cost(INSN_COST);
10957 format %{ "bicw $dst, $src1, $src2" %}
10958
10959 ins_encode %{
10960 __ bicw(as_Register($dst$$reg),
10961 as_Register($src1$$reg),
10962 as_Register($src2$$reg),
10963 Assembler::LSL, 0);
10964 %}
10965
10966 ins_pipe(ialu_reg_reg);
10967 %}
10968
10969 // This pattern is automatically generated from aarch64_ad.m4
10970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10971 instruct AndL_reg_not_reg(iRegLNoSp dst,
10972 iRegL src1, iRegL src2, immL_M1 m1) %{
10973 match(Set dst (AndL src1 (XorL src2 m1)));
10974 ins_cost(INSN_COST);
10975 format %{ "bic $dst, $src1, $src2" %}
10976
10977 ins_encode %{
10978 __ bic(as_Register($dst$$reg),
10979 as_Register($src1$$reg),
10980 as_Register($src2$$reg),
10981 Assembler::LSL, 0);
10982 %}
10983
10984 ins_pipe(ialu_reg_reg);
10985 %}
10986
10987 // This pattern is automatically generated from aarch64_ad.m4
10988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10989 instruct OrI_reg_not_reg(iRegINoSp dst,
10990 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10991 match(Set dst (OrI src1 (XorI src2 m1)));
10992 ins_cost(INSN_COST);
10993 format %{ "ornw $dst, $src1, $src2" %}
10994
10995 ins_encode %{
10996 __ ornw(as_Register($dst$$reg),
10997 as_Register($src1$$reg),
10998 as_Register($src2$$reg),
10999 Assembler::LSL, 0);
11000 %}
11001
11002 ins_pipe(ialu_reg_reg);
11003 %}
11004
11005 // This pattern is automatically generated from aarch64_ad.m4
11006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11007 instruct OrL_reg_not_reg(iRegLNoSp dst,
11008 iRegL src1, iRegL src2, immL_M1 m1) %{
11009 match(Set dst (OrL src1 (XorL src2 m1)));
11010 ins_cost(INSN_COST);
11011 format %{ "orn $dst, $src1, $src2" %}
11012
11013 ins_encode %{
11014 __ orn(as_Register($dst$$reg),
11015 as_Register($src1$$reg),
11016 as_Register($src2$$reg),
11017 Assembler::LSL, 0);
11018 %}
11019
11020 ins_pipe(ialu_reg_reg);
11021 %}
11022
11023 // This pattern is automatically generated from aarch64_ad.m4
11024 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11025 instruct XorI_reg_not_reg(iRegINoSp dst,
11026 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11027 match(Set dst (XorI m1 (XorI src2 src1)));
11028 ins_cost(INSN_COST);
11029 format %{ "eonw $dst, $src1, $src2" %}
11030
11031 ins_encode %{
11032 __ eonw(as_Register($dst$$reg),
11033 as_Register($src1$$reg),
11034 as_Register($src2$$reg),
11035 Assembler::LSL, 0);
11036 %}
11037
11038 ins_pipe(ialu_reg_reg);
11039 %}
11040
11041 // This pattern is automatically generated from aarch64_ad.m4
11042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11043 instruct XorL_reg_not_reg(iRegLNoSp dst,
11044 iRegL src1, iRegL src2, immL_M1 m1) %{
11045 match(Set dst (XorL m1 (XorL src2 src1)));
11046 ins_cost(INSN_COST);
11047 format %{ "eon $dst, $src1, $src2" %}
11048
11049 ins_encode %{
11050 __ eon(as_Register($dst$$reg),
11051 as_Register($src1$$reg),
11052 as_Register($src2$$reg),
11053 Assembler::LSL, 0);
11054 %}
11055
11056 ins_pipe(ialu_reg_reg);
11057 %}
11058
11059 // This pattern is automatically generated from aarch64_ad.m4
11060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11061 // val & (-1 ^ (val >>> shift)) ==> bicw
11062 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11063 iRegIorL2I src1, iRegIorL2I src2,
11064 immI src3, immI_M1 src4) %{
11065 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11066 ins_cost(1.9 * INSN_COST);
11067 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11068
11069 ins_encode %{
11070 __ bicw(as_Register($dst$$reg),
11071 as_Register($src1$$reg),
11072 as_Register($src2$$reg),
11073 Assembler::LSR,
11074 $src3$$constant & 0x1f);
11075 %}
11076
11077 ins_pipe(ialu_reg_reg_shift);
11078 %}
11079
11080 // This pattern is automatically generated from aarch64_ad.m4
11081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11082 // val & (-1 ^ (val >>> shift)) ==> bic
11083 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11084 iRegL src1, iRegL src2,
11085 immI src3, immL_M1 src4) %{
11086 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11087 ins_cost(1.9 * INSN_COST);
11088 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11089
11090 ins_encode %{
11091 __ bic(as_Register($dst$$reg),
11092 as_Register($src1$$reg),
11093 as_Register($src2$$reg),
11094 Assembler::LSR,
11095 $src3$$constant & 0x3f);
11096 %}
11097
11098 ins_pipe(ialu_reg_reg_shift);
11099 %}
11100
11101 // This pattern is automatically generated from aarch64_ad.m4
11102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11103 // val & (-1 ^ (val >> shift)) ==> bicw
11104 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11105 iRegIorL2I src1, iRegIorL2I src2,
11106 immI src3, immI_M1 src4) %{
11107 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11108 ins_cost(1.9 * INSN_COST);
11109 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11110
11111 ins_encode %{
11112 __ bicw(as_Register($dst$$reg),
11113 as_Register($src1$$reg),
11114 as_Register($src2$$reg),
11115 Assembler::ASR,
11116 $src3$$constant & 0x1f);
11117 %}
11118
11119 ins_pipe(ialu_reg_reg_shift);
11120 %}
11121
11122 // This pattern is automatically generated from aarch64_ad.m4
11123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11124 // val & (-1 ^ (val >> shift)) ==> bic
11125 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11126 iRegL src1, iRegL src2,
11127 immI src3, immL_M1 src4) %{
11128 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11129 ins_cost(1.9 * INSN_COST);
11130 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11131
11132 ins_encode %{
11133 __ bic(as_Register($dst$$reg),
11134 as_Register($src1$$reg),
11135 as_Register($src2$$reg),
11136 Assembler::ASR,
11137 $src3$$constant & 0x3f);
11138 %}
11139
11140 ins_pipe(ialu_reg_reg_shift);
11141 %}
11142
11143 // This pattern is automatically generated from aarch64_ad.m4
11144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11145 // val & (-1 ^ (val ror shift)) ==> bicw
11146 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11147 iRegIorL2I src1, iRegIorL2I src2,
11148 immI src3, immI_M1 src4) %{
11149 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11150 ins_cost(1.9 * INSN_COST);
11151 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11152
11153 ins_encode %{
11154 __ bicw(as_Register($dst$$reg),
11155 as_Register($src1$$reg),
11156 as_Register($src2$$reg),
11157 Assembler::ROR,
11158 $src3$$constant & 0x1f);
11159 %}
11160
11161 ins_pipe(ialu_reg_reg_shift);
11162 %}
11163
11164 // This pattern is automatically generated from aarch64_ad.m4
11165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11166 // val & (-1 ^ (val ror shift)) ==> bic
11167 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11168 iRegL src1, iRegL src2,
11169 immI src3, immL_M1 src4) %{
11170 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11171 ins_cost(1.9 * INSN_COST);
11172 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11173
11174 ins_encode %{
11175 __ bic(as_Register($dst$$reg),
11176 as_Register($src1$$reg),
11177 as_Register($src2$$reg),
11178 Assembler::ROR,
11179 $src3$$constant & 0x3f);
11180 %}
11181
11182 ins_pipe(ialu_reg_reg_shift);
11183 %}
11184
11185 // This pattern is automatically generated from aarch64_ad.m4
11186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11187 // val & (-1 ^ (val << shift)) ==> bicw
11188 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11189 iRegIorL2I src1, iRegIorL2I src2,
11190 immI src3, immI_M1 src4) %{
11191 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11192 ins_cost(1.9 * INSN_COST);
11193 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11194
11195 ins_encode %{
11196 __ bicw(as_Register($dst$$reg),
11197 as_Register($src1$$reg),
11198 as_Register($src2$$reg),
11199 Assembler::LSL,
11200 $src3$$constant & 0x1f);
11201 %}
11202
11203 ins_pipe(ialu_reg_reg_shift);
11204 %}
11205
11206 // This pattern is automatically generated from aarch64_ad.m4
11207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11208 // val & (-1 ^ (val << shift)) ==> bic
11209 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11210 iRegL src1, iRegL src2,
11211 immI src3, immL_M1 src4) %{
11212 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11213 ins_cost(1.9 * INSN_COST);
11214 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11215
11216 ins_encode %{
11217 __ bic(as_Register($dst$$reg),
11218 as_Register($src1$$reg),
11219 as_Register($src2$$reg),
11220 Assembler::LSL,
11221 $src3$$constant & 0x3f);
11222 %}
11223
11224 ins_pipe(ialu_reg_reg_shift);
11225 %}
11226
11227 // This pattern is automatically generated from aarch64_ad.m4
11228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11229 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11230 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11231 iRegIorL2I src1, iRegIorL2I src2,
11232 immI src3, immI_M1 src4) %{
11233 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11234 ins_cost(1.9 * INSN_COST);
11235 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11236
11237 ins_encode %{
11238 __ eonw(as_Register($dst$$reg),
11239 as_Register($src1$$reg),
11240 as_Register($src2$$reg),
11241 Assembler::LSR,
11242 $src3$$constant & 0x1f);
11243 %}
11244
11245 ins_pipe(ialu_reg_reg_shift);
11246 %}
11247
11248 // This pattern is automatically generated from aarch64_ad.m4
11249 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11250 // val ^ (-1 ^ (val >>> shift)) ==> eon
11251 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11252 iRegL src1, iRegL src2,
11253 immI src3, immL_M1 src4) %{
11254 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11255 ins_cost(1.9 * INSN_COST);
11256 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11257
11258 ins_encode %{
11259 __ eon(as_Register($dst$$reg),
11260 as_Register($src1$$reg),
11261 as_Register($src2$$reg),
11262 Assembler::LSR,
11263 $src3$$constant & 0x3f);
11264 %}
11265
11266 ins_pipe(ialu_reg_reg_shift);
11267 %}
11268
11269 // This pattern is automatically generated from aarch64_ad.m4
11270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11271 // val ^ (-1 ^ (val >> shift)) ==> eonw
11272 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11273 iRegIorL2I src1, iRegIorL2I src2,
11274 immI src3, immI_M1 src4) %{
11275 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11276 ins_cost(1.9 * INSN_COST);
11277 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11278
11279 ins_encode %{
11280 __ eonw(as_Register($dst$$reg),
11281 as_Register($src1$$reg),
11282 as_Register($src2$$reg),
11283 Assembler::ASR,
11284 $src3$$constant & 0x1f);
11285 %}
11286
11287 ins_pipe(ialu_reg_reg_shift);
11288 %}
11289
11290 // This pattern is automatically generated from aarch64_ad.m4
11291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11292 // val ^ (-1 ^ (val >> shift)) ==> eon
11293 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11294 iRegL src1, iRegL src2,
11295 immI src3, immL_M1 src4) %{
11296 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11297 ins_cost(1.9 * INSN_COST);
11298 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11299
11300 ins_encode %{
11301 __ eon(as_Register($dst$$reg),
11302 as_Register($src1$$reg),
11303 as_Register($src2$$reg),
11304 Assembler::ASR,
11305 $src3$$constant & 0x3f);
11306 %}
11307
11308 ins_pipe(ialu_reg_reg_shift);
11309 %}
11310
11311 // This pattern is automatically generated from aarch64_ad.m4
11312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11313 // val ^ (-1 ^ (val ror shift)) ==> eonw
11314 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11315 iRegIorL2I src1, iRegIorL2I src2,
11316 immI src3, immI_M1 src4) %{
11317 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11318 ins_cost(1.9 * INSN_COST);
11319 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11320
11321 ins_encode %{
11322 __ eonw(as_Register($dst$$reg),
11323 as_Register($src1$$reg),
11324 as_Register($src2$$reg),
11325 Assembler::ROR,
11326 $src3$$constant & 0x1f);
11327 %}
11328
11329 ins_pipe(ialu_reg_reg_shift);
11330 %}
11331
11332 // This pattern is automatically generated from aarch64_ad.m4
11333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11334 // val ^ (-1 ^ (val ror shift)) ==> eon
11335 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11336 iRegL src1, iRegL src2,
11337 immI src3, immL_M1 src4) %{
11338 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11339 ins_cost(1.9 * INSN_COST);
11340 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11341
11342 ins_encode %{
11343 __ eon(as_Register($dst$$reg),
11344 as_Register($src1$$reg),
11345 as_Register($src2$$reg),
11346 Assembler::ROR,
11347 $src3$$constant & 0x3f);
11348 %}
11349
11350 ins_pipe(ialu_reg_reg_shift);
11351 %}
11352
11353 // This pattern is automatically generated from aarch64_ad.m4
11354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11355 // val ^ (-1 ^ (val << shift)) ==> eonw
11356 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11357 iRegIorL2I src1, iRegIorL2I src2,
11358 immI src3, immI_M1 src4) %{
11359 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11360 ins_cost(1.9 * INSN_COST);
11361 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11362
11363 ins_encode %{
11364 __ eonw(as_Register($dst$$reg),
11365 as_Register($src1$$reg),
11366 as_Register($src2$$reg),
11367 Assembler::LSL,
11368 $src3$$constant & 0x1f);
11369 %}
11370
11371 ins_pipe(ialu_reg_reg_shift);
11372 %}
11373
11374 // This pattern is automatically generated from aarch64_ad.m4
11375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11376 // val ^ (-1 ^ (val << shift)) ==> eon
11377 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11378 iRegL src1, iRegL src2,
11379 immI src3, immL_M1 src4) %{
11380 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11381 ins_cost(1.9 * INSN_COST);
11382 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11383
11384 ins_encode %{
11385 __ eon(as_Register($dst$$reg),
11386 as_Register($src1$$reg),
11387 as_Register($src2$$reg),
11388 Assembler::LSL,
11389 $src3$$constant & 0x3f);
11390 %}
11391
11392 ins_pipe(ialu_reg_reg_shift);
11393 %}
11394
11395 // This pattern is automatically generated from aarch64_ad.m4
11396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11397 // val | (-1 ^ (val >>> shift)) ==> ornw
11398 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11399 iRegIorL2I src1, iRegIorL2I src2,
11400 immI src3, immI_M1 src4) %{
11401 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11402 ins_cost(1.9 * INSN_COST);
11403 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11404
11405 ins_encode %{
11406 __ ornw(as_Register($dst$$reg),
11407 as_Register($src1$$reg),
11408 as_Register($src2$$reg),
11409 Assembler::LSR,
11410 $src3$$constant & 0x1f);
11411 %}
11412
11413 ins_pipe(ialu_reg_reg_shift);
11414 %}
11415
11416 // This pattern is automatically generated from aarch64_ad.m4
11417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11418 // val | (-1 ^ (val >>> shift)) ==> orn
11419 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11420 iRegL src1, iRegL src2,
11421 immI src3, immL_M1 src4) %{
11422 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11423 ins_cost(1.9 * INSN_COST);
11424 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11425
11426 ins_encode %{
11427 __ orn(as_Register($dst$$reg),
11428 as_Register($src1$$reg),
11429 as_Register($src2$$reg),
11430 Assembler::LSR,
11431 $src3$$constant & 0x3f);
11432 %}
11433
11434 ins_pipe(ialu_reg_reg_shift);
11435 %}
11436
11437 // This pattern is automatically generated from aarch64_ad.m4
11438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11439 // val | (-1 ^ (val >> shift)) ==> ornw
11440 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11441 iRegIorL2I src1, iRegIorL2I src2,
11442 immI src3, immI_M1 src4) %{
11443 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11444 ins_cost(1.9 * INSN_COST);
11445 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11446
11447 ins_encode %{
11448 __ ornw(as_Register($dst$$reg),
11449 as_Register($src1$$reg),
11450 as_Register($src2$$reg),
11451 Assembler::ASR,
11452 $src3$$constant & 0x1f);
11453 %}
11454
11455 ins_pipe(ialu_reg_reg_shift);
11456 %}
11457
11458 // This pattern is automatically generated from aarch64_ad.m4
11459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11460 // val | (-1 ^ (val >> shift)) ==> orn
11461 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11462 iRegL src1, iRegL src2,
11463 immI src3, immL_M1 src4) %{
11464 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11465 ins_cost(1.9 * INSN_COST);
11466 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11467
11468 ins_encode %{
11469 __ orn(as_Register($dst$$reg),
11470 as_Register($src1$$reg),
11471 as_Register($src2$$reg),
11472 Assembler::ASR,
11473 $src3$$constant & 0x3f);
11474 %}
11475
11476 ins_pipe(ialu_reg_reg_shift);
11477 %}
11478
11479 // This pattern is automatically generated from aarch64_ad.m4
11480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11481 // val | (-1 ^ (val ror shift)) ==> ornw
11482 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11483 iRegIorL2I src1, iRegIorL2I src2,
11484 immI src3, immI_M1 src4) %{
11485 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11486 ins_cost(1.9 * INSN_COST);
11487 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11488
11489 ins_encode %{
11490 __ ornw(as_Register($dst$$reg),
11491 as_Register($src1$$reg),
11492 as_Register($src2$$reg),
11493 Assembler::ROR,
11494 $src3$$constant & 0x1f);
11495 %}
11496
11497 ins_pipe(ialu_reg_reg_shift);
11498 %}
11499
11500 // This pattern is automatically generated from aarch64_ad.m4
11501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11502 // val | (-1 ^ (val ror shift)) ==> orn
11503 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11504 iRegL src1, iRegL src2,
11505 immI src3, immL_M1 src4) %{
11506 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11507 ins_cost(1.9 * INSN_COST);
11508 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11509
11510 ins_encode %{
11511 __ orn(as_Register($dst$$reg),
11512 as_Register($src1$$reg),
11513 as_Register($src2$$reg),
11514 Assembler::ROR,
11515 $src3$$constant & 0x3f);
11516 %}
11517
11518 ins_pipe(ialu_reg_reg_shift);
11519 %}
11520
11521 // This pattern is automatically generated from aarch64_ad.m4
11522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11523 // val | (-1 ^ (val << shift)) ==> ornw
11524 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11525 iRegIorL2I src1, iRegIorL2I src2,
11526 immI src3, immI_M1 src4) %{
11527 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11528 ins_cost(1.9 * INSN_COST);
11529 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11530
11531 ins_encode %{
11532 __ ornw(as_Register($dst$$reg),
11533 as_Register($src1$$reg),
11534 as_Register($src2$$reg),
11535 Assembler::LSL,
11536 $src3$$constant & 0x1f);
11537 %}
11538
11539 ins_pipe(ialu_reg_reg_shift);
11540 %}
11541
11542 // This pattern is automatically generated from aarch64_ad.m4
11543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11544 // val | (-1 ^ (val << shift)) ==> orn
11545 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11546 iRegL src1, iRegL src2,
11547 immI src3, immL_M1 src4) %{
11548 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11549 ins_cost(1.9 * INSN_COST);
11550 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11551
11552 ins_encode %{
11553 __ orn(as_Register($dst$$reg),
11554 as_Register($src1$$reg),
11555 as_Register($src2$$reg),
11556 Assembler::LSL,
11557 $src3$$constant & 0x3f);
11558 %}
11559
11560 ins_pipe(ialu_reg_reg_shift);
11561 %}
11562
11563 // This pattern is automatically generated from aarch64_ad.m4
11564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11565 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11566 iRegIorL2I src1, iRegIorL2I src2,
11567 immI src3) %{
11568 match(Set dst (AndI src1 (URShiftI src2 src3)));
11569
11570 ins_cost(1.9 * INSN_COST);
11571 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11572
11573 ins_encode %{
11574 __ andw(as_Register($dst$$reg),
11575 as_Register($src1$$reg),
11576 as_Register($src2$$reg),
11577 Assembler::LSR,
11578 $src3$$constant & 0x1f);
11579 %}
11580
11581 ins_pipe(ialu_reg_reg_shift);
11582 %}
11583
11584 // This pattern is automatically generated from aarch64_ad.m4
11585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11586 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11587 iRegL src1, iRegL src2,
11588 immI src3) %{
11589 match(Set dst (AndL src1 (URShiftL src2 src3)));
11590
11591 ins_cost(1.9 * INSN_COST);
11592 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11593
11594 ins_encode %{
11595 __ andr(as_Register($dst$$reg),
11596 as_Register($src1$$reg),
11597 as_Register($src2$$reg),
11598 Assembler::LSR,
11599 $src3$$constant & 0x3f);
11600 %}
11601
11602 ins_pipe(ialu_reg_reg_shift);
11603 %}
11604
11605 // This pattern is automatically generated from aarch64_ad.m4
11606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11607 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11608 iRegIorL2I src1, iRegIorL2I src2,
11609 immI src3) %{
11610 match(Set dst (AndI src1 (RShiftI src2 src3)));
11611
11612 ins_cost(1.9 * INSN_COST);
11613 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11614
11615 ins_encode %{
11616 __ andw(as_Register($dst$$reg),
11617 as_Register($src1$$reg),
11618 as_Register($src2$$reg),
11619 Assembler::ASR,
11620 $src3$$constant & 0x1f);
11621 %}
11622
11623 ins_pipe(ialu_reg_reg_shift);
11624 %}
11625
11626 // This pattern is automatically generated from aarch64_ad.m4
11627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11628 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11629 iRegL src1, iRegL src2,
11630 immI src3) %{
11631 match(Set dst (AndL src1 (RShiftL src2 src3)));
11632
11633 ins_cost(1.9 * INSN_COST);
11634 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11635
11636 ins_encode %{
11637 __ andr(as_Register($dst$$reg),
11638 as_Register($src1$$reg),
11639 as_Register($src2$$reg),
11640 Assembler::ASR,
11641 $src3$$constant & 0x3f);
11642 %}
11643
11644 ins_pipe(ialu_reg_reg_shift);
11645 %}
11646
11647 // This pattern is automatically generated from aarch64_ad.m4
11648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11649 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11650 iRegIorL2I src1, iRegIorL2I src2,
11651 immI src3) %{
11652 match(Set dst (AndI src1 (LShiftI src2 src3)));
11653
11654 ins_cost(1.9 * INSN_COST);
11655 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11656
11657 ins_encode %{
11658 __ andw(as_Register($dst$$reg),
11659 as_Register($src1$$reg),
11660 as_Register($src2$$reg),
11661 Assembler::LSL,
11662 $src3$$constant & 0x1f);
11663 %}
11664
11665 ins_pipe(ialu_reg_reg_shift);
11666 %}
11667
11668 // This pattern is automatically generated from aarch64_ad.m4
11669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11670 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11671 iRegL src1, iRegL src2,
11672 immI src3) %{
11673 match(Set dst (AndL src1 (LShiftL src2 src3)));
11674
11675 ins_cost(1.9 * INSN_COST);
11676 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11677
11678 ins_encode %{
11679 __ andr(as_Register($dst$$reg),
11680 as_Register($src1$$reg),
11681 as_Register($src2$$reg),
11682 Assembler::LSL,
11683 $src3$$constant & 0x3f);
11684 %}
11685
11686 ins_pipe(ialu_reg_reg_shift);
11687 %}
11688
11689 // This pattern is automatically generated from aarch64_ad.m4
11690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11691 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11692 iRegIorL2I src1, iRegIorL2I src2,
11693 immI src3) %{
11694 match(Set dst (AndI src1 (RotateRight src2 src3)));
11695
11696 ins_cost(1.9 * INSN_COST);
11697 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11698
11699 ins_encode %{
11700 __ andw(as_Register($dst$$reg),
11701 as_Register($src1$$reg),
11702 as_Register($src2$$reg),
11703 Assembler::ROR,
11704 $src3$$constant & 0x1f);
11705 %}
11706
11707 ins_pipe(ialu_reg_reg_shift);
11708 %}
11709
11710 // This pattern is automatically generated from aarch64_ad.m4
11711 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11712 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11713 iRegL src1, iRegL src2,
11714 immI src3) %{
11715 match(Set dst (AndL src1 (RotateRight src2 src3)));
11716
11717 ins_cost(1.9 * INSN_COST);
11718 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11719
11720 ins_encode %{
11721 __ andr(as_Register($dst$$reg),
11722 as_Register($src1$$reg),
11723 as_Register($src2$$reg),
11724 Assembler::ROR,
11725 $src3$$constant & 0x3f);
11726 %}
11727
11728 ins_pipe(ialu_reg_reg_shift);
11729 %}
11730
11731 // This pattern is automatically generated from aarch64_ad.m4
11732 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11733 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11734 iRegIorL2I src1, iRegIorL2I src2,
11735 immI src3) %{
11736 match(Set dst (XorI src1 (URShiftI src2 src3)));
11737
11738 ins_cost(1.9 * INSN_COST);
11739 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11740
11741 ins_encode %{
11742 __ eorw(as_Register($dst$$reg),
11743 as_Register($src1$$reg),
11744 as_Register($src2$$reg),
11745 Assembler::LSR,
11746 $src3$$constant & 0x1f);
11747 %}
11748
11749 ins_pipe(ialu_reg_reg_shift);
11750 %}
11751
11752 // This pattern is automatically generated from aarch64_ad.m4
11753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11754 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11755 iRegL src1, iRegL src2,
11756 immI src3) %{
11757 match(Set dst (XorL src1 (URShiftL src2 src3)));
11758
11759 ins_cost(1.9 * INSN_COST);
11760 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11761
11762 ins_encode %{
11763 __ eor(as_Register($dst$$reg),
11764 as_Register($src1$$reg),
11765 as_Register($src2$$reg),
11766 Assembler::LSR,
11767 $src3$$constant & 0x3f);
11768 %}
11769
11770 ins_pipe(ialu_reg_reg_shift);
11771 %}
11772
11773 // This pattern is automatically generated from aarch64_ad.m4
11774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11775 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11776 iRegIorL2I src1, iRegIorL2I src2,
11777 immI src3) %{
11778 match(Set dst (XorI src1 (RShiftI src2 src3)));
11779
11780 ins_cost(1.9 * INSN_COST);
11781 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11782
11783 ins_encode %{
11784 __ eorw(as_Register($dst$$reg),
11785 as_Register($src1$$reg),
11786 as_Register($src2$$reg),
11787 Assembler::ASR,
11788 $src3$$constant & 0x1f);
11789 %}
11790
11791 ins_pipe(ialu_reg_reg_shift);
11792 %}
11793
11794 // This pattern is automatically generated from aarch64_ad.m4
11795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11796 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11797 iRegL src1, iRegL src2,
11798 immI src3) %{
11799 match(Set dst (XorL src1 (RShiftL src2 src3)));
11800
11801 ins_cost(1.9 * INSN_COST);
11802 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11803
11804 ins_encode %{
11805 __ eor(as_Register($dst$$reg),
11806 as_Register($src1$$reg),
11807 as_Register($src2$$reg),
11808 Assembler::ASR,
11809 $src3$$constant & 0x3f);
11810 %}
11811
11812 ins_pipe(ialu_reg_reg_shift);
11813 %}
11814
11815 // This pattern is automatically generated from aarch64_ad.m4
11816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11817 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11818 iRegIorL2I src1, iRegIorL2I src2,
11819 immI src3) %{
11820 match(Set dst (XorI src1 (LShiftI src2 src3)));
11821
11822 ins_cost(1.9 * INSN_COST);
11823 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11824
11825 ins_encode %{
11826 __ eorw(as_Register($dst$$reg),
11827 as_Register($src1$$reg),
11828 as_Register($src2$$reg),
11829 Assembler::LSL,
11830 $src3$$constant & 0x1f);
11831 %}
11832
11833 ins_pipe(ialu_reg_reg_shift);
11834 %}
11835
11836 // This pattern is automatically generated from aarch64_ad.m4
11837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11838 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11839 iRegL src1, iRegL src2,
11840 immI src3) %{
11841 match(Set dst (XorL src1 (LShiftL src2 src3)));
11842
11843 ins_cost(1.9 * INSN_COST);
11844 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11845
11846 ins_encode %{
11847 __ eor(as_Register($dst$$reg),
11848 as_Register($src1$$reg),
11849 as_Register($src2$$reg),
11850 Assembler::LSL,
11851 $src3$$constant & 0x3f);
11852 %}
11853
11854 ins_pipe(ialu_reg_reg_shift);
11855 %}
11856
11857 // This pattern is automatically generated from aarch64_ad.m4
11858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11859 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11860 iRegIorL2I src1, iRegIorL2I src2,
11861 immI src3) %{
11862 match(Set dst (XorI src1 (RotateRight src2 src3)));
11863
11864 ins_cost(1.9 * INSN_COST);
11865 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11866
11867 ins_encode %{
11868 __ eorw(as_Register($dst$$reg),
11869 as_Register($src1$$reg),
11870 as_Register($src2$$reg),
11871 Assembler::ROR,
11872 $src3$$constant & 0x1f);
11873 %}
11874
11875 ins_pipe(ialu_reg_reg_shift);
11876 %}
11877
11878 // This pattern is automatically generated from aarch64_ad.m4
11879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11880 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11881 iRegL src1, iRegL src2,
11882 immI src3) %{
11883 match(Set dst (XorL src1 (RotateRight src2 src3)));
11884
11885 ins_cost(1.9 * INSN_COST);
11886 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11887
11888 ins_encode %{
11889 __ eor(as_Register($dst$$reg),
11890 as_Register($src1$$reg),
11891 as_Register($src2$$reg),
11892 Assembler::ROR,
11893 $src3$$constant & 0x3f);
11894 %}
11895
11896 ins_pipe(ialu_reg_reg_shift);
11897 %}
11898
11899 // This pattern is automatically generated from aarch64_ad.m4
11900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11901 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11902 iRegIorL2I src1, iRegIorL2I src2,
11903 immI src3) %{
11904 match(Set dst (OrI src1 (URShiftI src2 src3)));
11905
11906 ins_cost(1.9 * INSN_COST);
11907 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11908
11909 ins_encode %{
11910 __ orrw(as_Register($dst$$reg),
11911 as_Register($src1$$reg),
11912 as_Register($src2$$reg),
11913 Assembler::LSR,
11914 $src3$$constant & 0x1f);
11915 %}
11916
11917 ins_pipe(ialu_reg_reg_shift);
11918 %}
11919
11920 // This pattern is automatically generated from aarch64_ad.m4
11921 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11922 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11923 iRegL src1, iRegL src2,
11924 immI src3) %{
11925 match(Set dst (OrL src1 (URShiftL src2 src3)));
11926
11927 ins_cost(1.9 * INSN_COST);
11928 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11929
11930 ins_encode %{
11931 __ orr(as_Register($dst$$reg),
11932 as_Register($src1$$reg),
11933 as_Register($src2$$reg),
11934 Assembler::LSR,
11935 $src3$$constant & 0x3f);
11936 %}
11937
11938 ins_pipe(ialu_reg_reg_shift);
11939 %}
11940
11941 // This pattern is automatically generated from aarch64_ad.m4
11942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11943 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11944 iRegIorL2I src1, iRegIorL2I src2,
11945 immI src3) %{
11946 match(Set dst (OrI src1 (RShiftI src2 src3)));
11947
11948 ins_cost(1.9 * INSN_COST);
11949 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11950
11951 ins_encode %{
11952 __ orrw(as_Register($dst$$reg),
11953 as_Register($src1$$reg),
11954 as_Register($src2$$reg),
11955 Assembler::ASR,
11956 $src3$$constant & 0x1f);
11957 %}
11958
11959 ins_pipe(ialu_reg_reg_shift);
11960 %}
11961
11962 // This pattern is automatically generated from aarch64_ad.m4
11963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11964 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11965 iRegL src1, iRegL src2,
11966 immI src3) %{
11967 match(Set dst (OrL src1 (RShiftL src2 src3)));
11968
11969 ins_cost(1.9 * INSN_COST);
11970 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11971
11972 ins_encode %{
11973 __ orr(as_Register($dst$$reg),
11974 as_Register($src1$$reg),
11975 as_Register($src2$$reg),
11976 Assembler::ASR,
11977 $src3$$constant & 0x3f);
11978 %}
11979
11980 ins_pipe(ialu_reg_reg_shift);
11981 %}
11982
11983 // This pattern is automatically generated from aarch64_ad.m4
11984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11985 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11986 iRegIorL2I src1, iRegIorL2I src2,
11987 immI src3) %{
11988 match(Set dst (OrI src1 (LShiftI src2 src3)));
11989
11990 ins_cost(1.9 * INSN_COST);
11991 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11992
11993 ins_encode %{
11994 __ orrw(as_Register($dst$$reg),
11995 as_Register($src1$$reg),
11996 as_Register($src2$$reg),
11997 Assembler::LSL,
11998 $src3$$constant & 0x1f);
11999 %}
12000
12001 ins_pipe(ialu_reg_reg_shift);
12002 %}
12003
12004 // This pattern is automatically generated from aarch64_ad.m4
12005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12006 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12007 iRegL src1, iRegL src2,
12008 immI src3) %{
12009 match(Set dst (OrL src1 (LShiftL src2 src3)));
12010
12011 ins_cost(1.9 * INSN_COST);
12012 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12013
12014 ins_encode %{
12015 __ orr(as_Register($dst$$reg),
12016 as_Register($src1$$reg),
12017 as_Register($src2$$reg),
12018 Assembler::LSL,
12019 $src3$$constant & 0x3f);
12020 %}
12021
12022 ins_pipe(ialu_reg_reg_shift);
12023 %}
12024
12025 // This pattern is automatically generated from aarch64_ad.m4
12026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12027 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12028 iRegIorL2I src1, iRegIorL2I src2,
12029 immI src3) %{
12030 match(Set dst (OrI src1 (RotateRight src2 src3)));
12031
12032 ins_cost(1.9 * INSN_COST);
12033 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12034
12035 ins_encode %{
12036 __ orrw(as_Register($dst$$reg),
12037 as_Register($src1$$reg),
12038 as_Register($src2$$reg),
12039 Assembler::ROR,
12040 $src3$$constant & 0x1f);
12041 %}
12042
12043 ins_pipe(ialu_reg_reg_shift);
12044 %}
12045
12046 // This pattern is automatically generated from aarch64_ad.m4
12047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12048 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12049 iRegL src1, iRegL src2,
12050 immI src3) %{
12051 match(Set dst (OrL src1 (RotateRight src2 src3)));
12052
12053 ins_cost(1.9 * INSN_COST);
12054 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12055
12056 ins_encode %{
12057 __ orr(as_Register($dst$$reg),
12058 as_Register($src1$$reg),
12059 as_Register($src2$$reg),
12060 Assembler::ROR,
12061 $src3$$constant & 0x3f);
12062 %}
12063
12064 ins_pipe(ialu_reg_reg_shift);
12065 %}
12066
12067 // This pattern is automatically generated from aarch64_ad.m4
12068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12069 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12070 iRegIorL2I src1, iRegIorL2I src2,
12071 immI src3) %{
12072 match(Set dst (AddI src1 (URShiftI src2 src3)));
12073
12074 ins_cost(1.9 * INSN_COST);
12075 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12076
12077 ins_encode %{
12078 __ addw(as_Register($dst$$reg),
12079 as_Register($src1$$reg),
12080 as_Register($src2$$reg),
12081 Assembler::LSR,
12082 $src3$$constant & 0x1f);
12083 %}
12084
12085 ins_pipe(ialu_reg_reg_shift);
12086 %}
12087
12088 // This pattern is automatically generated from aarch64_ad.m4
12089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12090 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12091 iRegL src1, iRegL src2,
12092 immI src3) %{
12093 match(Set dst (AddL src1 (URShiftL src2 src3)));
12094
12095 ins_cost(1.9 * INSN_COST);
12096 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12097
12098 ins_encode %{
12099 __ add(as_Register($dst$$reg),
12100 as_Register($src1$$reg),
12101 as_Register($src2$$reg),
12102 Assembler::LSR,
12103 $src3$$constant & 0x3f);
12104 %}
12105
12106 ins_pipe(ialu_reg_reg_shift);
12107 %}
12108
12109 // This pattern is automatically generated from aarch64_ad.m4
12110 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12111 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12112 iRegIorL2I src1, iRegIorL2I src2,
12113 immI src3) %{
12114 match(Set dst (AddI src1 (RShiftI src2 src3)));
12115
12116 ins_cost(1.9 * INSN_COST);
12117 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12118
12119 ins_encode %{
12120 __ addw(as_Register($dst$$reg),
12121 as_Register($src1$$reg),
12122 as_Register($src2$$reg),
12123 Assembler::ASR,
12124 $src3$$constant & 0x1f);
12125 %}
12126
12127 ins_pipe(ialu_reg_reg_shift);
12128 %}
12129
12130 // This pattern is automatically generated from aarch64_ad.m4
12131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12132 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12133 iRegL src1, iRegL src2,
12134 immI src3) %{
12135 match(Set dst (AddL src1 (RShiftL src2 src3)));
12136
12137 ins_cost(1.9 * INSN_COST);
12138 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12139
12140 ins_encode %{
12141 __ add(as_Register($dst$$reg),
12142 as_Register($src1$$reg),
12143 as_Register($src2$$reg),
12144 Assembler::ASR,
12145 $src3$$constant & 0x3f);
12146 %}
12147
12148 ins_pipe(ialu_reg_reg_shift);
12149 %}
12150
12151 // This pattern is automatically generated from aarch64_ad.m4
12152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12153 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12154 iRegIorL2I src1, iRegIorL2I src2,
12155 immI src3) %{
12156 match(Set dst (AddI src1 (LShiftI src2 src3)));
12157
12158 ins_cost(1.9 * INSN_COST);
12159 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12160
12161 ins_encode %{
12162 __ addw(as_Register($dst$$reg),
12163 as_Register($src1$$reg),
12164 as_Register($src2$$reg),
12165 Assembler::LSL,
12166 $src3$$constant & 0x1f);
12167 %}
12168
12169 ins_pipe(ialu_reg_reg_shift);
12170 %}
12171
12172 // This pattern is automatically generated from aarch64_ad.m4
12173 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12174 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12175 iRegL src1, iRegL src2,
12176 immI src3) %{
12177 match(Set dst (AddL src1 (LShiftL src2 src3)));
12178
12179 ins_cost(1.9 * INSN_COST);
12180 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12181
12182 ins_encode %{
12183 __ add(as_Register($dst$$reg),
12184 as_Register($src1$$reg),
12185 as_Register($src2$$reg),
12186 Assembler::LSL,
12187 $src3$$constant & 0x3f);
12188 %}
12189
12190 ins_pipe(ialu_reg_reg_shift);
12191 %}
12192
12193 // This pattern is automatically generated from aarch64_ad.m4
12194 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12195 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12196 iRegIorL2I src1, iRegIorL2I src2,
12197 immI src3) %{
12198 match(Set dst (SubI src1 (URShiftI src2 src3)));
12199
12200 ins_cost(1.9 * INSN_COST);
12201 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12202
12203 ins_encode %{
12204 __ subw(as_Register($dst$$reg),
12205 as_Register($src1$$reg),
12206 as_Register($src2$$reg),
12207 Assembler::LSR,
12208 $src3$$constant & 0x1f);
12209 %}
12210
12211 ins_pipe(ialu_reg_reg_shift);
12212 %}
12213
12214 // This pattern is automatically generated from aarch64_ad.m4
12215 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12216 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12217 iRegL src1, iRegL src2,
12218 immI src3) %{
12219 match(Set dst (SubL src1 (URShiftL src2 src3)));
12220
12221 ins_cost(1.9 * INSN_COST);
12222 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12223
12224 ins_encode %{
12225 __ sub(as_Register($dst$$reg),
12226 as_Register($src1$$reg),
12227 as_Register($src2$$reg),
12228 Assembler::LSR,
12229 $src3$$constant & 0x3f);
12230 %}
12231
12232 ins_pipe(ialu_reg_reg_shift);
12233 %}
12234
12235 // This pattern is automatically generated from aarch64_ad.m4
12236 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12237 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12238 iRegIorL2I src1, iRegIorL2I src2,
12239 immI src3) %{
12240 match(Set dst (SubI src1 (RShiftI src2 src3)));
12241
12242 ins_cost(1.9 * INSN_COST);
12243 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12244
12245 ins_encode %{
12246 __ subw(as_Register($dst$$reg),
12247 as_Register($src1$$reg),
12248 as_Register($src2$$reg),
12249 Assembler::ASR,
12250 $src3$$constant & 0x1f);
12251 %}
12252
12253 ins_pipe(ialu_reg_reg_shift);
12254 %}
12255
12256 // This pattern is automatically generated from aarch64_ad.m4
12257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12258 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12259 iRegL src1, iRegL src2,
12260 immI src3) %{
12261 match(Set dst (SubL src1 (RShiftL src2 src3)));
12262
12263 ins_cost(1.9 * INSN_COST);
12264 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12265
12266 ins_encode %{
12267 __ sub(as_Register($dst$$reg),
12268 as_Register($src1$$reg),
12269 as_Register($src2$$reg),
12270 Assembler::ASR,
12271 $src3$$constant & 0x3f);
12272 %}
12273
12274 ins_pipe(ialu_reg_reg_shift);
12275 %}
12276
12277 // This pattern is automatically generated from aarch64_ad.m4
12278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12279 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12280 iRegIorL2I src1, iRegIorL2I src2,
12281 immI src3) %{
12282 match(Set dst (SubI src1 (LShiftI src2 src3)));
12283
12284 ins_cost(1.9 * INSN_COST);
12285 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12286
12287 ins_encode %{
12288 __ subw(as_Register($dst$$reg),
12289 as_Register($src1$$reg),
12290 as_Register($src2$$reg),
12291 Assembler::LSL,
12292 $src3$$constant & 0x1f);
12293 %}
12294
12295 ins_pipe(ialu_reg_reg_shift);
12296 %}
12297
12298 // This pattern is automatically generated from aarch64_ad.m4
12299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12300 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12301 iRegL src1, iRegL src2,
12302 immI src3) %{
12303 match(Set dst (SubL src1 (LShiftL src2 src3)));
12304
12305 ins_cost(1.9 * INSN_COST);
12306 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12307
12308 ins_encode %{
12309 __ sub(as_Register($dst$$reg),
12310 as_Register($src1$$reg),
12311 as_Register($src2$$reg),
12312 Assembler::LSL,
12313 $src3$$constant & 0x3f);
12314 %}
12315
12316 ins_pipe(ialu_reg_reg_shift);
12317 %}
12318
12319 // This pattern is automatically generated from aarch64_ad.m4
12320 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12321
12322 // Shift Left followed by Shift Right.
12323 // This idiom is used by the compiler for the i2b bytecode etc.
12324 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12325 %{
12326 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12327 ins_cost(INSN_COST * 2);
12328 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12329 ins_encode %{
12330 int lshift = $lshift_count$$constant & 63;
12331 int rshift = $rshift_count$$constant & 63;
12332 int s = 63 - lshift;
12333 int r = (rshift - lshift) & 63;
12334 __ sbfm(as_Register($dst$$reg),
12335 as_Register($src$$reg),
12336 r, s);
12337 %}
12338
12339 ins_pipe(ialu_reg_shift);
12340 %}
12341
12342 // This pattern is automatically generated from aarch64_ad.m4
12343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12344
12345 // Shift Left followed by Shift Right.
12346 // This idiom is used by the compiler for the i2b bytecode etc.
12347 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12348 %{
12349 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12350 ins_cost(INSN_COST * 2);
12351 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12352 ins_encode %{
12353 int lshift = $lshift_count$$constant & 31;
12354 int rshift = $rshift_count$$constant & 31;
12355 int s = 31 - lshift;
12356 int r = (rshift - lshift) & 31;
12357 __ sbfmw(as_Register($dst$$reg),
12358 as_Register($src$$reg),
12359 r, s);
12360 %}
12361
12362 ins_pipe(ialu_reg_shift);
12363 %}
12364
12365 // This pattern is automatically generated from aarch64_ad.m4
12366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12367
12368 // Shift Left followed by Shift Right.
12369 // This idiom is used by the compiler for the i2b bytecode etc.
12370 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12371 %{
12372 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12373 ins_cost(INSN_COST * 2);
12374 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12375 ins_encode %{
12376 int lshift = $lshift_count$$constant & 63;
12377 int rshift = $rshift_count$$constant & 63;
12378 int s = 63 - lshift;
12379 int r = (rshift - lshift) & 63;
12380 __ ubfm(as_Register($dst$$reg),
12381 as_Register($src$$reg),
12382 r, s);
12383 %}
12384
12385 ins_pipe(ialu_reg_shift);
12386 %}
12387
12388 // This pattern is automatically generated from aarch64_ad.m4
12389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12390
12391 // Shift Left followed by Shift Right.
12392 // This idiom is used by the compiler for the i2b bytecode etc.
12393 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12394 %{
12395 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12396 ins_cost(INSN_COST * 2);
12397 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12398 ins_encode %{
12399 int lshift = $lshift_count$$constant & 31;
12400 int rshift = $rshift_count$$constant & 31;
12401 int s = 31 - lshift;
12402 int r = (rshift - lshift) & 31;
12403 __ ubfmw(as_Register($dst$$reg),
12404 as_Register($src$$reg),
12405 r, s);
12406 %}
12407
12408 ins_pipe(ialu_reg_shift);
12409 %}
12410
12411 // Bitfield extract with shift & mask
12412
12413 // This pattern is automatically generated from aarch64_ad.m4
12414 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12415 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12416 %{
12417 match(Set dst (AndI (URShiftI src rshift) mask));
12418 // Make sure we are not going to exceed what ubfxw can do.
12419 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12420
12421 ins_cost(INSN_COST);
12422 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12423 ins_encode %{
12424 int rshift = $rshift$$constant & 31;
12425 intptr_t mask = $mask$$constant;
12426 int width = exact_log2(mask+1);
12427 __ ubfxw(as_Register($dst$$reg),
12428 as_Register($src$$reg), rshift, width);
12429 %}
12430 ins_pipe(ialu_reg_shift);
12431 %}
12432
12433 // This pattern is automatically generated from aarch64_ad.m4
12434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12435 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12436 %{
12437 match(Set dst (AndL (URShiftL src rshift) mask));
12438 // Make sure we are not going to exceed what ubfx can do.
12439 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12440
12441 ins_cost(INSN_COST);
12442 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12443 ins_encode %{
12444 int rshift = $rshift$$constant & 63;
12445 intptr_t mask = $mask$$constant;
12446 int width = exact_log2_long(mask+1);
12447 __ ubfx(as_Register($dst$$reg),
12448 as_Register($src$$reg), rshift, width);
12449 %}
12450 ins_pipe(ialu_reg_shift);
12451 %}
12452
12453
12454 // This pattern is automatically generated from aarch64_ad.m4
12455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12456
12457 // We can use ubfx when extending an And with a mask when we know mask
12458 // is positive. We know that because immI_bitmask guarantees it.
12459 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12460 %{
12461 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12462 // Make sure we are not going to exceed what ubfxw can do.
12463 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12464
12465 ins_cost(INSN_COST * 2);
12466 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12467 ins_encode %{
12468 int rshift = $rshift$$constant & 31;
12469 intptr_t mask = $mask$$constant;
12470 int width = exact_log2(mask+1);
12471 __ ubfx(as_Register($dst$$reg),
12472 as_Register($src$$reg), rshift, width);
12473 %}
12474 ins_pipe(ialu_reg_shift);
12475 %}
12476
12477
12478 // This pattern is automatically generated from aarch64_ad.m4
12479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12480
12481 // We can use ubfiz when masking by a positive number and then left shifting the result.
12482 // We know that the mask is positive because immI_bitmask guarantees it.
12483 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12484 %{
12485 match(Set dst (LShiftI (AndI src mask) lshift));
12486 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12487
12488 ins_cost(INSN_COST);
12489 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12490 ins_encode %{
12491 int lshift = $lshift$$constant & 31;
12492 intptr_t mask = $mask$$constant;
12493 int width = exact_log2(mask+1);
12494 __ ubfizw(as_Register($dst$$reg),
12495 as_Register($src$$reg), lshift, width);
12496 %}
12497 ins_pipe(ialu_reg_shift);
12498 %}
12499
12500 // This pattern is automatically generated from aarch64_ad.m4
12501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12502
12503 // We can use ubfiz when masking by a positive number and then left shifting the result.
12504 // We know that the mask is positive because immL_bitmask guarantees it.
12505 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12506 %{
12507 match(Set dst (LShiftL (AndL src mask) lshift));
12508 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12509
12510 ins_cost(INSN_COST);
12511 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12512 ins_encode %{
12513 int lshift = $lshift$$constant & 63;
12514 intptr_t mask = $mask$$constant;
12515 int width = exact_log2_long(mask+1);
12516 __ ubfiz(as_Register($dst$$reg),
12517 as_Register($src$$reg), lshift, width);
12518 %}
12519 ins_pipe(ialu_reg_shift);
12520 %}
12521
12522 // This pattern is automatically generated from aarch64_ad.m4
12523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12524
12525 // We can use ubfiz when masking by a positive number and then left shifting the result.
12526 // We know that the mask is positive because immI_bitmask guarantees it.
12527 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12528 %{
12529 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12530 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12531
12532 ins_cost(INSN_COST);
12533 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12534 ins_encode %{
12535 int lshift = $lshift$$constant & 31;
12536 intptr_t mask = $mask$$constant;
12537 int width = exact_log2(mask+1);
12538 __ ubfizw(as_Register($dst$$reg),
12539 as_Register($src$$reg), lshift, width);
12540 %}
12541 ins_pipe(ialu_reg_shift);
12542 %}
12543
12544 // This pattern is automatically generated from aarch64_ad.m4
12545 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12546
12547 // We can use ubfiz when masking by a positive number and then left shifting the result.
12548 // We know that the mask is positive because immL_bitmask guarantees it.
12549 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12550 %{
12551 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12552 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12553
12554 ins_cost(INSN_COST);
12555 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12556 ins_encode %{
12557 int lshift = $lshift$$constant & 63;
12558 intptr_t mask = $mask$$constant;
12559 int width = exact_log2_long(mask+1);
12560 __ ubfiz(as_Register($dst$$reg),
12561 as_Register($src$$reg), lshift, width);
12562 %}
12563 ins_pipe(ialu_reg_shift);
12564 %}
12565
12566
12567 // This pattern is automatically generated from aarch64_ad.m4
12568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12569
12570 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12571 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12572 %{
12573 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12574 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12575
12576 ins_cost(INSN_COST);
12577 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12578 ins_encode %{
12579 int lshift = $lshift$$constant & 63;
12580 intptr_t mask = $mask$$constant;
12581 int width = exact_log2(mask+1);
12582 __ ubfiz(as_Register($dst$$reg),
12583 as_Register($src$$reg), lshift, width);
12584 %}
12585 ins_pipe(ialu_reg_shift);
12586 %}
12587
12588 // This pattern is automatically generated from aarch64_ad.m4
12589 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12590
12591 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12592 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12593 %{
12594 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12595 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12596
12597 ins_cost(INSN_COST);
12598 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12599 ins_encode %{
12600 int lshift = $lshift$$constant & 31;
12601 intptr_t mask = $mask$$constant;
12602 int width = exact_log2(mask+1);
12603 __ ubfiz(as_Register($dst$$reg),
12604 as_Register($src$$reg), lshift, width);
12605 %}
12606 ins_pipe(ialu_reg_shift);
12607 %}
12608
12609 // This pattern is automatically generated from aarch64_ad.m4
12610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12611
12612 // Can skip int2long conversions after AND with small bitmask
12613 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12614 %{
12615 match(Set dst (ConvI2L (AndI src msk)));
12616 ins_cost(INSN_COST);
12617 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12618 ins_encode %{
12619 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12620 %}
12621 ins_pipe(ialu_reg_shift);
12622 %}
12623
12624
12625 // Rotations
12626
12627 // This pattern is automatically generated from aarch64_ad.m4
12628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12629 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12630 %{
12631 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12632 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12633
12634 ins_cost(INSN_COST);
12635 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12636
12637 ins_encode %{
12638 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12639 $rshift$$constant & 63);
12640 %}
12641 ins_pipe(ialu_reg_reg_extr);
12642 %}
12643
12644
12645 // This pattern is automatically generated from aarch64_ad.m4
12646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12647 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12648 %{
12649 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12650 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12651
12652 ins_cost(INSN_COST);
12653 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12654
12655 ins_encode %{
12656 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12657 $rshift$$constant & 31);
12658 %}
12659 ins_pipe(ialu_reg_reg_extr);
12660 %}
12661
12662
12663 // This pattern is automatically generated from aarch64_ad.m4
12664 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12665 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12666 %{
12667 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12668 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12669
12670 ins_cost(INSN_COST);
12671 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12672
12673 ins_encode %{
12674 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12675 $rshift$$constant & 63);
12676 %}
12677 ins_pipe(ialu_reg_reg_extr);
12678 %}
12679
12680
12681 // This pattern is automatically generated from aarch64_ad.m4
12682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12683 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12684 %{
12685 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12686 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12687
12688 ins_cost(INSN_COST);
12689 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12690
12691 ins_encode %{
12692 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12693 $rshift$$constant & 31);
12694 %}
12695 ins_pipe(ialu_reg_reg_extr);
12696 %}
12697
12698 // This pattern is automatically generated from aarch64_ad.m4
12699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12700 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12701 %{
12702 match(Set dst (RotateRight src shift));
12703
12704 ins_cost(INSN_COST);
12705 format %{ "ror $dst, $src, $shift" %}
12706
12707 ins_encode %{
12708 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12709 $shift$$constant & 0x1f);
12710 %}
12711 ins_pipe(ialu_reg_reg_vshift);
12712 %}
12713
12714 // This pattern is automatically generated from aarch64_ad.m4
12715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12716 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12717 %{
12718 match(Set dst (RotateRight src shift));
12719
12720 ins_cost(INSN_COST);
12721 format %{ "ror $dst, $src, $shift" %}
12722
12723 ins_encode %{
12724 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12725 $shift$$constant & 0x3f);
12726 %}
12727 ins_pipe(ialu_reg_reg_vshift);
12728 %}
12729
12730 // This pattern is automatically generated from aarch64_ad.m4
12731 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12732 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12733 %{
12734 match(Set dst (RotateRight src shift));
12735
12736 ins_cost(INSN_COST);
12737 format %{ "ror $dst, $src, $shift" %}
12738
12739 ins_encode %{
12740 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12741 %}
12742 ins_pipe(ialu_reg_reg_vshift);
12743 %}
12744
12745 // This pattern is automatically generated from aarch64_ad.m4
12746 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12747 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12748 %{
12749 match(Set dst (RotateRight src shift));
12750
12751 ins_cost(INSN_COST);
12752 format %{ "ror $dst, $src, $shift" %}
12753
12754 ins_encode %{
12755 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12756 %}
12757 ins_pipe(ialu_reg_reg_vshift);
12758 %}
12759
12760 // This pattern is automatically generated from aarch64_ad.m4
12761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12762 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12763 %{
12764 match(Set dst (RotateLeft src shift));
12765
12766 ins_cost(INSN_COST);
12767 format %{ "rol $dst, $src, $shift" %}
12768
12769 ins_encode %{
12770 __ subw(rscratch1, zr, as_Register($shift$$reg));
12771 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12772 %}
12773 ins_pipe(ialu_reg_reg_vshift);
12774 %}
12775
12776 // This pattern is automatically generated from aarch64_ad.m4
12777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12778 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12779 %{
12780 match(Set dst (RotateLeft src shift));
12781
12782 ins_cost(INSN_COST);
12783 format %{ "rol $dst, $src, $shift" %}
12784
12785 ins_encode %{
12786 __ subw(rscratch1, zr, as_Register($shift$$reg));
12787 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12788 %}
12789 ins_pipe(ialu_reg_reg_vshift);
12790 %}
12791
12792
12793 // Add/subtract (extended)
12794
12795 // This pattern is automatically generated from aarch64_ad.m4
12796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12797 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12798 %{
12799 match(Set dst (AddL src1 (ConvI2L src2)));
12800 ins_cost(INSN_COST);
12801 format %{ "add $dst, $src1, $src2, sxtw" %}
12802
12803 ins_encode %{
12804 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12805 as_Register($src2$$reg), ext::sxtw);
12806 %}
12807 ins_pipe(ialu_reg_reg);
12808 %}
12809
12810 // This pattern is automatically generated from aarch64_ad.m4
12811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12812 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12813 %{
12814 match(Set dst (SubL src1 (ConvI2L src2)));
12815 ins_cost(INSN_COST);
12816 format %{ "sub $dst, $src1, $src2, sxtw" %}
12817
12818 ins_encode %{
12819 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12820 as_Register($src2$$reg), ext::sxtw);
12821 %}
12822 ins_pipe(ialu_reg_reg);
12823 %}
12824
12825 // This pattern is automatically generated from aarch64_ad.m4
12826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12827 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12828 %{
12829 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12830 ins_cost(INSN_COST);
12831 format %{ "add $dst, $src1, $src2, sxth" %}
12832
12833 ins_encode %{
12834 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12835 as_Register($src2$$reg), ext::sxth);
12836 %}
12837 ins_pipe(ialu_reg_reg);
12838 %}
12839
12840 // This pattern is automatically generated from aarch64_ad.m4
12841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12842 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12843 %{
12844 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12845 ins_cost(INSN_COST);
12846 format %{ "add $dst, $src1, $src2, sxtb" %}
12847
12848 ins_encode %{
12849 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12850 as_Register($src2$$reg), ext::sxtb);
12851 %}
12852 ins_pipe(ialu_reg_reg);
12853 %}
12854
12855 // This pattern is automatically generated from aarch64_ad.m4
12856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12857 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12858 %{
12859 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12860 ins_cost(INSN_COST);
12861 format %{ "add $dst, $src1, $src2, uxtb" %}
12862
12863 ins_encode %{
12864 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12865 as_Register($src2$$reg), ext::uxtb);
12866 %}
12867 ins_pipe(ialu_reg_reg);
12868 %}
12869
12870 // This pattern is automatically generated from aarch64_ad.m4
12871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12872 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12873 %{
12874 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12875 ins_cost(INSN_COST);
12876 format %{ "add $dst, $src1, $src2, sxth" %}
12877
12878 ins_encode %{
12879 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12880 as_Register($src2$$reg), ext::sxth);
12881 %}
12882 ins_pipe(ialu_reg_reg);
12883 %}
12884
12885 // This pattern is automatically generated from aarch64_ad.m4
12886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12887 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12888 %{
12889 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12890 ins_cost(INSN_COST);
12891 format %{ "add $dst, $src1, $src2, sxtw" %}
12892
12893 ins_encode %{
12894 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12895 as_Register($src2$$reg), ext::sxtw);
12896 %}
12897 ins_pipe(ialu_reg_reg);
12898 %}
12899
12900 // This pattern is automatically generated from aarch64_ad.m4
12901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12902 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12903 %{
12904 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12905 ins_cost(INSN_COST);
12906 format %{ "add $dst, $src1, $src2, sxtb" %}
12907
12908 ins_encode %{
12909 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12910 as_Register($src2$$reg), ext::sxtb);
12911 %}
12912 ins_pipe(ialu_reg_reg);
12913 %}
12914
12915 // This pattern is automatically generated from aarch64_ad.m4
12916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12917 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12918 %{
12919 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12920 ins_cost(INSN_COST);
12921 format %{ "add $dst, $src1, $src2, uxtb" %}
12922
12923 ins_encode %{
12924 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12925 as_Register($src2$$reg), ext::uxtb);
12926 %}
12927 ins_pipe(ialu_reg_reg);
12928 %}
12929
12930 // This pattern is automatically generated from aarch64_ad.m4
12931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12932 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12933 %{
12934 match(Set dst (AddI src1 (AndI src2 mask)));
12935 ins_cost(INSN_COST);
12936 format %{ "addw $dst, $src1, $src2, uxtb" %}
12937
12938 ins_encode %{
12939 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12940 as_Register($src2$$reg), ext::uxtb);
12941 %}
12942 ins_pipe(ialu_reg_reg);
12943 %}
12944
12945 // This pattern is automatically generated from aarch64_ad.m4
12946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12947 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12948 %{
12949 match(Set dst (AddI src1 (AndI src2 mask)));
12950 ins_cost(INSN_COST);
12951 format %{ "addw $dst, $src1, $src2, uxth" %}
12952
12953 ins_encode %{
12954 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12955 as_Register($src2$$reg), ext::uxth);
12956 %}
12957 ins_pipe(ialu_reg_reg);
12958 %}
12959
12960 // This pattern is automatically generated from aarch64_ad.m4
12961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12962 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12963 %{
12964 match(Set dst (AddL src1 (AndL src2 mask)));
12965 ins_cost(INSN_COST);
12966 format %{ "add $dst, $src1, $src2, uxtb" %}
12967
12968 ins_encode %{
12969 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12970 as_Register($src2$$reg), ext::uxtb);
12971 %}
12972 ins_pipe(ialu_reg_reg);
12973 %}
12974
12975 // This pattern is automatically generated from aarch64_ad.m4
12976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12977 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12978 %{
12979 match(Set dst (AddL src1 (AndL src2 mask)));
12980 ins_cost(INSN_COST);
12981 format %{ "add $dst, $src1, $src2, uxth" %}
12982
12983 ins_encode %{
12984 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12985 as_Register($src2$$reg), ext::uxth);
12986 %}
12987 ins_pipe(ialu_reg_reg);
12988 %}
12989
12990 // This pattern is automatically generated from aarch64_ad.m4
12991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12992 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12993 %{
12994 match(Set dst (AddL src1 (AndL src2 mask)));
12995 ins_cost(INSN_COST);
12996 format %{ "add $dst, $src1, $src2, uxtw" %}
12997
12998 ins_encode %{
12999 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13000 as_Register($src2$$reg), ext::uxtw);
13001 %}
13002 ins_pipe(ialu_reg_reg);
13003 %}
13004
13005 // This pattern is automatically generated from aarch64_ad.m4
13006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13007 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13008 %{
13009 match(Set dst (SubI src1 (AndI src2 mask)));
13010 ins_cost(INSN_COST);
13011 format %{ "subw $dst, $src1, $src2, uxtb" %}
13012
13013 ins_encode %{
13014 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13015 as_Register($src2$$reg), ext::uxtb);
13016 %}
13017 ins_pipe(ialu_reg_reg);
13018 %}
13019
13020 // This pattern is automatically generated from aarch64_ad.m4
13021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13022 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13023 %{
13024 match(Set dst (SubI src1 (AndI src2 mask)));
13025 ins_cost(INSN_COST);
13026 format %{ "subw $dst, $src1, $src2, uxth" %}
13027
13028 ins_encode %{
13029 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13030 as_Register($src2$$reg), ext::uxth);
13031 %}
13032 ins_pipe(ialu_reg_reg);
13033 %}
13034
13035 // This pattern is automatically generated from aarch64_ad.m4
13036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13037 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13038 %{
13039 match(Set dst (SubL src1 (AndL src2 mask)));
13040 ins_cost(INSN_COST);
13041 format %{ "sub $dst, $src1, $src2, uxtb" %}
13042
13043 ins_encode %{
13044 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13045 as_Register($src2$$reg), ext::uxtb);
13046 %}
13047 ins_pipe(ialu_reg_reg);
13048 %}
13049
13050 // This pattern is automatically generated from aarch64_ad.m4
13051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13052 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13053 %{
13054 match(Set dst (SubL src1 (AndL src2 mask)));
13055 ins_cost(INSN_COST);
13056 format %{ "sub $dst, $src1, $src2, uxth" %}
13057
13058 ins_encode %{
13059 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13060 as_Register($src2$$reg), ext::uxth);
13061 %}
13062 ins_pipe(ialu_reg_reg);
13063 %}
13064
13065 // This pattern is automatically generated from aarch64_ad.m4
13066 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13067 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13068 %{
13069 match(Set dst (SubL src1 (AndL src2 mask)));
13070 ins_cost(INSN_COST);
13071 format %{ "sub $dst, $src1, $src2, uxtw" %}
13072
13073 ins_encode %{
13074 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13075 as_Register($src2$$reg), ext::uxtw);
13076 %}
13077 ins_pipe(ialu_reg_reg);
13078 %}
13079
13080
13081 // This pattern is automatically generated from aarch64_ad.m4
13082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13083 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13084 %{
13085 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13086 ins_cost(1.9 * INSN_COST);
13087 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13088
13089 ins_encode %{
13090 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13091 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13092 %}
13093 ins_pipe(ialu_reg_reg_shift);
13094 %}
13095
13096 // This pattern is automatically generated from aarch64_ad.m4
13097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13098 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13099 %{
13100 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13101 ins_cost(1.9 * INSN_COST);
13102 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13103
13104 ins_encode %{
13105 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13106 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13107 %}
13108 ins_pipe(ialu_reg_reg_shift);
13109 %}
13110
13111 // This pattern is automatically generated from aarch64_ad.m4
13112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13113 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13114 %{
13115 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13116 ins_cost(1.9 * INSN_COST);
13117 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13118
13119 ins_encode %{
13120 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13121 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13122 %}
13123 ins_pipe(ialu_reg_reg_shift);
13124 %}
13125
13126 // This pattern is automatically generated from aarch64_ad.m4
13127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13128 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13129 %{
13130 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13131 ins_cost(1.9 * INSN_COST);
13132 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13133
13134 ins_encode %{
13135 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13136 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13137 %}
13138 ins_pipe(ialu_reg_reg_shift);
13139 %}
13140
13141 // This pattern is automatically generated from aarch64_ad.m4
13142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13143 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13144 %{
13145 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13146 ins_cost(1.9 * INSN_COST);
13147 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13148
13149 ins_encode %{
13150 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13151 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13152 %}
13153 ins_pipe(ialu_reg_reg_shift);
13154 %}
13155
13156 // This pattern is automatically generated from aarch64_ad.m4
13157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13158 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13159 %{
13160 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13161 ins_cost(1.9 * INSN_COST);
13162 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13163
13164 ins_encode %{
13165 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13166 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13167 %}
13168 ins_pipe(ialu_reg_reg_shift);
13169 %}
13170
13171 // This pattern is automatically generated from aarch64_ad.m4
13172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13173 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13174 %{
13175 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13176 ins_cost(1.9 * INSN_COST);
13177 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13178
13179 ins_encode %{
13180 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13181 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13182 %}
13183 ins_pipe(ialu_reg_reg_shift);
13184 %}
13185
13186 // This pattern is automatically generated from aarch64_ad.m4
13187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13188 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13189 %{
13190 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13191 ins_cost(1.9 * INSN_COST);
13192 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13193
13194 ins_encode %{
13195 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13196 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13197 %}
13198 ins_pipe(ialu_reg_reg_shift);
13199 %}
13200
13201 // This pattern is automatically generated from aarch64_ad.m4
13202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13203 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13204 %{
13205 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13206 ins_cost(1.9 * INSN_COST);
13207 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13208
13209 ins_encode %{
13210 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13211 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13212 %}
13213 ins_pipe(ialu_reg_reg_shift);
13214 %}
13215
13216 // This pattern is automatically generated from aarch64_ad.m4
13217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13218 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13219 %{
13220 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13221 ins_cost(1.9 * INSN_COST);
13222 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13223
13224 ins_encode %{
13225 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13226 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13227 %}
13228 ins_pipe(ialu_reg_reg_shift);
13229 %}
13230
13231 // This pattern is automatically generated from aarch64_ad.m4
13232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13233 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13234 %{
13235 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13236 ins_cost(1.9 * INSN_COST);
13237 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13238
13239 ins_encode %{
13240 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13241 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13242 %}
13243 ins_pipe(ialu_reg_reg_shift);
13244 %}
13245
13246 // This pattern is automatically generated from aarch64_ad.m4
13247 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13248 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13249 %{
13250 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13251 ins_cost(1.9 * INSN_COST);
13252 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13253
13254 ins_encode %{
13255 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13256 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13257 %}
13258 ins_pipe(ialu_reg_reg_shift);
13259 %}
13260
13261 // This pattern is automatically generated from aarch64_ad.m4
13262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13263 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13264 %{
13265 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13266 ins_cost(1.9 * INSN_COST);
13267 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13268
13269 ins_encode %{
13270 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13271 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13272 %}
13273 ins_pipe(ialu_reg_reg_shift);
13274 %}
13275
13276 // This pattern is automatically generated from aarch64_ad.m4
13277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13278 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13279 %{
13280 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13281 ins_cost(1.9 * INSN_COST);
13282 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13283
13284 ins_encode %{
13285 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13286 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13287 %}
13288 ins_pipe(ialu_reg_reg_shift);
13289 %}
13290
13291 // This pattern is automatically generated from aarch64_ad.m4
13292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13293 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13294 %{
13295 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13296 ins_cost(1.9 * INSN_COST);
13297 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13298
13299 ins_encode %{
13300 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13301 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13302 %}
13303 ins_pipe(ialu_reg_reg_shift);
13304 %}
13305
13306 // This pattern is automatically generated from aarch64_ad.m4
13307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13308 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13309 %{
13310 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13311 ins_cost(1.9 * INSN_COST);
13312 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13313
13314 ins_encode %{
13315 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13316 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13317 %}
13318 ins_pipe(ialu_reg_reg_shift);
13319 %}
13320
13321 // This pattern is automatically generated from aarch64_ad.m4
13322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13323 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13324 %{
13325 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13326 ins_cost(1.9 * INSN_COST);
13327 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13328
13329 ins_encode %{
13330 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13331 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13332 %}
13333 ins_pipe(ialu_reg_reg_shift);
13334 %}
13335
13336 // This pattern is automatically generated from aarch64_ad.m4
13337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13338 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13339 %{
13340 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13341 ins_cost(1.9 * INSN_COST);
13342 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13343
13344 ins_encode %{
13345 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13346 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13347 %}
13348 ins_pipe(ialu_reg_reg_shift);
13349 %}
13350
13351 // This pattern is automatically generated from aarch64_ad.m4
13352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13353 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13354 %{
13355 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13356 ins_cost(1.9 * INSN_COST);
13357 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13358
13359 ins_encode %{
13360 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13361 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13362 %}
13363 ins_pipe(ialu_reg_reg_shift);
13364 %}
13365
13366 // This pattern is automatically generated from aarch64_ad.m4
13367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13368 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13369 %{
13370 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13371 ins_cost(1.9 * INSN_COST);
13372 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13373
13374 ins_encode %{
13375 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13376 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13377 %}
13378 ins_pipe(ialu_reg_reg_shift);
13379 %}
13380
13381 // This pattern is automatically generated from aarch64_ad.m4
13382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13383 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13384 %{
13385 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13386 ins_cost(1.9 * INSN_COST);
13387 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13388
13389 ins_encode %{
13390 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13391 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13392 %}
13393 ins_pipe(ialu_reg_reg_shift);
13394 %}
13395
13396 // This pattern is automatically generated from aarch64_ad.m4
13397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13398 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13399 %{
13400 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13401 ins_cost(1.9 * INSN_COST);
13402 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13403
13404 ins_encode %{
13405 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13406 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13407 %}
13408 ins_pipe(ialu_reg_reg_shift);
13409 %}
13410
13411 // This pattern is automatically generated from aarch64_ad.m4
13412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13413 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13414 %{
13415 effect(DEF dst, USE src1, USE src2, USE cr);
13416 ins_cost(INSN_COST * 2);
13417 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13418
13419 ins_encode %{
13420 __ cselw($dst$$Register,
13421 $src1$$Register,
13422 $src2$$Register,
13423 Assembler::LT);
13424 %}
13425 ins_pipe(icond_reg_reg);
13426 %}
13427
13428 // This pattern is automatically generated from aarch64_ad.m4
13429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13430 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13431 %{
13432 effect(DEF dst, USE src1, USE src2, USE cr);
13433 ins_cost(INSN_COST * 2);
13434 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13435
13436 ins_encode %{
13437 __ cselw($dst$$Register,
13438 $src1$$Register,
13439 $src2$$Register,
13440 Assembler::GT);
13441 %}
13442 ins_pipe(icond_reg_reg);
13443 %}
13444
13445 // This pattern is automatically generated from aarch64_ad.m4
13446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13447 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13448 %{
13449 effect(DEF dst, USE src1, USE cr);
13450 ins_cost(INSN_COST * 2);
13451 format %{ "cselw $dst, $src1, zr lt\t" %}
13452
13453 ins_encode %{
13454 __ cselw($dst$$Register,
13455 $src1$$Register,
13456 zr,
13457 Assembler::LT);
13458 %}
13459 ins_pipe(icond_reg);
13460 %}
13461
13462 // This pattern is automatically generated from aarch64_ad.m4
13463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13464 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13465 %{
13466 effect(DEF dst, USE src1, USE cr);
13467 ins_cost(INSN_COST * 2);
13468 format %{ "cselw $dst, $src1, zr gt\t" %}
13469
13470 ins_encode %{
13471 __ cselw($dst$$Register,
13472 $src1$$Register,
13473 zr,
13474 Assembler::GT);
13475 %}
13476 ins_pipe(icond_reg);
13477 %}
13478
13479 // This pattern is automatically generated from aarch64_ad.m4
13480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13481 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13482 %{
13483 effect(DEF dst, USE src1, USE cr);
13484 ins_cost(INSN_COST * 2);
13485 format %{ "csincw $dst, $src1, zr le\t" %}
13486
13487 ins_encode %{
13488 __ csincw($dst$$Register,
13489 $src1$$Register,
13490 zr,
13491 Assembler::LE);
13492 %}
13493 ins_pipe(icond_reg);
13494 %}
13495
13496 // This pattern is automatically generated from aarch64_ad.m4
13497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13498 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13499 %{
13500 effect(DEF dst, USE src1, USE cr);
13501 ins_cost(INSN_COST * 2);
13502 format %{ "csincw $dst, $src1, zr gt\t" %}
13503
13504 ins_encode %{
13505 __ csincw($dst$$Register,
13506 $src1$$Register,
13507 zr,
13508 Assembler::GT);
13509 %}
13510 ins_pipe(icond_reg);
13511 %}
13512
13513 // This pattern is automatically generated from aarch64_ad.m4
13514 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13515 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13516 %{
13517 effect(DEF dst, USE src1, USE cr);
13518 ins_cost(INSN_COST * 2);
13519 format %{ "csinvw $dst, $src1, zr lt\t" %}
13520
13521 ins_encode %{
13522 __ csinvw($dst$$Register,
13523 $src1$$Register,
13524 zr,
13525 Assembler::LT);
13526 %}
13527 ins_pipe(icond_reg);
13528 %}
13529
13530 // This pattern is automatically generated from aarch64_ad.m4
13531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13532 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13533 %{
13534 effect(DEF dst, USE src1, USE cr);
13535 ins_cost(INSN_COST * 2);
13536 format %{ "csinvw $dst, $src1, zr ge\t" %}
13537
13538 ins_encode %{
13539 __ csinvw($dst$$Register,
13540 $src1$$Register,
13541 zr,
13542 Assembler::GE);
13543 %}
13544 ins_pipe(icond_reg);
13545 %}
13546
13547 // This pattern is automatically generated from aarch64_ad.m4
13548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13549 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13550 %{
13551 match(Set dst (MinI src imm));
13552 ins_cost(INSN_COST * 3);
13553 expand %{
13554 rFlagsReg cr;
13555 compI_reg_imm0(cr, src);
13556 cmovI_reg_imm0_lt(dst, src, cr);
13557 %}
13558 %}
13559
13560 // This pattern is automatically generated from aarch64_ad.m4
13561 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13562 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13563 %{
13564 match(Set dst (MinI imm src));
13565 ins_cost(INSN_COST * 3);
13566 expand %{
13567 rFlagsReg cr;
13568 compI_reg_imm0(cr, src);
13569 cmovI_reg_imm0_lt(dst, src, cr);
13570 %}
13571 %}
13572
13573 // This pattern is automatically generated from aarch64_ad.m4
13574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13575 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13576 %{
13577 match(Set dst (MinI src imm));
13578 ins_cost(INSN_COST * 3);
13579 expand %{
13580 rFlagsReg cr;
13581 compI_reg_imm0(cr, src);
13582 cmovI_reg_imm1_le(dst, src, cr);
13583 %}
13584 %}
13585
13586 // This pattern is automatically generated from aarch64_ad.m4
13587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13588 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13589 %{
13590 match(Set dst (MinI imm src));
13591 ins_cost(INSN_COST * 3);
13592 expand %{
13593 rFlagsReg cr;
13594 compI_reg_imm0(cr, src);
13595 cmovI_reg_imm1_le(dst, src, cr);
13596 %}
13597 %}
13598
13599 // This pattern is automatically generated from aarch64_ad.m4
13600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13601 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13602 %{
13603 match(Set dst (MinI src imm));
13604 ins_cost(INSN_COST * 3);
13605 expand %{
13606 rFlagsReg cr;
13607 compI_reg_imm0(cr, src);
13608 cmovI_reg_immM1_lt(dst, src, cr);
13609 %}
13610 %}
13611
13612 // This pattern is automatically generated from aarch64_ad.m4
13613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13614 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13615 %{
13616 match(Set dst (MinI imm src));
13617 ins_cost(INSN_COST * 3);
13618 expand %{
13619 rFlagsReg cr;
13620 compI_reg_imm0(cr, src);
13621 cmovI_reg_immM1_lt(dst, src, cr);
13622 %}
13623 %}
13624
13625 // This pattern is automatically generated from aarch64_ad.m4
13626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13627 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13628 %{
13629 match(Set dst (MaxI src imm));
13630 ins_cost(INSN_COST * 3);
13631 expand %{
13632 rFlagsReg cr;
13633 compI_reg_imm0(cr, src);
13634 cmovI_reg_imm0_gt(dst, src, cr);
13635 %}
13636 %}
13637
13638 // This pattern is automatically generated from aarch64_ad.m4
13639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13640 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13641 %{
13642 match(Set dst (MaxI imm src));
13643 ins_cost(INSN_COST * 3);
13644 expand %{
13645 rFlagsReg cr;
13646 compI_reg_imm0(cr, src);
13647 cmovI_reg_imm0_gt(dst, src, cr);
13648 %}
13649 %}
13650
13651 // This pattern is automatically generated from aarch64_ad.m4
13652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13653 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13654 %{
13655 match(Set dst (MaxI src imm));
13656 ins_cost(INSN_COST * 3);
13657 expand %{
13658 rFlagsReg cr;
13659 compI_reg_imm0(cr, src);
13660 cmovI_reg_imm1_gt(dst, src, cr);
13661 %}
13662 %}
13663
13664 // This pattern is automatically generated from aarch64_ad.m4
13665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13666 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13667 %{
13668 match(Set dst (MaxI imm src));
13669 ins_cost(INSN_COST * 3);
13670 expand %{
13671 rFlagsReg cr;
13672 compI_reg_imm0(cr, src);
13673 cmovI_reg_imm1_gt(dst, src, cr);
13674 %}
13675 %}
13676
13677 // This pattern is automatically generated from aarch64_ad.m4
13678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13679 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13680 %{
13681 match(Set dst (MaxI src imm));
13682 ins_cost(INSN_COST * 3);
13683 expand %{
13684 rFlagsReg cr;
13685 compI_reg_imm0(cr, src);
13686 cmovI_reg_immM1_ge(dst, src, cr);
13687 %}
13688 %}
13689
13690 // This pattern is automatically generated from aarch64_ad.m4
13691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13692 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13693 %{
13694 match(Set dst (MaxI imm src));
13695 ins_cost(INSN_COST * 3);
13696 expand %{
13697 rFlagsReg cr;
13698 compI_reg_imm0(cr, src);
13699 cmovI_reg_immM1_ge(dst, src, cr);
13700 %}
13701 %}
13702
13703 // This pattern is automatically generated from aarch64_ad.m4
13704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13705 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13706 %{
13707 match(Set dst (ReverseI src));
13708 ins_cost(INSN_COST);
13709 format %{ "rbitw $dst, $src" %}
13710 ins_encode %{
13711 __ rbitw($dst$$Register, $src$$Register);
13712 %}
13713 ins_pipe(ialu_reg);
13714 %}
13715
13716 // This pattern is automatically generated from aarch64_ad.m4
13717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13718 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13719 %{
13720 match(Set dst (ReverseL src));
13721 ins_cost(INSN_COST);
13722 format %{ "rbit $dst, $src" %}
13723 ins_encode %{
13724 __ rbit($dst$$Register, $src$$Register);
13725 %}
13726 ins_pipe(ialu_reg);
13727 %}
13728
13729
13730 // END This section of the file is automatically generated. Do not edit --------------
13731
13732
13733 // ============================================================================
13734 // Floating Point Arithmetic Instructions
13735
13736 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13737 match(Set dst (AddHF src1 src2));
13738 format %{ "faddh $dst, $src1, $src2" %}
13739 ins_encode %{
13740 __ faddh($dst$$FloatRegister,
13741 $src1$$FloatRegister,
13742 $src2$$FloatRegister);
13743 %}
13744 ins_pipe(fp_dop_reg_reg_s);
13745 %}
13746
13747 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13748 match(Set dst (AddF src1 src2));
13749
13750 ins_cost(INSN_COST * 5);
13751 format %{ "fadds $dst, $src1, $src2" %}
13752
13753 ins_encode %{
13754 __ fadds(as_FloatRegister($dst$$reg),
13755 as_FloatRegister($src1$$reg),
13756 as_FloatRegister($src2$$reg));
13757 %}
13758
13759 ins_pipe(fp_dop_reg_reg_s);
13760 %}
13761
13762 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13763 match(Set dst (AddD src1 src2));
13764
13765 ins_cost(INSN_COST * 5);
13766 format %{ "faddd $dst, $src1, $src2" %}
13767
13768 ins_encode %{
13769 __ faddd(as_FloatRegister($dst$$reg),
13770 as_FloatRegister($src1$$reg),
13771 as_FloatRegister($src2$$reg));
13772 %}
13773
13774 ins_pipe(fp_dop_reg_reg_d);
13775 %}
13776
13777 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13778 match(Set dst (SubHF src1 src2));
13779 format %{ "fsubh $dst, $src1, $src2" %}
13780 ins_encode %{
13781 __ fsubh($dst$$FloatRegister,
13782 $src1$$FloatRegister,
13783 $src2$$FloatRegister);
13784 %}
13785 ins_pipe(fp_dop_reg_reg_s);
13786 %}
13787
13788 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13789 match(Set dst (SubF src1 src2));
13790
13791 ins_cost(INSN_COST * 5);
13792 format %{ "fsubs $dst, $src1, $src2" %}
13793
13794 ins_encode %{
13795 __ fsubs(as_FloatRegister($dst$$reg),
13796 as_FloatRegister($src1$$reg),
13797 as_FloatRegister($src2$$reg));
13798 %}
13799
13800 ins_pipe(fp_dop_reg_reg_s);
13801 %}
13802
13803 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13804 match(Set dst (SubD src1 src2));
13805
13806 ins_cost(INSN_COST * 5);
13807 format %{ "fsubd $dst, $src1, $src2" %}
13808
13809 ins_encode %{
13810 __ fsubd(as_FloatRegister($dst$$reg),
13811 as_FloatRegister($src1$$reg),
13812 as_FloatRegister($src2$$reg));
13813 %}
13814
13815 ins_pipe(fp_dop_reg_reg_d);
13816 %}
13817
13818 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13819 match(Set dst (MulHF src1 src2));
13820 format %{ "fmulh $dst, $src1, $src2" %}
13821 ins_encode %{
13822 __ fmulh($dst$$FloatRegister,
13823 $src1$$FloatRegister,
13824 $src2$$FloatRegister);
13825 %}
13826 ins_pipe(fp_dop_reg_reg_s);
13827 %}
13828
13829 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13830 match(Set dst (MulF src1 src2));
13831
13832 ins_cost(INSN_COST * 6);
13833 format %{ "fmuls $dst, $src1, $src2" %}
13834
13835 ins_encode %{
13836 __ fmuls(as_FloatRegister($dst$$reg),
13837 as_FloatRegister($src1$$reg),
13838 as_FloatRegister($src2$$reg));
13839 %}
13840
13841 ins_pipe(fp_dop_reg_reg_s);
13842 %}
13843
13844 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13845 match(Set dst (MulD src1 src2));
13846
13847 ins_cost(INSN_COST * 6);
13848 format %{ "fmuld $dst, $src1, $src2" %}
13849
13850 ins_encode %{
13851 __ fmuld(as_FloatRegister($dst$$reg),
13852 as_FloatRegister($src1$$reg),
13853 as_FloatRegister($src2$$reg));
13854 %}
13855
13856 ins_pipe(fp_dop_reg_reg_d);
13857 %}
13858
13859 // src1 * src2 + src3 (half-precision float)
13860 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13861 match(Set dst (FmaHF src3 (Binary src1 src2)));
13862 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13863 ins_encode %{
13864 assert(UseFMA, "Needs FMA instructions support.");
13865 __ fmaddh($dst$$FloatRegister,
13866 $src1$$FloatRegister,
13867 $src2$$FloatRegister,
13868 $src3$$FloatRegister);
13869 %}
13870 ins_pipe(pipe_class_default);
13871 %}
13872
13873 // src1 * src2 + src3
13874 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13875 match(Set dst (FmaF src3 (Binary src1 src2)));
13876
13877 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13878
13879 ins_encode %{
13880 assert(UseFMA, "Needs FMA instructions support.");
13881 __ fmadds(as_FloatRegister($dst$$reg),
13882 as_FloatRegister($src1$$reg),
13883 as_FloatRegister($src2$$reg),
13884 as_FloatRegister($src3$$reg));
13885 %}
13886
13887 ins_pipe(pipe_class_default);
13888 %}
13889
13890 // src1 * src2 + src3
13891 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13892 match(Set dst (FmaD src3 (Binary src1 src2)));
13893
13894 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13895
13896 ins_encode %{
13897 assert(UseFMA, "Needs FMA instructions support.");
13898 __ fmaddd(as_FloatRegister($dst$$reg),
13899 as_FloatRegister($src1$$reg),
13900 as_FloatRegister($src2$$reg),
13901 as_FloatRegister($src3$$reg));
13902 %}
13903
13904 ins_pipe(pipe_class_default);
13905 %}
13906
13907 // src1 * (-src2) + src3
13908 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13909 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13910 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13911
13912 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13913
13914 ins_encode %{
13915 assert(UseFMA, "Needs FMA instructions support.");
13916 __ fmsubs(as_FloatRegister($dst$$reg),
13917 as_FloatRegister($src1$$reg),
13918 as_FloatRegister($src2$$reg),
13919 as_FloatRegister($src3$$reg));
13920 %}
13921
13922 ins_pipe(pipe_class_default);
13923 %}
13924
13925 // src1 * (-src2) + src3
13926 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13927 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13928 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13929
13930 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13931
13932 ins_encode %{
13933 assert(UseFMA, "Needs FMA instructions support.");
13934 __ fmsubd(as_FloatRegister($dst$$reg),
13935 as_FloatRegister($src1$$reg),
13936 as_FloatRegister($src2$$reg),
13937 as_FloatRegister($src3$$reg));
13938 %}
13939
13940 ins_pipe(pipe_class_default);
13941 %}
13942
13943 // src1 * (-src2) - src3
13944 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13945 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13946 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13947
13948 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13949
13950 ins_encode %{
13951 assert(UseFMA, "Needs FMA instructions support.");
13952 __ fnmadds(as_FloatRegister($dst$$reg),
13953 as_FloatRegister($src1$$reg),
13954 as_FloatRegister($src2$$reg),
13955 as_FloatRegister($src3$$reg));
13956 %}
13957
13958 ins_pipe(pipe_class_default);
13959 %}
13960
13961 // src1 * (-src2) - src3
13962 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13963 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13964 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13965
13966 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13967
13968 ins_encode %{
13969 assert(UseFMA, "Needs FMA instructions support.");
13970 __ fnmaddd(as_FloatRegister($dst$$reg),
13971 as_FloatRegister($src1$$reg),
13972 as_FloatRegister($src2$$reg),
13973 as_FloatRegister($src3$$reg));
13974 %}
13975
13976 ins_pipe(pipe_class_default);
13977 %}
13978
13979 // src1 * src2 - src3
13980 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13981 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13982
13983 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13984
13985 ins_encode %{
13986 assert(UseFMA, "Needs FMA instructions support.");
13987 __ fnmsubs(as_FloatRegister($dst$$reg),
13988 as_FloatRegister($src1$$reg),
13989 as_FloatRegister($src2$$reg),
13990 as_FloatRegister($src3$$reg));
13991 %}
13992
13993 ins_pipe(pipe_class_default);
13994 %}
13995
13996 // src1 * src2 - src3
13997 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13998 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13999
14000 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14001
14002 ins_encode %{
14003 assert(UseFMA, "Needs FMA instructions support.");
14004 // n.b. insn name should be fnmsubd
14005 __ fnmsub(as_FloatRegister($dst$$reg),
14006 as_FloatRegister($src1$$reg),
14007 as_FloatRegister($src2$$reg),
14008 as_FloatRegister($src3$$reg));
14009 %}
14010
14011 ins_pipe(pipe_class_default);
14012 %}
14013
14014 // Math.max(HH)H (half-precision float)
14015 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14016 match(Set dst (MaxHF src1 src2));
14017 format %{ "fmaxh $dst, $src1, $src2" %}
14018 ins_encode %{
14019 __ fmaxh($dst$$FloatRegister,
14020 $src1$$FloatRegister,
14021 $src2$$FloatRegister);
14022 %}
14023 ins_pipe(fp_dop_reg_reg_s);
14024 %}
14025
14026 // Math.min(HH)H (half-precision float)
14027 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14028 match(Set dst (MinHF src1 src2));
14029 format %{ "fminh $dst, $src1, $src2" %}
14030 ins_encode %{
14031 __ fminh($dst$$FloatRegister,
14032 $src1$$FloatRegister,
14033 $src2$$FloatRegister);
14034 %}
14035 ins_pipe(fp_dop_reg_reg_s);
14036 %}
14037
14038 // Math.max(FF)F
14039 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14040 match(Set dst (MaxF src1 src2));
14041
14042 format %{ "fmaxs $dst, $src1, $src2" %}
14043 ins_encode %{
14044 __ fmaxs(as_FloatRegister($dst$$reg),
14045 as_FloatRegister($src1$$reg),
14046 as_FloatRegister($src2$$reg));
14047 %}
14048
14049 ins_pipe(fp_dop_reg_reg_s);
14050 %}
14051
14052 // Math.min(FF)F
14053 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14054 match(Set dst (MinF src1 src2));
14055
14056 format %{ "fmins $dst, $src1, $src2" %}
14057 ins_encode %{
14058 __ fmins(as_FloatRegister($dst$$reg),
14059 as_FloatRegister($src1$$reg),
14060 as_FloatRegister($src2$$reg));
14061 %}
14062
14063 ins_pipe(fp_dop_reg_reg_s);
14064 %}
14065
14066 // Math.max(DD)D
14067 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14068 match(Set dst (MaxD src1 src2));
14069
14070 format %{ "fmaxd $dst, $src1, $src2" %}
14071 ins_encode %{
14072 __ fmaxd(as_FloatRegister($dst$$reg),
14073 as_FloatRegister($src1$$reg),
14074 as_FloatRegister($src2$$reg));
14075 %}
14076
14077 ins_pipe(fp_dop_reg_reg_d);
14078 %}
14079
14080 // Math.min(DD)D
14081 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14082 match(Set dst (MinD src1 src2));
14083
14084 format %{ "fmind $dst, $src1, $src2" %}
14085 ins_encode %{
14086 __ fmind(as_FloatRegister($dst$$reg),
14087 as_FloatRegister($src1$$reg),
14088 as_FloatRegister($src2$$reg));
14089 %}
14090
14091 ins_pipe(fp_dop_reg_reg_d);
14092 %}
14093
14094 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14095 match(Set dst (DivHF src1 src2));
14096 format %{ "fdivh $dst, $src1, $src2" %}
14097 ins_encode %{
14098 __ fdivh($dst$$FloatRegister,
14099 $src1$$FloatRegister,
14100 $src2$$FloatRegister);
14101 %}
14102 ins_pipe(fp_div_s);
14103 %}
14104
14105 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14106 match(Set dst (DivF src1 src2));
14107
14108 ins_cost(INSN_COST * 18);
14109 format %{ "fdivs $dst, $src1, $src2" %}
14110
14111 ins_encode %{
14112 __ fdivs(as_FloatRegister($dst$$reg),
14113 as_FloatRegister($src1$$reg),
14114 as_FloatRegister($src2$$reg));
14115 %}
14116
14117 ins_pipe(fp_div_s);
14118 %}
14119
14120 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14121 match(Set dst (DivD src1 src2));
14122
14123 ins_cost(INSN_COST * 32);
14124 format %{ "fdivd $dst, $src1, $src2" %}
14125
14126 ins_encode %{
14127 __ fdivd(as_FloatRegister($dst$$reg),
14128 as_FloatRegister($src1$$reg),
14129 as_FloatRegister($src2$$reg));
14130 %}
14131
14132 ins_pipe(fp_div_d);
14133 %}
14134
14135 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14136 match(Set dst (NegF src));
14137
14138 ins_cost(INSN_COST * 3);
14139 format %{ "fneg $dst, $src" %}
14140
14141 ins_encode %{
14142 __ fnegs(as_FloatRegister($dst$$reg),
14143 as_FloatRegister($src$$reg));
14144 %}
14145
14146 ins_pipe(fp_uop_s);
14147 %}
14148
14149 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14150 match(Set dst (NegD src));
14151
14152 ins_cost(INSN_COST * 3);
14153 format %{ "fnegd $dst, $src" %}
14154
14155 ins_encode %{
14156 __ fnegd(as_FloatRegister($dst$$reg),
14157 as_FloatRegister($src$$reg));
14158 %}
14159
14160 ins_pipe(fp_uop_d);
14161 %}
14162
14163 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14164 %{
14165 match(Set dst (AbsI src));
14166
14167 effect(KILL cr);
14168 ins_cost(INSN_COST * 2);
14169 format %{ "cmpw $src, zr\n\t"
14170 "cnegw $dst, $src, Assembler::LT\t# int abs"
14171 %}
14172
14173 ins_encode %{
14174 __ cmpw(as_Register($src$$reg), zr);
14175 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14176 %}
14177 ins_pipe(pipe_class_default);
14178 %}
14179
14180 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14181 %{
14182 match(Set dst (AbsL src));
14183
14184 effect(KILL cr);
14185 ins_cost(INSN_COST * 2);
14186 format %{ "cmp $src, zr\n\t"
14187 "cneg $dst, $src, Assembler::LT\t# long abs"
14188 %}
14189
14190 ins_encode %{
14191 __ cmp(as_Register($src$$reg), zr);
14192 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14193 %}
14194 ins_pipe(pipe_class_default);
14195 %}
14196
14197 instruct absF_reg(vRegF dst, vRegF src) %{
14198 match(Set dst (AbsF src));
14199
14200 ins_cost(INSN_COST * 3);
14201 format %{ "fabss $dst, $src" %}
14202 ins_encode %{
14203 __ fabss(as_FloatRegister($dst$$reg),
14204 as_FloatRegister($src$$reg));
14205 %}
14206
14207 ins_pipe(fp_uop_s);
14208 %}
14209
14210 instruct absD_reg(vRegD dst, vRegD src) %{
14211 match(Set dst (AbsD src));
14212
14213 ins_cost(INSN_COST * 3);
14214 format %{ "fabsd $dst, $src" %}
14215 ins_encode %{
14216 __ fabsd(as_FloatRegister($dst$$reg),
14217 as_FloatRegister($src$$reg));
14218 %}
14219
14220 ins_pipe(fp_uop_d);
14221 %}
14222
14223 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14224 match(Set dst (AbsF (SubF src1 src2)));
14225
14226 ins_cost(INSN_COST * 3);
14227 format %{ "fabds $dst, $src1, $src2" %}
14228 ins_encode %{
14229 __ fabds(as_FloatRegister($dst$$reg),
14230 as_FloatRegister($src1$$reg),
14231 as_FloatRegister($src2$$reg));
14232 %}
14233
14234 ins_pipe(fp_uop_s);
14235 %}
14236
14237 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14238 match(Set dst (AbsD (SubD src1 src2)));
14239
14240 ins_cost(INSN_COST * 3);
14241 format %{ "fabdd $dst, $src1, $src2" %}
14242 ins_encode %{
14243 __ fabdd(as_FloatRegister($dst$$reg),
14244 as_FloatRegister($src1$$reg),
14245 as_FloatRegister($src2$$reg));
14246 %}
14247
14248 ins_pipe(fp_uop_d);
14249 %}
14250
14251 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14252 match(Set dst (SqrtD src));
14253
14254 ins_cost(INSN_COST * 50);
14255 format %{ "fsqrtd $dst, $src" %}
14256 ins_encode %{
14257 __ fsqrtd(as_FloatRegister($dst$$reg),
14258 as_FloatRegister($src$$reg));
14259 %}
14260
14261 ins_pipe(fp_div_s);
14262 %}
14263
14264 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14265 match(Set dst (SqrtF src));
14266
14267 ins_cost(INSN_COST * 50);
14268 format %{ "fsqrts $dst, $src" %}
14269 ins_encode %{
14270 __ fsqrts(as_FloatRegister($dst$$reg),
14271 as_FloatRegister($src$$reg));
14272 %}
14273
14274 ins_pipe(fp_div_d);
14275 %}
14276
14277 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14278 match(Set dst (SqrtHF src));
14279 format %{ "fsqrth $dst, $src" %}
14280 ins_encode %{
14281 __ fsqrth($dst$$FloatRegister,
14282 $src$$FloatRegister);
14283 %}
14284 ins_pipe(fp_div_s);
14285 %}
14286
14287 // Math.rint, floor, ceil
14288 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14289 match(Set dst (RoundDoubleMode src rmode));
14290 format %{ "frint $dst, $src, $rmode" %}
14291 ins_encode %{
14292 switch ($rmode$$constant) {
14293 case RoundDoubleModeNode::rmode_rint:
14294 __ frintnd(as_FloatRegister($dst$$reg),
14295 as_FloatRegister($src$$reg));
14296 break;
14297 case RoundDoubleModeNode::rmode_floor:
14298 __ frintmd(as_FloatRegister($dst$$reg),
14299 as_FloatRegister($src$$reg));
14300 break;
14301 case RoundDoubleModeNode::rmode_ceil:
14302 __ frintpd(as_FloatRegister($dst$$reg),
14303 as_FloatRegister($src$$reg));
14304 break;
14305 }
14306 %}
14307 ins_pipe(fp_uop_d);
14308 %}
14309
14310 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14311 match(Set dst (CopySignD src1 (Binary src2 zero)));
14312 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14313 format %{ "CopySignD $dst $src1 $src2" %}
14314 ins_encode %{
14315 FloatRegister dst = as_FloatRegister($dst$$reg),
14316 src1 = as_FloatRegister($src1$$reg),
14317 src2 = as_FloatRegister($src2$$reg),
14318 zero = as_FloatRegister($zero$$reg);
14319 __ fnegd(dst, zero);
14320 __ bsl(dst, __ T8B, src2, src1);
14321 %}
14322 ins_pipe(fp_uop_d);
14323 %}
14324
14325 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14326 match(Set dst (CopySignF src1 src2));
14327 effect(TEMP_DEF dst, USE src1, USE src2);
14328 format %{ "CopySignF $dst $src1 $src2" %}
14329 ins_encode %{
14330 FloatRegister dst = as_FloatRegister($dst$$reg),
14331 src1 = as_FloatRegister($src1$$reg),
14332 src2 = as_FloatRegister($src2$$reg);
14333 __ movi(dst, __ T2S, 0x80, 24);
14334 __ bsl(dst, __ T8B, src2, src1);
14335 %}
14336 ins_pipe(fp_uop_d);
14337 %}
14338
14339 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14340 match(Set dst (SignumD src (Binary zero one)));
14341 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14342 format %{ "signumD $dst, $src" %}
14343 ins_encode %{
14344 FloatRegister src = as_FloatRegister($src$$reg),
14345 dst = as_FloatRegister($dst$$reg),
14346 zero = as_FloatRegister($zero$$reg),
14347 one = as_FloatRegister($one$$reg);
14348 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14349 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14350 // Bit selection instruction gets bit from "one" for each enabled bit in
14351 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14352 // NaN the whole "src" will be copied because "dst" is zero. For all other
14353 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14354 // from "src", and all other bits are copied from 1.0.
14355 __ bsl(dst, __ T8B, one, src);
14356 %}
14357 ins_pipe(fp_uop_d);
14358 %}
14359
14360 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14361 match(Set dst (SignumF src (Binary zero one)));
14362 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14363 format %{ "signumF $dst, $src" %}
14364 ins_encode %{
14365 FloatRegister src = as_FloatRegister($src$$reg),
14366 dst = as_FloatRegister($dst$$reg),
14367 zero = as_FloatRegister($zero$$reg),
14368 one = as_FloatRegister($one$$reg);
14369 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14370 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14371 // Bit selection instruction gets bit from "one" for each enabled bit in
14372 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14373 // NaN the whole "src" will be copied because "dst" is zero. For all other
14374 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14375 // from "src", and all other bits are copied from 1.0.
14376 __ bsl(dst, __ T8B, one, src);
14377 %}
14378 ins_pipe(fp_uop_d);
14379 %}
14380
14381 instruct onspinwait() %{
14382 match(OnSpinWait);
14383 ins_cost(INSN_COST);
14384
14385 format %{ "onspinwait" %}
14386
14387 ins_encode %{
14388 __ spin_wait();
14389 %}
14390 ins_pipe(pipe_class_empty);
14391 %}
14392
14393 // ============================================================================
14394 // Logical Instructions
14395
14396 // Integer Logical Instructions
14397
14398 // And Instructions
14399
14400
14401 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14402 match(Set dst (AndI src1 src2));
14403
14404 format %{ "andw $dst, $src1, $src2\t# int" %}
14405
14406 ins_cost(INSN_COST);
14407 ins_encode %{
14408 __ andw(as_Register($dst$$reg),
14409 as_Register($src1$$reg),
14410 as_Register($src2$$reg));
14411 %}
14412
14413 ins_pipe(ialu_reg_reg);
14414 %}
14415
14416 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14417 match(Set dst (AndI src1 src2));
14418
14419 format %{ "andsw $dst, $src1, $src2\t# int" %}
14420
14421 ins_cost(INSN_COST);
14422 ins_encode %{
14423 __ andw(as_Register($dst$$reg),
14424 as_Register($src1$$reg),
14425 (uint64_t)($src2$$constant));
14426 %}
14427
14428 ins_pipe(ialu_reg_imm);
14429 %}
14430
14431 // Or Instructions
14432
14433 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14434 match(Set dst (OrI src1 src2));
14435
14436 format %{ "orrw $dst, $src1, $src2\t# int" %}
14437
14438 ins_cost(INSN_COST);
14439 ins_encode %{
14440 __ orrw(as_Register($dst$$reg),
14441 as_Register($src1$$reg),
14442 as_Register($src2$$reg));
14443 %}
14444
14445 ins_pipe(ialu_reg_reg);
14446 %}
14447
14448 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14449 match(Set dst (OrI src1 src2));
14450
14451 format %{ "orrw $dst, $src1, $src2\t# int" %}
14452
14453 ins_cost(INSN_COST);
14454 ins_encode %{
14455 __ orrw(as_Register($dst$$reg),
14456 as_Register($src1$$reg),
14457 (uint64_t)($src2$$constant));
14458 %}
14459
14460 ins_pipe(ialu_reg_imm);
14461 %}
14462
14463 // Xor Instructions
14464
14465 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14466 match(Set dst (XorI src1 src2));
14467
14468 format %{ "eorw $dst, $src1, $src2\t# int" %}
14469
14470 ins_cost(INSN_COST);
14471 ins_encode %{
14472 __ eorw(as_Register($dst$$reg),
14473 as_Register($src1$$reg),
14474 as_Register($src2$$reg));
14475 %}
14476
14477 ins_pipe(ialu_reg_reg);
14478 %}
14479
14480 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14481 match(Set dst (XorI src1 src2));
14482
14483 format %{ "eorw $dst, $src1, $src2\t# int" %}
14484
14485 ins_cost(INSN_COST);
14486 ins_encode %{
14487 __ eorw(as_Register($dst$$reg),
14488 as_Register($src1$$reg),
14489 (uint64_t)($src2$$constant));
14490 %}
14491
14492 ins_pipe(ialu_reg_imm);
14493 %}
14494
14495 // Long Logical Instructions
14496 // TODO
14497
14498 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14499 match(Set dst (AndL src1 src2));
14500
14501 format %{ "and $dst, $src1, $src2\t# int" %}
14502
14503 ins_cost(INSN_COST);
14504 ins_encode %{
14505 __ andr(as_Register($dst$$reg),
14506 as_Register($src1$$reg),
14507 as_Register($src2$$reg));
14508 %}
14509
14510 ins_pipe(ialu_reg_reg);
14511 %}
14512
14513 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14514 match(Set dst (AndL src1 src2));
14515
14516 format %{ "and $dst, $src1, $src2\t# int" %}
14517
14518 ins_cost(INSN_COST);
14519 ins_encode %{
14520 __ andr(as_Register($dst$$reg),
14521 as_Register($src1$$reg),
14522 (uint64_t)($src2$$constant));
14523 %}
14524
14525 ins_pipe(ialu_reg_imm);
14526 %}
14527
14528 // Or Instructions
14529
14530 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14531 match(Set dst (OrL src1 src2));
14532
14533 format %{ "orr $dst, $src1, $src2\t# int" %}
14534
14535 ins_cost(INSN_COST);
14536 ins_encode %{
14537 __ orr(as_Register($dst$$reg),
14538 as_Register($src1$$reg),
14539 as_Register($src2$$reg));
14540 %}
14541
14542 ins_pipe(ialu_reg_reg);
14543 %}
14544
14545 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14546 match(Set dst (OrL src1 src2));
14547
14548 format %{ "orr $dst, $src1, $src2\t# int" %}
14549
14550 ins_cost(INSN_COST);
14551 ins_encode %{
14552 __ orr(as_Register($dst$$reg),
14553 as_Register($src1$$reg),
14554 (uint64_t)($src2$$constant));
14555 %}
14556
14557 ins_pipe(ialu_reg_imm);
14558 %}
14559
14560 // Xor Instructions
14561
14562 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14563 match(Set dst (XorL src1 src2));
14564
14565 format %{ "eor $dst, $src1, $src2\t# int" %}
14566
14567 ins_cost(INSN_COST);
14568 ins_encode %{
14569 __ eor(as_Register($dst$$reg),
14570 as_Register($src1$$reg),
14571 as_Register($src2$$reg));
14572 %}
14573
14574 ins_pipe(ialu_reg_reg);
14575 %}
14576
14577 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14578 match(Set dst (XorL src1 src2));
14579
14580 ins_cost(INSN_COST);
14581 format %{ "eor $dst, $src1, $src2\t# int" %}
14582
14583 ins_encode %{
14584 __ eor(as_Register($dst$$reg),
14585 as_Register($src1$$reg),
14586 (uint64_t)($src2$$constant));
14587 %}
14588
14589 ins_pipe(ialu_reg_imm);
14590 %}
14591
14592 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14593 %{
14594 match(Set dst (ConvI2L src));
14595
14596 ins_cost(INSN_COST);
14597 format %{ "sxtw $dst, $src\t# i2l" %}
14598 ins_encode %{
14599 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14600 %}
14601 ins_pipe(ialu_reg_shift);
14602 %}
14603
14604 // this pattern occurs in bigmath arithmetic
14605 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14606 %{
14607 match(Set dst (AndL (ConvI2L src) mask));
14608
14609 ins_cost(INSN_COST);
14610 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14611 ins_encode %{
14612 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14613 %}
14614
14615 ins_pipe(ialu_reg_shift);
14616 %}
14617
14618 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14619 match(Set dst (ConvL2I src));
14620
14621 ins_cost(INSN_COST);
14622 format %{ "movw $dst, $src \t// l2i" %}
14623
14624 ins_encode %{
14625 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14626 %}
14627
14628 ins_pipe(ialu_reg);
14629 %}
14630
14631 instruct convD2F_reg(vRegF dst, vRegD src) %{
14632 match(Set dst (ConvD2F src));
14633
14634 ins_cost(INSN_COST * 5);
14635 format %{ "fcvtd $dst, $src \t// d2f" %}
14636
14637 ins_encode %{
14638 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14639 %}
14640
14641 ins_pipe(fp_d2f);
14642 %}
14643
14644 instruct convF2D_reg(vRegD dst, vRegF src) %{
14645 match(Set dst (ConvF2D src));
14646
14647 ins_cost(INSN_COST * 5);
14648 format %{ "fcvts $dst, $src \t// f2d" %}
14649
14650 ins_encode %{
14651 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14652 %}
14653
14654 ins_pipe(fp_f2d);
14655 %}
14656
14657 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14658 match(Set dst (ConvF2I src));
14659
14660 ins_cost(INSN_COST * 5);
14661 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14662
14663 ins_encode %{
14664 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14665 %}
14666
14667 ins_pipe(fp_f2i);
14668 %}
14669
14670 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14671 match(Set dst (ConvF2L src));
14672
14673 ins_cost(INSN_COST * 5);
14674 format %{ "fcvtzs $dst, $src \t// f2l" %}
14675
14676 ins_encode %{
14677 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14678 %}
14679
14680 ins_pipe(fp_f2l);
14681 %}
14682
14683 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14684 match(Set dst (ConvF2HF src));
14685 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14686 "smov $dst, $tmp\t# move result from $tmp to $dst"
14687 %}
14688 effect(TEMP tmp);
14689 ins_encode %{
14690 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14691 %}
14692 ins_pipe(pipe_slow);
14693 %}
14694
14695 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14696 match(Set dst (ConvHF2F src));
14697 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14698 "fcvt $dst, $tmp\t# convert half to single precision"
14699 %}
14700 effect(TEMP tmp);
14701 ins_encode %{
14702 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14703 %}
14704 ins_pipe(pipe_slow);
14705 %}
14706
14707 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14708 match(Set dst (ConvI2F src));
14709
14710 ins_cost(INSN_COST * 5);
14711 format %{ "scvtfws $dst, $src \t// i2f" %}
14712
14713 ins_encode %{
14714 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14715 %}
14716
14717 ins_pipe(fp_i2f);
14718 %}
14719
14720 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14721 match(Set dst (ConvL2F src));
14722
14723 ins_cost(INSN_COST * 5);
14724 format %{ "scvtfs $dst, $src \t// l2f" %}
14725
14726 ins_encode %{
14727 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14728 %}
14729
14730 ins_pipe(fp_l2f);
14731 %}
14732
14733 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14734 match(Set dst (ConvD2I src));
14735
14736 ins_cost(INSN_COST * 5);
14737 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14738
14739 ins_encode %{
14740 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14741 %}
14742
14743 ins_pipe(fp_d2i);
14744 %}
14745
14746 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14747 match(Set dst (ConvD2L src));
14748
14749 ins_cost(INSN_COST * 5);
14750 format %{ "fcvtzd $dst, $src \t// d2l" %}
14751
14752 ins_encode %{
14753 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14754 %}
14755
14756 ins_pipe(fp_d2l);
14757 %}
14758
14759 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14760 match(Set dst (ConvI2D src));
14761
14762 ins_cost(INSN_COST * 5);
14763 format %{ "scvtfwd $dst, $src \t// i2d" %}
14764
14765 ins_encode %{
14766 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14767 %}
14768
14769 ins_pipe(fp_i2d);
14770 %}
14771
14772 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14773 match(Set dst (ConvL2D src));
14774
14775 ins_cost(INSN_COST * 5);
14776 format %{ "scvtfd $dst, $src \t// l2d" %}
14777
14778 ins_encode %{
14779 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14780 %}
14781
14782 ins_pipe(fp_l2d);
14783 %}
14784
14785 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14786 %{
14787 match(Set dst (RoundD src));
14788 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14789 format %{ "java_round_double $dst,$src"%}
14790 ins_encode %{
14791 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14792 as_FloatRegister($ftmp$$reg));
14793 %}
14794 ins_pipe(pipe_slow);
14795 %}
14796
14797 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14798 %{
14799 match(Set dst (RoundF src));
14800 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14801 format %{ "java_round_float $dst,$src"%}
14802 ins_encode %{
14803 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14804 as_FloatRegister($ftmp$$reg));
14805 %}
14806 ins_pipe(pipe_slow);
14807 %}
14808
14809 // stack <-> reg and reg <-> reg shuffles with no conversion
14810
14811 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14812
14813 match(Set dst (MoveF2I src));
14814
14815 effect(DEF dst, USE src);
14816
14817 ins_cost(4 * INSN_COST);
14818
14819 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14820
14821 ins_encode %{
14822 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14823 %}
14824
14825 ins_pipe(iload_reg_reg);
14826
14827 %}
14828
14829 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14830
14831 match(Set dst (MoveI2F src));
14832
14833 effect(DEF dst, USE src);
14834
14835 ins_cost(4 * INSN_COST);
14836
14837 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14838
14839 ins_encode %{
14840 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14841 %}
14842
14843 ins_pipe(pipe_class_memory);
14844
14845 %}
14846
14847 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14848
14849 match(Set dst (MoveD2L src));
14850
14851 effect(DEF dst, USE src);
14852
14853 ins_cost(4 * INSN_COST);
14854
14855 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14856
14857 ins_encode %{
14858 __ ldr($dst$$Register, Address(sp, $src$$disp));
14859 %}
14860
14861 ins_pipe(iload_reg_reg);
14862
14863 %}
14864
14865 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14866
14867 match(Set dst (MoveL2D src));
14868
14869 effect(DEF dst, USE src);
14870
14871 ins_cost(4 * INSN_COST);
14872
14873 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14874
14875 ins_encode %{
14876 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14877 %}
14878
14879 ins_pipe(pipe_class_memory);
14880
14881 %}
14882
14883 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14884
14885 match(Set dst (MoveF2I src));
14886
14887 effect(DEF dst, USE src);
14888
14889 ins_cost(INSN_COST);
14890
14891 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14892
14893 ins_encode %{
14894 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14895 %}
14896
14897 ins_pipe(pipe_class_memory);
14898
14899 %}
14900
14901 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14902
14903 match(Set dst (MoveI2F src));
14904
14905 effect(DEF dst, USE src);
14906
14907 ins_cost(INSN_COST);
14908
14909 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14910
14911 ins_encode %{
14912 __ strw($src$$Register, Address(sp, $dst$$disp));
14913 %}
14914
14915 ins_pipe(istore_reg_reg);
14916
14917 %}
14918
14919 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14920
14921 match(Set dst (MoveD2L src));
14922
14923 effect(DEF dst, USE src);
14924
14925 ins_cost(INSN_COST);
14926
14927 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14928
14929 ins_encode %{
14930 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14931 %}
14932
14933 ins_pipe(pipe_class_memory);
14934
14935 %}
14936
14937 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14938
14939 match(Set dst (MoveL2D src));
14940
14941 effect(DEF dst, USE src);
14942
14943 ins_cost(INSN_COST);
14944
14945 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14946
14947 ins_encode %{
14948 __ str($src$$Register, Address(sp, $dst$$disp));
14949 %}
14950
14951 ins_pipe(istore_reg_reg);
14952
14953 %}
14954
14955 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14956
14957 match(Set dst (MoveF2I src));
14958
14959 effect(DEF dst, USE src);
14960
14961 ins_cost(INSN_COST);
14962
14963 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14964
14965 ins_encode %{
14966 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14967 %}
14968
14969 ins_pipe(fp_f2i);
14970
14971 %}
14972
14973 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14974
14975 match(Set dst (MoveI2F src));
14976
14977 effect(DEF dst, USE src);
14978
14979 ins_cost(INSN_COST);
14980
14981 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14982
14983 ins_encode %{
14984 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14985 %}
14986
14987 ins_pipe(fp_i2f);
14988
14989 %}
14990
14991 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14992
14993 match(Set dst (MoveD2L src));
14994
14995 effect(DEF dst, USE src);
14996
14997 ins_cost(INSN_COST);
14998
14999 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15000
15001 ins_encode %{
15002 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15003 %}
15004
15005 ins_pipe(fp_d2l);
15006
15007 %}
15008
15009 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15010
15011 match(Set dst (MoveL2D src));
15012
15013 effect(DEF dst, USE src);
15014
15015 ins_cost(INSN_COST);
15016
15017 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15018
15019 ins_encode %{
15020 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15021 %}
15022
15023 ins_pipe(fp_l2d);
15024
15025 %}
15026
15027 // ============================================================================
15028 // clearing of an array
15029
15030 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
15031 %{
15032 match(Set dummy (ClearArray (Binary cnt base) zero));
15033 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15034
15035 ins_cost(4 * INSN_COST);
15036 format %{ "ClearArray $cnt, $base" %}
15037
15038 ins_encode %{
15039 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15040 if (tpc == nullptr) {
15041 ciEnv::current()->record_failure("CodeCache is full");
15042 return;
15043 }
15044 %}
15045
15046 ins_pipe(pipe_class_memory);
15047 %}
15048
15049 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
15050 %{
15051 predicate(((ClearArrayNode*)n)->word_copy_only());
15052 match(Set dummy (ClearArray (Binary cnt base) val));
15053 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15054
15055 ins_cost(4 * INSN_COST);
15056 format %{ "ClearArray $cnt, $base, $val" %}
15057
15058 ins_encode %{
15059 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
15060 %}
15061
15062 ins_pipe(pipe_class_memory);
15063 %}
15064
15065 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15066 %{
15067 predicate((uint64_t)n->in(2)->get_long()
15068 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
15069 && !((ClearArrayNode*)n)->word_copy_only());
15070 match(Set dummy (ClearArray cnt base));
15071 effect(TEMP temp, USE_KILL base, KILL cr);
15072
15073 ins_cost(4 * INSN_COST);
15074 format %{ "ClearArray $cnt, $base" %}
15075
15076 ins_encode %{
15077 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15078 if (tpc == nullptr) {
15079 ciEnv::current()->record_failure("CodeCache is full");
15080 return;
15081 }
15082 %}
15083
15084 ins_pipe(pipe_class_memory);
15085 %}
15086
15087 // ============================================================================
15088 // Overflow Math Instructions
15089
15090 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15091 %{
15092 match(Set cr (OverflowAddI op1 op2));
15093
15094 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15095 ins_cost(INSN_COST);
15096 ins_encode %{
15097 __ cmnw($op1$$Register, $op2$$Register);
15098 %}
15099
15100 ins_pipe(icmp_reg_reg);
15101 %}
15102
15103 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15104 %{
15105 match(Set cr (OverflowAddI op1 op2));
15106
15107 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15108 ins_cost(INSN_COST);
15109 ins_encode %{
15110 __ cmnw($op1$$Register, $op2$$constant);
15111 %}
15112
15113 ins_pipe(icmp_reg_imm);
15114 %}
15115
15116 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15117 %{
15118 match(Set cr (OverflowAddL op1 op2));
15119
15120 format %{ "cmn $op1, $op2\t# overflow check long" %}
15121 ins_cost(INSN_COST);
15122 ins_encode %{
15123 __ cmn($op1$$Register, $op2$$Register);
15124 %}
15125
15126 ins_pipe(icmp_reg_reg);
15127 %}
15128
15129 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15130 %{
15131 match(Set cr (OverflowAddL op1 op2));
15132
15133 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15134 ins_cost(INSN_COST);
15135 ins_encode %{
15136 __ adds(zr, $op1$$Register, $op2$$constant);
15137 %}
15138
15139 ins_pipe(icmp_reg_imm);
15140 %}
15141
15142 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15143 %{
15144 match(Set cr (OverflowSubI op1 op2));
15145
15146 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15147 ins_cost(INSN_COST);
15148 ins_encode %{
15149 __ cmpw($op1$$Register, $op2$$Register);
15150 %}
15151
15152 ins_pipe(icmp_reg_reg);
15153 %}
15154
15155 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15156 %{
15157 match(Set cr (OverflowSubI op1 op2));
15158
15159 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15160 ins_cost(INSN_COST);
15161 ins_encode %{
15162 __ cmpw($op1$$Register, $op2$$constant);
15163 %}
15164
15165 ins_pipe(icmp_reg_imm);
15166 %}
15167
15168 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15169 %{
15170 match(Set cr (OverflowSubL op1 op2));
15171
15172 format %{ "cmp $op1, $op2\t# overflow check long" %}
15173 ins_cost(INSN_COST);
15174 ins_encode %{
15175 __ cmp($op1$$Register, $op2$$Register);
15176 %}
15177
15178 ins_pipe(icmp_reg_reg);
15179 %}
15180
15181 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15182 %{
15183 match(Set cr (OverflowSubL op1 op2));
15184
15185 format %{ "cmp $op1, $op2\t# overflow check long" %}
15186 ins_cost(INSN_COST);
15187 ins_encode %{
15188 __ subs(zr, $op1$$Register, $op2$$constant);
15189 %}
15190
15191 ins_pipe(icmp_reg_imm);
15192 %}
15193
15194 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15195 %{
15196 match(Set cr (OverflowSubI zero op1));
15197
15198 format %{ "cmpw zr, $op1\t# overflow check int" %}
15199 ins_cost(INSN_COST);
15200 ins_encode %{
15201 __ cmpw(zr, $op1$$Register);
15202 %}
15203
15204 ins_pipe(icmp_reg_imm);
15205 %}
15206
15207 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15208 %{
15209 match(Set cr (OverflowSubL zero op1));
15210
15211 format %{ "cmp zr, $op1\t# overflow check long" %}
15212 ins_cost(INSN_COST);
15213 ins_encode %{
15214 __ cmp(zr, $op1$$Register);
15215 %}
15216
15217 ins_pipe(icmp_reg_imm);
15218 %}
15219
15220 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15221 %{
15222 match(Set cr (OverflowMulI op1 op2));
15223
15224 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15225 "cmp rscratch1, rscratch1, sxtw\n\t"
15226 "movw rscratch1, #0x80000000\n\t"
15227 "cselw rscratch1, rscratch1, zr, NE\n\t"
15228 "cmpw rscratch1, #1" %}
15229 ins_cost(5 * INSN_COST);
15230 ins_encode %{
15231 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15232 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15233 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15234 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15235 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15236 %}
15237
15238 ins_pipe(pipe_slow);
15239 %}
15240
15241 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15242 %{
15243 match(If cmp (OverflowMulI op1 op2));
15244 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15245 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15246 effect(USE labl, KILL cr);
15247
15248 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15249 "cmp rscratch1, rscratch1, sxtw\n\t"
15250 "b$cmp $labl" %}
15251 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15252 ins_encode %{
15253 Label* L = $labl$$label;
15254 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15255 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15256 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15257 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15258 %}
15259
15260 ins_pipe(pipe_serial);
15261 %}
15262
15263 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15264 %{
15265 match(Set cr (OverflowMulL op1 op2));
15266
15267 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15268 "smulh rscratch2, $op1, $op2\n\t"
15269 "cmp rscratch2, rscratch1, ASR #63\n\t"
15270 "movw rscratch1, #0x80000000\n\t"
15271 "cselw rscratch1, rscratch1, zr, NE\n\t"
15272 "cmpw rscratch1, #1" %}
15273 ins_cost(6 * INSN_COST);
15274 ins_encode %{
15275 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15276 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15277 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15278 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15279 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15280 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15281 %}
15282
15283 ins_pipe(pipe_slow);
15284 %}
15285
15286 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15287 %{
15288 match(If cmp (OverflowMulL op1 op2));
15289 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15290 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15291 effect(USE labl, KILL cr);
15292
15293 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15294 "smulh rscratch2, $op1, $op2\n\t"
15295 "cmp rscratch2, rscratch1, ASR #63\n\t"
15296 "b$cmp $labl" %}
15297 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15298 ins_encode %{
15299 Label* L = $labl$$label;
15300 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15301 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15302 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15303 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15304 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15305 %}
15306
15307 ins_pipe(pipe_serial);
15308 %}
15309
15310 // ============================================================================
15311 // Compare Instructions
15312
15313 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15314 %{
15315 match(Set cr (CmpI op1 op2));
15316
15317 effect(DEF cr, USE op1, USE op2);
15318
15319 ins_cost(INSN_COST);
15320 format %{ "cmpw $op1, $op2" %}
15321
15322 ins_encode(aarch64_enc_cmpw(op1, op2));
15323
15324 ins_pipe(icmp_reg_reg);
15325 %}
15326
15327 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15328 %{
15329 match(Set cr (CmpI op1 zero));
15330
15331 effect(DEF cr, USE op1);
15332
15333 ins_cost(INSN_COST);
15334 format %{ "cmpw $op1, 0" %}
15335
15336 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15337
15338 ins_pipe(icmp_reg_imm);
15339 %}
15340
15341 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15342 %{
15343 match(Set cr (CmpI op1 op2));
15344
15345 effect(DEF cr, USE op1);
15346
15347 ins_cost(INSN_COST);
15348 format %{ "cmpw $op1, $op2" %}
15349
15350 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15351
15352 ins_pipe(icmp_reg_imm);
15353 %}
15354
15355 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15356 %{
15357 match(Set cr (CmpI op1 op2));
15358
15359 effect(DEF cr, USE op1);
15360
15361 ins_cost(INSN_COST * 2);
15362 format %{ "cmpw $op1, $op2" %}
15363
15364 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15365
15366 ins_pipe(icmp_reg_imm);
15367 %}
15368
15369 // Unsigned compare Instructions; really, same as signed compare
15370 // except it should only be used to feed an If or a CMovI which takes a
15371 // cmpOpU.
15372
15373 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15374 %{
15375 match(Set cr (CmpU op1 op2));
15376
15377 effect(DEF cr, USE op1, USE op2);
15378
15379 ins_cost(INSN_COST);
15380 format %{ "cmpw $op1, $op2\t# unsigned" %}
15381
15382 ins_encode(aarch64_enc_cmpw(op1, op2));
15383
15384 ins_pipe(icmp_reg_reg);
15385 %}
15386
15387 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15388 %{
15389 match(Set cr (CmpU op1 zero));
15390
15391 effect(DEF cr, USE op1);
15392
15393 ins_cost(INSN_COST);
15394 format %{ "cmpw $op1, #0\t# unsigned" %}
15395
15396 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15397
15398 ins_pipe(icmp_reg_imm);
15399 %}
15400
15401 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15402 %{
15403 match(Set cr (CmpU op1 op2));
15404
15405 effect(DEF cr, USE op1);
15406
15407 ins_cost(INSN_COST);
15408 format %{ "cmpw $op1, $op2\t# unsigned" %}
15409
15410 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15411
15412 ins_pipe(icmp_reg_imm);
15413 %}
15414
15415 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15416 %{
15417 match(Set cr (CmpU op1 op2));
15418
15419 effect(DEF cr, USE op1);
15420
15421 ins_cost(INSN_COST * 2);
15422 format %{ "cmpw $op1, $op2\t# unsigned" %}
15423
15424 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15425
15426 ins_pipe(icmp_reg_imm);
15427 %}
15428
15429 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15430 %{
15431 match(Set cr (CmpL op1 op2));
15432
15433 effect(DEF cr, USE op1, USE op2);
15434
15435 ins_cost(INSN_COST);
15436 format %{ "cmp $op1, $op2" %}
15437
15438 ins_encode(aarch64_enc_cmp(op1, op2));
15439
15440 ins_pipe(icmp_reg_reg);
15441 %}
15442
15443 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15444 %{
15445 match(Set cr (CmpL op1 zero));
15446
15447 effect(DEF cr, USE op1);
15448
15449 ins_cost(INSN_COST);
15450 format %{ "tst $op1" %}
15451
15452 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15453
15454 ins_pipe(icmp_reg_imm);
15455 %}
15456
15457 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15458 %{
15459 match(Set cr (CmpL op1 op2));
15460
15461 effect(DEF cr, USE op1);
15462
15463 ins_cost(INSN_COST);
15464 format %{ "cmp $op1, $op2" %}
15465
15466 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15467
15468 ins_pipe(icmp_reg_imm);
15469 %}
15470
15471 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15472 %{
15473 match(Set cr (CmpL op1 op2));
15474
15475 effect(DEF cr, USE op1);
15476
15477 ins_cost(INSN_COST * 2);
15478 format %{ "cmp $op1, $op2" %}
15479
15480 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15481
15482 ins_pipe(icmp_reg_imm);
15483 %}
15484
15485 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15486 %{
15487 match(Set cr (CmpUL op1 op2));
15488
15489 effect(DEF cr, USE op1, USE op2);
15490
15491 ins_cost(INSN_COST);
15492 format %{ "cmp $op1, $op2" %}
15493
15494 ins_encode(aarch64_enc_cmp(op1, op2));
15495
15496 ins_pipe(icmp_reg_reg);
15497 %}
15498
15499 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15500 %{
15501 match(Set cr (CmpUL op1 zero));
15502
15503 effect(DEF cr, USE op1);
15504
15505 ins_cost(INSN_COST);
15506 format %{ "tst $op1" %}
15507
15508 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15509
15510 ins_pipe(icmp_reg_imm);
15511 %}
15512
15513 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15514 %{
15515 match(Set cr (CmpUL op1 op2));
15516
15517 effect(DEF cr, USE op1);
15518
15519 ins_cost(INSN_COST);
15520 format %{ "cmp $op1, $op2" %}
15521
15522 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15523
15524 ins_pipe(icmp_reg_imm);
15525 %}
15526
15527 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15528 %{
15529 match(Set cr (CmpUL op1 op2));
15530
15531 effect(DEF cr, USE op1);
15532
15533 ins_cost(INSN_COST * 2);
15534 format %{ "cmp $op1, $op2" %}
15535
15536 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15537
15538 ins_pipe(icmp_reg_imm);
15539 %}
15540
15541 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15542 %{
15543 match(Set cr (CmpP op1 op2));
15544
15545 effect(DEF cr, USE op1, USE op2);
15546
15547 ins_cost(INSN_COST);
15548 format %{ "cmp $op1, $op2\t // ptr" %}
15549
15550 ins_encode(aarch64_enc_cmpp(op1, op2));
15551
15552 ins_pipe(icmp_reg_reg);
15553 %}
15554
15555 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15556 %{
15557 match(Set cr (CmpN op1 op2));
15558
15559 effect(DEF cr, USE op1, USE op2);
15560
15561 ins_cost(INSN_COST);
15562 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15563
15564 ins_encode(aarch64_enc_cmpn(op1, op2));
15565
15566 ins_pipe(icmp_reg_reg);
15567 %}
15568
15569 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15570 %{
15571 match(Set cr (CmpP op1 zero));
15572
15573 effect(DEF cr, USE op1, USE zero);
15574
15575 ins_cost(INSN_COST);
15576 format %{ "cmp $op1, 0\t // ptr" %}
15577
15578 ins_encode(aarch64_enc_testp(op1));
15579
15580 ins_pipe(icmp_reg_imm);
15581 %}
15582
15583 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15584 %{
15585 match(Set cr (CmpN op1 zero));
15586
15587 effect(DEF cr, USE op1, USE zero);
15588
15589 ins_cost(INSN_COST);
15590 format %{ "cmp $op1, 0\t // compressed ptr" %}
15591
15592 ins_encode(aarch64_enc_testn(op1));
15593
15594 ins_pipe(icmp_reg_imm);
15595 %}
15596
15597 // FP comparisons
15598 //
15599 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15600 // using normal cmpOp. See declaration of rFlagsReg for details.
15601
15602 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15603 %{
15604 match(Set cr (CmpF src1 src2));
15605
15606 ins_cost(3 * INSN_COST);
15607 format %{ "fcmps $src1, $src2" %}
15608
15609 ins_encode %{
15610 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15611 %}
15612
15613 ins_pipe(pipe_class_compare);
15614 %}
15615
15616 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15617 %{
15618 match(Set cr (CmpF src1 src2));
15619
15620 ins_cost(3 * INSN_COST);
15621 format %{ "fcmps $src1, 0.0" %}
15622
15623 ins_encode %{
15624 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15625 %}
15626
15627 ins_pipe(pipe_class_compare);
15628 %}
15629 // FROM HERE
15630
15631 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15632 %{
15633 match(Set cr (CmpD src1 src2));
15634
15635 ins_cost(3 * INSN_COST);
15636 format %{ "fcmpd $src1, $src2" %}
15637
15638 ins_encode %{
15639 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15640 %}
15641
15642 ins_pipe(pipe_class_compare);
15643 %}
15644
15645 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15646 %{
15647 match(Set cr (CmpD src1 src2));
15648
15649 ins_cost(3 * INSN_COST);
15650 format %{ "fcmpd $src1, 0.0" %}
15651
15652 ins_encode %{
15653 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15654 %}
15655
15656 ins_pipe(pipe_class_compare);
15657 %}
15658
15659 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15660 %{
15661 match(Set dst (CmpF3 src1 src2));
15662 effect(KILL cr);
15663
15664 ins_cost(5 * INSN_COST);
15665 format %{ "fcmps $src1, $src2\n\t"
15666 "csinvw($dst, zr, zr, eq\n\t"
15667 "csnegw($dst, $dst, $dst, lt)"
15668 %}
15669
15670 ins_encode %{
15671 Label done;
15672 FloatRegister s1 = as_FloatRegister($src1$$reg);
15673 FloatRegister s2 = as_FloatRegister($src2$$reg);
15674 Register d = as_Register($dst$$reg);
15675 __ fcmps(s1, s2);
15676 // installs 0 if EQ else -1
15677 __ csinvw(d, zr, zr, Assembler::EQ);
15678 // keeps -1 if less or unordered else installs 1
15679 __ csnegw(d, d, d, Assembler::LT);
15680 __ bind(done);
15681 %}
15682
15683 ins_pipe(pipe_class_default);
15684
15685 %}
15686
15687 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15688 %{
15689 match(Set dst (CmpD3 src1 src2));
15690 effect(KILL cr);
15691
15692 ins_cost(5 * INSN_COST);
15693 format %{ "fcmpd $src1, $src2\n\t"
15694 "csinvw($dst, zr, zr, eq\n\t"
15695 "csnegw($dst, $dst, $dst, lt)"
15696 %}
15697
15698 ins_encode %{
15699 Label done;
15700 FloatRegister s1 = as_FloatRegister($src1$$reg);
15701 FloatRegister s2 = as_FloatRegister($src2$$reg);
15702 Register d = as_Register($dst$$reg);
15703 __ fcmpd(s1, s2);
15704 // installs 0 if EQ else -1
15705 __ csinvw(d, zr, zr, Assembler::EQ);
15706 // keeps -1 if less or unordered else installs 1
15707 __ csnegw(d, d, d, Assembler::LT);
15708 __ bind(done);
15709 %}
15710 ins_pipe(pipe_class_default);
15711
15712 %}
15713
15714 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15715 %{
15716 match(Set dst (CmpF3 src1 zero));
15717 effect(KILL cr);
15718
15719 ins_cost(5 * INSN_COST);
15720 format %{ "fcmps $src1, 0.0\n\t"
15721 "csinvw($dst, zr, zr, eq\n\t"
15722 "csnegw($dst, $dst, $dst, lt)"
15723 %}
15724
15725 ins_encode %{
15726 Label done;
15727 FloatRegister s1 = as_FloatRegister($src1$$reg);
15728 Register d = as_Register($dst$$reg);
15729 __ fcmps(s1, 0.0);
15730 // installs 0 if EQ else -1
15731 __ csinvw(d, zr, zr, Assembler::EQ);
15732 // keeps -1 if less or unordered else installs 1
15733 __ csnegw(d, d, d, Assembler::LT);
15734 __ bind(done);
15735 %}
15736
15737 ins_pipe(pipe_class_default);
15738
15739 %}
15740
15741 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15742 %{
15743 match(Set dst (CmpD3 src1 zero));
15744 effect(KILL cr);
15745
15746 ins_cost(5 * INSN_COST);
15747 format %{ "fcmpd $src1, 0.0\n\t"
15748 "csinvw($dst, zr, zr, eq\n\t"
15749 "csnegw($dst, $dst, $dst, lt)"
15750 %}
15751
15752 ins_encode %{
15753 Label done;
15754 FloatRegister s1 = as_FloatRegister($src1$$reg);
15755 Register d = as_Register($dst$$reg);
15756 __ fcmpd(s1, 0.0);
15757 // installs 0 if EQ else -1
15758 __ csinvw(d, zr, zr, Assembler::EQ);
15759 // keeps -1 if less or unordered else installs 1
15760 __ csnegw(d, d, d, Assembler::LT);
15761 __ bind(done);
15762 %}
15763 ins_pipe(pipe_class_default);
15764
15765 %}
15766
15767 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15768 %{
15769 match(Set dst (CmpLTMask p q));
15770 effect(KILL cr);
15771
15772 ins_cost(3 * INSN_COST);
15773
15774 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15775 "csetw $dst, lt\n\t"
15776 "subw $dst, zr, $dst"
15777 %}
15778
15779 ins_encode %{
15780 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15781 __ csetw(as_Register($dst$$reg), Assembler::LT);
15782 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15783 %}
15784
15785 ins_pipe(ialu_reg_reg);
15786 %}
15787
15788 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15789 %{
15790 match(Set dst (CmpLTMask src zero));
15791 effect(KILL cr);
15792
15793 ins_cost(INSN_COST);
15794
15795 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15796
15797 ins_encode %{
15798 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15799 %}
15800
15801 ins_pipe(ialu_reg_shift);
15802 %}
15803
15804 // ============================================================================
15805 // Max and Min
15806
15807 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15808
15809 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15810 %{
15811 effect(DEF cr, USE src);
15812 ins_cost(INSN_COST);
15813 format %{ "cmpw $src, 0" %}
15814
15815 ins_encode %{
15816 __ cmpw($src$$Register, 0);
15817 %}
15818 ins_pipe(icmp_reg_imm);
15819 %}
15820
15821 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15822 %{
15823 match(Set dst (MinI src1 src2));
15824 ins_cost(INSN_COST * 3);
15825
15826 expand %{
15827 rFlagsReg cr;
15828 compI_reg_reg(cr, src1, src2);
15829 cmovI_reg_reg_lt(dst, src1, src2, cr);
15830 %}
15831 %}
15832
15833 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15834 %{
15835 match(Set dst (MaxI src1 src2));
15836 ins_cost(INSN_COST * 3);
15837
15838 expand %{
15839 rFlagsReg cr;
15840 compI_reg_reg(cr, src1, src2);
15841 cmovI_reg_reg_gt(dst, src1, src2, cr);
15842 %}
15843 %}
15844
15845
15846 // ============================================================================
15847 // Branch Instructions
15848
15849 // Direct Branch.
15850 instruct branch(label lbl)
15851 %{
15852 match(Goto);
15853
15854 effect(USE lbl);
15855
15856 ins_cost(BRANCH_COST);
15857 format %{ "b $lbl" %}
15858
15859 ins_encode(aarch64_enc_b(lbl));
15860
15861 ins_pipe(pipe_branch);
15862 %}
15863
15864 // Conditional Near Branch
15865 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15866 %{
15867 // Same match rule as `branchConFar'.
15868 match(If cmp cr);
15869
15870 effect(USE lbl);
15871
15872 ins_cost(BRANCH_COST);
15873 // If set to 1 this indicates that the current instruction is a
15874 // short variant of a long branch. This avoids using this
15875 // instruction in first-pass matching. It will then only be used in
15876 // the `Shorten_branches' pass.
15877 // ins_short_branch(1);
15878 format %{ "b$cmp $lbl" %}
15879
15880 ins_encode(aarch64_enc_br_con(cmp, lbl));
15881
15882 ins_pipe(pipe_branch_cond);
15883 %}
15884
15885 // Conditional Near Branch Unsigned
15886 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15887 %{
15888 // Same match rule as `branchConFar'.
15889 match(If cmp cr);
15890
15891 effect(USE lbl);
15892
15893 ins_cost(BRANCH_COST);
15894 // If set to 1 this indicates that the current instruction is a
15895 // short variant of a long branch. This avoids using this
15896 // instruction in first-pass matching. It will then only be used in
15897 // the `Shorten_branches' pass.
15898 // ins_short_branch(1);
15899 format %{ "b$cmp $lbl\t# unsigned" %}
15900
15901 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15902
15903 ins_pipe(pipe_branch_cond);
15904 %}
15905
15906 // Make use of CBZ and CBNZ. These instructions, as well as being
15907 // shorter than (cmp; branch), have the additional benefit of not
15908 // killing the flags.
15909
15910 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15911 match(If cmp (CmpI op1 op2));
15912 effect(USE labl);
15913
15914 ins_cost(BRANCH_COST);
15915 format %{ "cbw$cmp $op1, $labl" %}
15916 ins_encode %{
15917 Label* L = $labl$$label;
15918 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15919 if (cond == Assembler::EQ)
15920 __ cbzw($op1$$Register, *L);
15921 else
15922 __ cbnzw($op1$$Register, *L);
15923 %}
15924 ins_pipe(pipe_cmp_branch);
15925 %}
15926
15927 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15928 match(If cmp (CmpL op1 op2));
15929 effect(USE labl);
15930
15931 ins_cost(BRANCH_COST);
15932 format %{ "cb$cmp $op1, $labl" %}
15933 ins_encode %{
15934 Label* L = $labl$$label;
15935 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15936 if (cond == Assembler::EQ)
15937 __ cbz($op1$$Register, *L);
15938 else
15939 __ cbnz($op1$$Register, *L);
15940 %}
15941 ins_pipe(pipe_cmp_branch);
15942 %}
15943
15944 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15945 match(If cmp (CmpP op1 op2));
15946 effect(USE labl);
15947
15948 ins_cost(BRANCH_COST);
15949 format %{ "cb$cmp $op1, $labl" %}
15950 ins_encode %{
15951 Label* L = $labl$$label;
15952 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15953 if (cond == Assembler::EQ)
15954 __ cbz($op1$$Register, *L);
15955 else
15956 __ cbnz($op1$$Register, *L);
15957 %}
15958 ins_pipe(pipe_cmp_branch);
15959 %}
15960
15961 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15962 match(If cmp (CmpN op1 op2));
15963 effect(USE labl);
15964
15965 ins_cost(BRANCH_COST);
15966 format %{ "cbw$cmp $op1, $labl" %}
15967 ins_encode %{
15968 Label* L = $labl$$label;
15969 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15970 if (cond == Assembler::EQ)
15971 __ cbzw($op1$$Register, *L);
15972 else
15973 __ cbnzw($op1$$Register, *L);
15974 %}
15975 ins_pipe(pipe_cmp_branch);
15976 %}
15977
15978 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15979 match(If cmp (CmpP (DecodeN oop) zero));
15980 effect(USE labl);
15981
15982 ins_cost(BRANCH_COST);
15983 format %{ "cb$cmp $oop, $labl" %}
15984 ins_encode %{
15985 Label* L = $labl$$label;
15986 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15987 if (cond == Assembler::EQ)
15988 __ cbzw($oop$$Register, *L);
15989 else
15990 __ cbnzw($oop$$Register, *L);
15991 %}
15992 ins_pipe(pipe_cmp_branch);
15993 %}
15994
15995 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15996 match(If cmp (CmpU op1 op2));
15997 effect(USE labl);
15998
15999 ins_cost(BRANCH_COST);
16000 format %{ "cbw$cmp $op1, $labl" %}
16001 ins_encode %{
16002 Label* L = $labl$$label;
16003 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16004 if (cond == Assembler::EQ || cond == Assembler::LS) {
16005 __ cbzw($op1$$Register, *L);
16006 } else {
16007 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16008 __ cbnzw($op1$$Register, *L);
16009 }
16010 %}
16011 ins_pipe(pipe_cmp_branch);
16012 %}
16013
16014 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
16015 match(If cmp (CmpUL op1 op2));
16016 effect(USE labl);
16017
16018 ins_cost(BRANCH_COST);
16019 format %{ "cb$cmp $op1, $labl" %}
16020 ins_encode %{
16021 Label* L = $labl$$label;
16022 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16023 if (cond == Assembler::EQ || cond == Assembler::LS) {
16024 __ cbz($op1$$Register, *L);
16025 } else {
16026 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16027 __ cbnz($op1$$Register, *L);
16028 }
16029 %}
16030 ins_pipe(pipe_cmp_branch);
16031 %}
16032
16033 // Test bit and Branch
16034
16035 // Patterns for short (< 32KiB) variants
16036 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16037 match(If cmp (CmpL op1 op2));
16038 effect(USE labl);
16039
16040 ins_cost(BRANCH_COST);
16041 format %{ "cb$cmp $op1, $labl # long" %}
16042 ins_encode %{
16043 Label* L = $labl$$label;
16044 Assembler::Condition cond =
16045 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16046 __ tbr(cond, $op1$$Register, 63, *L);
16047 %}
16048 ins_pipe(pipe_cmp_branch);
16049 ins_short_branch(1);
16050 %}
16051
16052 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16053 match(If cmp (CmpI op1 op2));
16054 effect(USE labl);
16055
16056 ins_cost(BRANCH_COST);
16057 format %{ "cb$cmp $op1, $labl # int" %}
16058 ins_encode %{
16059 Label* L = $labl$$label;
16060 Assembler::Condition cond =
16061 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16062 __ tbr(cond, $op1$$Register, 31, *L);
16063 %}
16064 ins_pipe(pipe_cmp_branch);
16065 ins_short_branch(1);
16066 %}
16067
16068 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16069 match(If cmp (CmpL (AndL op1 op2) op3));
16070 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16071 effect(USE labl);
16072
16073 ins_cost(BRANCH_COST);
16074 format %{ "tb$cmp $op1, $op2, $labl" %}
16075 ins_encode %{
16076 Label* L = $labl$$label;
16077 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16078 int bit = exact_log2_long($op2$$constant);
16079 __ tbr(cond, $op1$$Register, bit, *L);
16080 %}
16081 ins_pipe(pipe_cmp_branch);
16082 ins_short_branch(1);
16083 %}
16084
16085 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16086 match(If cmp (CmpI (AndI op1 op2) op3));
16087 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16088 effect(USE labl);
16089
16090 ins_cost(BRANCH_COST);
16091 format %{ "tb$cmp $op1, $op2, $labl" %}
16092 ins_encode %{
16093 Label* L = $labl$$label;
16094 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16095 int bit = exact_log2((juint)$op2$$constant);
16096 __ tbr(cond, $op1$$Register, bit, *L);
16097 %}
16098 ins_pipe(pipe_cmp_branch);
16099 ins_short_branch(1);
16100 %}
16101
16102 // And far variants
16103 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16104 match(If cmp (CmpL op1 op2));
16105 effect(USE labl);
16106
16107 ins_cost(BRANCH_COST);
16108 format %{ "cb$cmp $op1, $labl # long" %}
16109 ins_encode %{
16110 Label* L = $labl$$label;
16111 Assembler::Condition cond =
16112 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16113 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16114 %}
16115 ins_pipe(pipe_cmp_branch);
16116 %}
16117
16118 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16119 match(If cmp (CmpI op1 op2));
16120 effect(USE labl);
16121
16122 ins_cost(BRANCH_COST);
16123 format %{ "cb$cmp $op1, $labl # int" %}
16124 ins_encode %{
16125 Label* L = $labl$$label;
16126 Assembler::Condition cond =
16127 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16128 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16129 %}
16130 ins_pipe(pipe_cmp_branch);
16131 %}
16132
16133 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16134 match(If cmp (CmpL (AndL op1 op2) op3));
16135 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16136 effect(USE labl);
16137
16138 ins_cost(BRANCH_COST);
16139 format %{ "tb$cmp $op1, $op2, $labl" %}
16140 ins_encode %{
16141 Label* L = $labl$$label;
16142 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16143 int bit = exact_log2_long($op2$$constant);
16144 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16145 %}
16146 ins_pipe(pipe_cmp_branch);
16147 %}
16148
16149 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16150 match(If cmp (CmpI (AndI op1 op2) op3));
16151 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16152 effect(USE labl);
16153
16154 ins_cost(BRANCH_COST);
16155 format %{ "tb$cmp $op1, $op2, $labl" %}
16156 ins_encode %{
16157 Label* L = $labl$$label;
16158 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16159 int bit = exact_log2((juint)$op2$$constant);
16160 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16161 %}
16162 ins_pipe(pipe_cmp_branch);
16163 %}
16164
16165 // Test bits
16166
16167 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16168 match(Set cr (CmpL (AndL op1 op2) op3));
16169 predicate(Assembler::operand_valid_for_logical_immediate
16170 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16171
16172 ins_cost(INSN_COST);
16173 format %{ "tst $op1, $op2 # long" %}
16174 ins_encode %{
16175 __ tst($op1$$Register, $op2$$constant);
16176 %}
16177 ins_pipe(ialu_reg_reg);
16178 %}
16179
16180 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16181 match(Set cr (CmpI (AndI op1 op2) op3));
16182 predicate(Assembler::operand_valid_for_logical_immediate
16183 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16184
16185 ins_cost(INSN_COST);
16186 format %{ "tst $op1, $op2 # int" %}
16187 ins_encode %{
16188 __ tstw($op1$$Register, $op2$$constant);
16189 %}
16190 ins_pipe(ialu_reg_reg);
16191 %}
16192
16193 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16194 match(Set cr (CmpL (AndL op1 op2) op3));
16195
16196 ins_cost(INSN_COST);
16197 format %{ "tst $op1, $op2 # long" %}
16198 ins_encode %{
16199 __ tst($op1$$Register, $op2$$Register);
16200 %}
16201 ins_pipe(ialu_reg_reg);
16202 %}
16203
16204 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16205 match(Set cr (CmpI (AndI op1 op2) op3));
16206
16207 ins_cost(INSN_COST);
16208 format %{ "tstw $op1, $op2 # int" %}
16209 ins_encode %{
16210 __ tstw($op1$$Register, $op2$$Register);
16211 %}
16212 ins_pipe(ialu_reg_reg);
16213 %}
16214
16215
16216 // Conditional Far Branch
16217 // Conditional Far Branch Unsigned
16218 // TODO: fixme
16219
16220 // counted loop end branch near
16221 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16222 %{
16223 match(CountedLoopEnd cmp cr);
16224
16225 effect(USE lbl);
16226
16227 ins_cost(BRANCH_COST);
16228 // short variant.
16229 // ins_short_branch(1);
16230 format %{ "b$cmp $lbl \t// counted loop end" %}
16231
16232 ins_encode(aarch64_enc_br_con(cmp, lbl));
16233
16234 ins_pipe(pipe_branch);
16235 %}
16236
16237 // counted loop end branch far
16238 // TODO: fixme
16239
16240 // ============================================================================
16241 // inlined locking and unlocking
16242
16243 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16244 %{
16245 predicate(LockingMode != LM_LIGHTWEIGHT);
16246 match(Set cr (FastLock object box));
16247 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16248
16249 ins_cost(5 * INSN_COST);
16250 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16251
16252 ins_encode %{
16253 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16254 %}
16255
16256 ins_pipe(pipe_serial);
16257 %}
16258
16259 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16260 %{
16261 predicate(LockingMode != LM_LIGHTWEIGHT);
16262 match(Set cr (FastUnlock object box));
16263 effect(TEMP tmp, TEMP tmp2);
16264
16265 ins_cost(5 * INSN_COST);
16266 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16267
16268 ins_encode %{
16269 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16270 %}
16271
16272 ins_pipe(pipe_serial);
16273 %}
16274
16275 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16276 %{
16277 predicate(LockingMode == LM_LIGHTWEIGHT);
16278 match(Set cr (FastLock object box));
16279 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16280
16281 ins_cost(5 * INSN_COST);
16282 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16283
16284 ins_encode %{
16285 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16286 %}
16287
16288 ins_pipe(pipe_serial);
16289 %}
16290
16291 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16292 %{
16293 predicate(LockingMode == LM_LIGHTWEIGHT);
16294 match(Set cr (FastUnlock object box));
16295 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16296
16297 ins_cost(5 * INSN_COST);
16298 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16299
16300 ins_encode %{
16301 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16302 %}
16303
16304 ins_pipe(pipe_serial);
16305 %}
16306
16307 // ============================================================================
16308 // Safepoint Instructions
16309
16310 // TODO
16311 // provide a near and far version of this code
16312
16313 instruct safePoint(rFlagsReg cr, iRegP poll)
16314 %{
16315 match(SafePoint poll);
16316 effect(KILL cr);
16317
16318 format %{
16319 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16320 %}
16321 ins_encode %{
16322 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16323 %}
16324 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16325 %}
16326
16327
16328 // ============================================================================
16329 // Procedure Call/Return Instructions
16330
16331 // Call Java Static Instruction
16332
16333 instruct CallStaticJavaDirect(method meth)
16334 %{
16335 match(CallStaticJava);
16336
16337 effect(USE meth);
16338
16339 ins_cost(CALL_COST);
16340
16341 format %{ "call,static $meth \t// ==> " %}
16342
16343 ins_encode(aarch64_enc_java_static_call(meth),
16344 aarch64_enc_call_epilog);
16345
16346 ins_pipe(pipe_class_call);
16347 %}
16348
16349 // TO HERE
16350
16351 // Call Java Dynamic Instruction
16352 instruct CallDynamicJavaDirect(method meth)
16353 %{
16354 match(CallDynamicJava);
16355
16356 effect(USE meth);
16357
16358 ins_cost(CALL_COST);
16359
16360 format %{ "CALL,dynamic $meth \t// ==> " %}
16361
16362 ins_encode(aarch64_enc_java_dynamic_call(meth),
16363 aarch64_enc_call_epilog);
16364
16365 ins_pipe(pipe_class_call);
16366 %}
16367
16368 // Call Runtime Instruction
16369
16370 instruct CallRuntimeDirect(method meth)
16371 %{
16372 match(CallRuntime);
16373
16374 effect(USE meth);
16375
16376 ins_cost(CALL_COST);
16377
16378 format %{ "CALL, runtime $meth" %}
16379
16380 ins_encode( aarch64_enc_java_to_runtime(meth) );
16381
16382 ins_pipe(pipe_class_call);
16383 %}
16384
16385 // Call Runtime Instruction
16386
16387 instruct CallLeafDirect(method meth)
16388 %{
16389 match(CallLeaf);
16390
16391 effect(USE meth);
16392
16393 ins_cost(CALL_COST);
16394
16395 format %{ "CALL, runtime leaf $meth" %}
16396
16397 ins_encode( aarch64_enc_java_to_runtime(meth) );
16398
16399 ins_pipe(pipe_class_call);
16400 %}
16401
16402 // Call Runtime Instruction without safepoint and with vector arguments
16403 instruct CallLeafDirectVector(method meth)
16404 %{
16405 match(CallLeafVector);
16406
16407 effect(USE meth);
16408
16409 ins_cost(CALL_COST);
16410
16411 format %{ "CALL, runtime leaf vector $meth" %}
16412
16413 ins_encode(aarch64_enc_java_to_runtime(meth));
16414
16415 ins_pipe(pipe_class_call);
16416 %}
16417
16418 // Call Runtime Instruction
16419
16420 // entry point is null, target holds the address to call
16421 instruct CallLeafNoFPIndirect(iRegP target)
16422 %{
16423 predicate(n->as_Call()->entry_point() == nullptr);
16424
16425 match(CallLeafNoFP target);
16426
16427 ins_cost(CALL_COST);
16428
16429 format %{ "CALL, runtime leaf nofp indirect $target" %}
16430
16431 ins_encode %{
16432 __ blr($target$$Register);
16433 %}
16434
16435 ins_pipe(pipe_class_call);
16436 %}
16437
16438 instruct CallLeafNoFPDirect(method meth)
16439 %{
16440 predicate(n->as_Call()->entry_point() != nullptr);
16441
16442 match(CallLeafNoFP);
16443
16444 effect(USE meth);
16445
16446 ins_cost(CALL_COST);
16447
16448 format %{ "CALL, runtime leaf nofp $meth" %}
16449
16450 ins_encode( aarch64_enc_java_to_runtime(meth) );
16451
16452 ins_pipe(pipe_class_call);
16453 %}
16454
16455 // Tail Call; Jump from runtime stub to Java code.
16456 // Also known as an 'interprocedural jump'.
16457 // Target of jump will eventually return to caller.
16458 // TailJump below removes the return address.
16459 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16460 // emitted just above the TailCall which has reset rfp to the caller state.
16461 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16462 %{
16463 match(TailCall jump_target method_ptr);
16464
16465 ins_cost(CALL_COST);
16466
16467 format %{ "br $jump_target\t# $method_ptr holds method" %}
16468
16469 ins_encode(aarch64_enc_tail_call(jump_target));
16470
16471 ins_pipe(pipe_class_call);
16472 %}
16473
16474 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16475 %{
16476 match(TailJump jump_target ex_oop);
16477
16478 ins_cost(CALL_COST);
16479
16480 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16481
16482 ins_encode(aarch64_enc_tail_jmp(jump_target));
16483
16484 ins_pipe(pipe_class_call);
16485 %}
16486
16487 // Forward exception.
16488 instruct ForwardExceptionjmp()
16489 %{
16490 match(ForwardException);
16491 ins_cost(CALL_COST);
16492
16493 format %{ "b forward_exception_stub" %}
16494 ins_encode %{
16495 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16496 %}
16497 ins_pipe(pipe_class_call);
16498 %}
16499
16500 // Create exception oop: created by stack-crawling runtime code.
16501 // Created exception is now available to this handler, and is setup
16502 // just prior to jumping to this handler. No code emitted.
16503 // TODO check
16504 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16505 instruct CreateException(iRegP_R0 ex_oop)
16506 %{
16507 match(Set ex_oop (CreateEx));
16508
16509 format %{ " -- \t// exception oop; no code emitted" %}
16510
16511 size(0);
16512
16513 ins_encode( /*empty*/ );
16514
16515 ins_pipe(pipe_class_empty);
16516 %}
16517
16518 // Rethrow exception: The exception oop will come in the first
16519 // argument position. Then JUMP (not call) to the rethrow stub code.
16520 instruct RethrowException() %{
16521 match(Rethrow);
16522 ins_cost(CALL_COST);
16523
16524 format %{ "b rethrow_stub" %}
16525
16526 ins_encode( aarch64_enc_rethrow() );
16527
16528 ins_pipe(pipe_class_call);
16529 %}
16530
16531
16532 // Return Instruction
16533 // epilog node loads ret address into lr as part of frame pop
16534 instruct Ret()
16535 %{
16536 match(Return);
16537
16538 format %{ "ret\t// return register" %}
16539
16540 ins_encode( aarch64_enc_ret() );
16541
16542 ins_pipe(pipe_branch);
16543 %}
16544
16545 // Die now.
16546 instruct ShouldNotReachHere() %{
16547 match(Halt);
16548
16549 ins_cost(CALL_COST);
16550 format %{ "ShouldNotReachHere" %}
16551
16552 ins_encode %{
16553 if (is_reachable()) {
16554 const char* str = __ code_string(_halt_reason);
16555 __ stop(str);
16556 }
16557 %}
16558
16559 ins_pipe(pipe_class_default);
16560 %}
16561
16562 // ============================================================================
16563 // Partial Subtype Check
16564 //
16565 // superklass array for an instance of the superklass. Set a hidden
16566 // internal cache on a hit (cache is checked with exposed code in
16567 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16568 // encoding ALSO sets flags.
16569
16570 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16571 %{
16572 match(Set result (PartialSubtypeCheck sub super));
16573 predicate(!UseSecondarySupersTable);
16574 effect(KILL cr, KILL temp);
16575
16576 ins_cost(20 * INSN_COST); // slightly larger than the next version
16577 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16578
16579 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16580
16581 opcode(0x1); // Force zero of result reg on hit
16582
16583 ins_pipe(pipe_class_memory);
16584 %}
16585
16586 // Two versions of partialSubtypeCheck, both used when we need to
16587 // search for a super class in the secondary supers array. The first
16588 // is used when we don't know _a priori_ the class being searched
16589 // for. The second, far more common, is used when we do know: this is
16590 // used for instanceof, checkcast, and any case where C2 can determine
16591 // it by constant propagation.
16592
16593 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16594 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16595 rFlagsReg cr)
16596 %{
16597 match(Set result (PartialSubtypeCheck sub super));
16598 predicate(UseSecondarySupersTable);
16599 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16600
16601 ins_cost(10 * INSN_COST); // slightly larger than the next version
16602 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16603
16604 ins_encode %{
16605 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16606 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16607 $vtemp$$FloatRegister,
16608 $result$$Register, /*L_success*/nullptr);
16609 %}
16610
16611 ins_pipe(pipe_class_memory);
16612 %}
16613
16614 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16615 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16616 rFlagsReg cr)
16617 %{
16618 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16619 predicate(UseSecondarySupersTable);
16620 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16621
16622 ins_cost(5 * INSN_COST); // smaller than the next version
16623 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16624
16625 ins_encode %{
16626 bool success = false;
16627 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16628 if (InlineSecondarySupersTest) {
16629 success =
16630 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16631 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16632 $vtemp$$FloatRegister,
16633 $result$$Register,
16634 super_klass_slot);
16635 } else {
16636 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16637 success = (call != nullptr);
16638 }
16639 if (!success) {
16640 ciEnv::current()->record_failure("CodeCache is full");
16641 return;
16642 }
16643 %}
16644
16645 ins_pipe(pipe_class_memory);
16646 %}
16647
16648 // Intrisics for String.compareTo()
16649
16650 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16651 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16652 %{
16653 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16654 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16655 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16656
16657 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16658 ins_encode %{
16659 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16660 __ string_compare($str1$$Register, $str2$$Register,
16661 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16662 $tmp1$$Register, $tmp2$$Register,
16663 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16664 %}
16665 ins_pipe(pipe_class_memory);
16666 %}
16667
16668 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16669 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16670 %{
16671 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16672 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16673 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16674
16675 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16676 ins_encode %{
16677 __ string_compare($str1$$Register, $str2$$Register,
16678 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16679 $tmp1$$Register, $tmp2$$Register,
16680 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16681 %}
16682 ins_pipe(pipe_class_memory);
16683 %}
16684
16685 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16686 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16687 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16688 %{
16689 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16690 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16691 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16692 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16693
16694 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16695 ins_encode %{
16696 __ string_compare($str1$$Register, $str2$$Register,
16697 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16698 $tmp1$$Register, $tmp2$$Register,
16699 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16700 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16701 %}
16702 ins_pipe(pipe_class_memory);
16703 %}
16704
16705 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16706 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16707 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16708 %{
16709 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16710 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16711 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16712 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16713
16714 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16715 ins_encode %{
16716 __ string_compare($str1$$Register, $str2$$Register,
16717 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16718 $tmp1$$Register, $tmp2$$Register,
16719 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16720 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16721 %}
16722 ins_pipe(pipe_class_memory);
16723 %}
16724
16725 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16726 // these string_compare variants as NEON register type for convenience so that the prototype of
16727 // string_compare can be shared with all variants.
16728
16729 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16730 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16731 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16732 pRegGov_P1 pgtmp2, rFlagsReg cr)
16733 %{
16734 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16735 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16736 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16737 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16738
16739 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16740 ins_encode %{
16741 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16742 __ string_compare($str1$$Register, $str2$$Register,
16743 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16744 $tmp1$$Register, $tmp2$$Register,
16745 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16746 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16747 StrIntrinsicNode::LL);
16748 %}
16749 ins_pipe(pipe_class_memory);
16750 %}
16751
16752 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16753 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16754 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16755 pRegGov_P1 pgtmp2, rFlagsReg cr)
16756 %{
16757 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16758 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16759 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16760 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16761
16762 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16763 ins_encode %{
16764 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16765 __ string_compare($str1$$Register, $str2$$Register,
16766 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16767 $tmp1$$Register, $tmp2$$Register,
16768 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16769 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16770 StrIntrinsicNode::LU);
16771 %}
16772 ins_pipe(pipe_class_memory);
16773 %}
16774
16775 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16776 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16777 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16778 pRegGov_P1 pgtmp2, rFlagsReg cr)
16779 %{
16780 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16781 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16782 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16783 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16784
16785 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16786 ins_encode %{
16787 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16788 __ string_compare($str1$$Register, $str2$$Register,
16789 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16790 $tmp1$$Register, $tmp2$$Register,
16791 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16792 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16793 StrIntrinsicNode::UL);
16794 %}
16795 ins_pipe(pipe_class_memory);
16796 %}
16797
16798 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16799 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16800 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16801 pRegGov_P1 pgtmp2, rFlagsReg cr)
16802 %{
16803 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16804 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16805 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16806 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16807
16808 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16809 ins_encode %{
16810 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16811 __ string_compare($str1$$Register, $str2$$Register,
16812 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16813 $tmp1$$Register, $tmp2$$Register,
16814 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16815 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16816 StrIntrinsicNode::UU);
16817 %}
16818 ins_pipe(pipe_class_memory);
16819 %}
16820
16821 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16822 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16823 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16824 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16825 %{
16826 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16827 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16828 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16829 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16830 TEMP vtmp0, TEMP vtmp1, KILL cr);
16831 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16832 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16833
16834 ins_encode %{
16835 __ string_indexof($str1$$Register, $str2$$Register,
16836 $cnt1$$Register, $cnt2$$Register,
16837 $tmp1$$Register, $tmp2$$Register,
16838 $tmp3$$Register, $tmp4$$Register,
16839 $tmp5$$Register, $tmp6$$Register,
16840 -1, $result$$Register, StrIntrinsicNode::UU);
16841 %}
16842 ins_pipe(pipe_class_memory);
16843 %}
16844
16845 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16846 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16847 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16848 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16849 %{
16850 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16851 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16852 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16853 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16854 TEMP vtmp0, TEMP vtmp1, KILL cr);
16855 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16856 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16857
16858 ins_encode %{
16859 __ string_indexof($str1$$Register, $str2$$Register,
16860 $cnt1$$Register, $cnt2$$Register,
16861 $tmp1$$Register, $tmp2$$Register,
16862 $tmp3$$Register, $tmp4$$Register,
16863 $tmp5$$Register, $tmp6$$Register,
16864 -1, $result$$Register, StrIntrinsicNode::LL);
16865 %}
16866 ins_pipe(pipe_class_memory);
16867 %}
16868
16869 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16870 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16871 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16872 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16873 %{
16874 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16875 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16876 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16877 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16878 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16879 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16880 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16881
16882 ins_encode %{
16883 __ string_indexof($str1$$Register, $str2$$Register,
16884 $cnt1$$Register, $cnt2$$Register,
16885 $tmp1$$Register, $tmp2$$Register,
16886 $tmp3$$Register, $tmp4$$Register,
16887 $tmp5$$Register, $tmp6$$Register,
16888 -1, $result$$Register, StrIntrinsicNode::UL);
16889 %}
16890 ins_pipe(pipe_class_memory);
16891 %}
16892
16893 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16894 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16895 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16896 %{
16897 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16898 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16899 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16900 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16901 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16902 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16903
16904 ins_encode %{
16905 int icnt2 = (int)$int_cnt2$$constant;
16906 __ string_indexof($str1$$Register, $str2$$Register,
16907 $cnt1$$Register, zr,
16908 $tmp1$$Register, $tmp2$$Register,
16909 $tmp3$$Register, $tmp4$$Register, zr, zr,
16910 icnt2, $result$$Register, StrIntrinsicNode::UU);
16911 %}
16912 ins_pipe(pipe_class_memory);
16913 %}
16914
16915 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16916 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16917 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16918 %{
16919 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16920 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16921 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16922 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16923 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16924 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16925
16926 ins_encode %{
16927 int icnt2 = (int)$int_cnt2$$constant;
16928 __ string_indexof($str1$$Register, $str2$$Register,
16929 $cnt1$$Register, zr,
16930 $tmp1$$Register, $tmp2$$Register,
16931 $tmp3$$Register, $tmp4$$Register, zr, zr,
16932 icnt2, $result$$Register, StrIntrinsicNode::LL);
16933 %}
16934 ins_pipe(pipe_class_memory);
16935 %}
16936
16937 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16938 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16939 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16940 %{
16941 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16942 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16943 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16944 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16945 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16946 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16947
16948 ins_encode %{
16949 int icnt2 = (int)$int_cnt2$$constant;
16950 __ string_indexof($str1$$Register, $str2$$Register,
16951 $cnt1$$Register, zr,
16952 $tmp1$$Register, $tmp2$$Register,
16953 $tmp3$$Register, $tmp4$$Register, zr, zr,
16954 icnt2, $result$$Register, StrIntrinsicNode::UL);
16955 %}
16956 ins_pipe(pipe_class_memory);
16957 %}
16958
16959 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16960 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16961 iRegINoSp tmp3, rFlagsReg cr)
16962 %{
16963 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16964 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16965 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16966 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16967
16968 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16969
16970 ins_encode %{
16971 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16972 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16973 $tmp3$$Register);
16974 %}
16975 ins_pipe(pipe_class_memory);
16976 %}
16977
16978 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16979 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16980 iRegINoSp tmp3, rFlagsReg cr)
16981 %{
16982 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16983 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16984 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16985 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16986
16987 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16988
16989 ins_encode %{
16990 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16991 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16992 $tmp3$$Register);
16993 %}
16994 ins_pipe(pipe_class_memory);
16995 %}
16996
16997 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16998 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16999 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17000 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17001 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17002 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17003 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17004 ins_encode %{
17005 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17006 $result$$Register, $ztmp1$$FloatRegister,
17007 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17008 $ptmp$$PRegister, true /* isL */);
17009 %}
17010 ins_pipe(pipe_class_memory);
17011 %}
17012
17013 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17014 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17015 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17016 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17017 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17018 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17019 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17020 ins_encode %{
17021 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17022 $result$$Register, $ztmp1$$FloatRegister,
17023 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17024 $ptmp$$PRegister, false /* isL */);
17025 %}
17026 ins_pipe(pipe_class_memory);
17027 %}
17028
17029 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17030 iRegI_R0 result, rFlagsReg cr)
17031 %{
17032 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17033 match(Set result (StrEquals (Binary str1 str2) cnt));
17034 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17035
17036 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17037 ins_encode %{
17038 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17039 __ string_equals($str1$$Register, $str2$$Register,
17040 $result$$Register, $cnt$$Register);
17041 %}
17042 ins_pipe(pipe_class_memory);
17043 %}
17044
17045 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17046 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17047 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17048 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17049 iRegP_R10 tmp, rFlagsReg cr)
17050 %{
17051 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17052 match(Set result (AryEq ary1 ary2));
17053 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17054 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17055 TEMP vtmp6, TEMP vtmp7, KILL cr);
17056
17057 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17058 ins_encode %{
17059 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17060 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17061 $result$$Register, $tmp$$Register, 1);
17062 if (tpc == nullptr) {
17063 ciEnv::current()->record_failure("CodeCache is full");
17064 return;
17065 }
17066 %}
17067 ins_pipe(pipe_class_memory);
17068 %}
17069
17070 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17071 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17072 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17073 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17074 iRegP_R10 tmp, rFlagsReg cr)
17075 %{
17076 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17077 match(Set result (AryEq ary1 ary2));
17078 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17079 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17080 TEMP vtmp6, TEMP vtmp7, KILL cr);
17081
17082 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17083 ins_encode %{
17084 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17085 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17086 $result$$Register, $tmp$$Register, 2);
17087 if (tpc == nullptr) {
17088 ciEnv::current()->record_failure("CodeCache is full");
17089 return;
17090 }
17091 %}
17092 ins_pipe(pipe_class_memory);
17093 %}
17094
17095 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17096 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17097 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17098 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17099 %{
17100 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17101 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17102 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17103
17104 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
17105 ins_encode %{
17106 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17107 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17108 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17109 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17110 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17111 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17112 (BasicType)$basic_type$$constant);
17113 if (tpc == nullptr) {
17114 ciEnv::current()->record_failure("CodeCache is full");
17115 return;
17116 }
17117 %}
17118 ins_pipe(pipe_class_memory);
17119 %}
17120
17121 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17122 %{
17123 match(Set result (CountPositives ary1 len));
17124 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17125 format %{ "count positives byte[] $ary1,$len -> $result" %}
17126 ins_encode %{
17127 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17128 if (tpc == nullptr) {
17129 ciEnv::current()->record_failure("CodeCache is full");
17130 return;
17131 }
17132 %}
17133 ins_pipe( pipe_slow );
17134 %}
17135
17136 // fast char[] to byte[] compression
17137 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17138 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17139 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17140 iRegI_R0 result, rFlagsReg cr)
17141 %{
17142 match(Set result (StrCompressedCopy src (Binary dst len)));
17143 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17144 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17145
17146 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17147 ins_encode %{
17148 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17149 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17150 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17151 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17152 %}
17153 ins_pipe(pipe_slow);
17154 %}
17155
17156 // fast byte[] to char[] inflation
17157 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17158 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17159 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17160 %{
17161 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17162 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17163 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17164 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17165
17166 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17167 ins_encode %{
17168 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17169 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17170 $vtmp2$$FloatRegister, $tmp$$Register);
17171 if (tpc == nullptr) {
17172 ciEnv::current()->record_failure("CodeCache is full");
17173 return;
17174 }
17175 %}
17176 ins_pipe(pipe_class_memory);
17177 %}
17178
17179 // encode char[] to byte[] in ISO_8859_1
17180 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17181 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17182 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17183 iRegI_R0 result, rFlagsReg cr)
17184 %{
17185 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17186 match(Set result (EncodeISOArray src (Binary dst len)));
17187 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17188 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17189
17190 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17191 ins_encode %{
17192 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17193 $result$$Register, false,
17194 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17195 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17196 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17197 %}
17198 ins_pipe(pipe_class_memory);
17199 %}
17200
17201 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17202 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17203 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17204 iRegI_R0 result, rFlagsReg cr)
17205 %{
17206 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17207 match(Set result (EncodeISOArray src (Binary dst len)));
17208 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17209 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17210
17211 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17212 ins_encode %{
17213 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17214 $result$$Register, true,
17215 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17216 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17217 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17218 %}
17219 ins_pipe(pipe_class_memory);
17220 %}
17221
17222 //----------------------------- CompressBits/ExpandBits ------------------------
17223
17224 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17225 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17226 match(Set dst (CompressBits src mask));
17227 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17228 format %{ "mov $tsrc, $src\n\t"
17229 "mov $tmask, $mask\n\t"
17230 "bext $tdst, $tsrc, $tmask\n\t"
17231 "mov $dst, $tdst"
17232 %}
17233 ins_encode %{
17234 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17235 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17236 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17237 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17238 %}
17239 ins_pipe(pipe_slow);
17240 %}
17241
17242 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17243 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17244 match(Set dst (CompressBits (LoadI mem) mask));
17245 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17246 format %{ "ldrs $tsrc, $mem\n\t"
17247 "ldrs $tmask, $mask\n\t"
17248 "bext $tdst, $tsrc, $tmask\n\t"
17249 "mov $dst, $tdst"
17250 %}
17251 ins_encode %{
17252 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17253 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17254 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17255 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17256 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17257 %}
17258 ins_pipe(pipe_slow);
17259 %}
17260
17261 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17262 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17263 match(Set dst (CompressBits src mask));
17264 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17265 format %{ "mov $tsrc, $src\n\t"
17266 "mov $tmask, $mask\n\t"
17267 "bext $tdst, $tsrc, $tmask\n\t"
17268 "mov $dst, $tdst"
17269 %}
17270 ins_encode %{
17271 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17272 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17273 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17274 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17275 %}
17276 ins_pipe(pipe_slow);
17277 %}
17278
17279 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17280 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17281 match(Set dst (CompressBits (LoadL mem) mask));
17282 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17283 format %{ "ldrd $tsrc, $mem\n\t"
17284 "ldrd $tmask, $mask\n\t"
17285 "bext $tdst, $tsrc, $tmask\n\t"
17286 "mov $dst, $tdst"
17287 %}
17288 ins_encode %{
17289 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17290 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17291 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17292 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17293 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17294 %}
17295 ins_pipe(pipe_slow);
17296 %}
17297
17298 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17299 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17300 match(Set dst (ExpandBits src mask));
17301 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17302 format %{ "mov $tsrc, $src\n\t"
17303 "mov $tmask, $mask\n\t"
17304 "bdep $tdst, $tsrc, $tmask\n\t"
17305 "mov $dst, $tdst"
17306 %}
17307 ins_encode %{
17308 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17309 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17310 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17311 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17312 %}
17313 ins_pipe(pipe_slow);
17314 %}
17315
17316 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17317 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17318 match(Set dst (ExpandBits (LoadI mem) mask));
17319 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17320 format %{ "ldrs $tsrc, $mem\n\t"
17321 "ldrs $tmask, $mask\n\t"
17322 "bdep $tdst, $tsrc, $tmask\n\t"
17323 "mov $dst, $tdst"
17324 %}
17325 ins_encode %{
17326 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17327 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17328 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17329 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17330 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17331 %}
17332 ins_pipe(pipe_slow);
17333 %}
17334
17335 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17336 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17337 match(Set dst (ExpandBits src mask));
17338 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17339 format %{ "mov $tsrc, $src\n\t"
17340 "mov $tmask, $mask\n\t"
17341 "bdep $tdst, $tsrc, $tmask\n\t"
17342 "mov $dst, $tdst"
17343 %}
17344 ins_encode %{
17345 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17346 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17347 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17348 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17349 %}
17350 ins_pipe(pipe_slow);
17351 %}
17352
17353
17354 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17355 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17356 match(Set dst (ExpandBits (LoadL mem) mask));
17357 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17358 format %{ "ldrd $tsrc, $mem\n\t"
17359 "ldrd $tmask, $mask\n\t"
17360 "bdep $tdst, $tsrc, $tmask\n\t"
17361 "mov $dst, $tdst"
17362 %}
17363 ins_encode %{
17364 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17365 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17366 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17367 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17368 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17369 %}
17370 ins_pipe(pipe_slow);
17371 %}
17372
17373 //----------------------------- Reinterpret ----------------------------------
17374 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17375 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17376 match(Set dst (ReinterpretHF2S src));
17377 format %{ "reinterpretHF2S $dst, $src" %}
17378 ins_encode %{
17379 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17380 %}
17381 ins_pipe(pipe_slow);
17382 %}
17383
17384 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17385 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17386 match(Set dst (ReinterpretS2HF src));
17387 format %{ "reinterpretS2HF $dst, $src" %}
17388 ins_encode %{
17389 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17390 %}
17391 ins_pipe(pipe_slow);
17392 %}
17393
17394 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17395 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17396 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
17397 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
17398 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
17399 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17400 // can be omitted in this pattern, resulting in -
17401 // fcvt $dst, $src // Convert float to half-precision float
17402 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17403 %{
17404 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17405 format %{ "convF2HFAndS2HF $dst, $src" %}
17406 ins_encode %{
17407 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17408 %}
17409 ins_pipe(pipe_slow);
17410 %}
17411
17412 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17413 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17414 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
17415 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17416 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
17417 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17418 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17419 // resulting in -
17420 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
17421 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17422 %{
17423 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17424 format %{ "convHF2SAndHF2F $dst, $src" %}
17425 ins_encode %{
17426 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17427 %}
17428 ins_pipe(pipe_slow);
17429 %}
17430
17431 // ============================================================================
17432 // This name is KNOWN by the ADLC and cannot be changed.
17433 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17434 // for this guy.
17435 instruct tlsLoadP(thread_RegP dst)
17436 %{
17437 match(Set dst (ThreadLocal));
17438
17439 ins_cost(0);
17440
17441 format %{ " -- \t// $dst=Thread::current(), empty" %}
17442
17443 size(0);
17444
17445 ins_encode( /*empty*/ );
17446
17447 ins_pipe(pipe_class_empty);
17448 %}
17449
17450 //----------PEEPHOLE RULES-----------------------------------------------------
17451 // These must follow all instruction definitions as they use the names
17452 // defined in the instructions definitions.
17453 //
17454 // peepmatch ( root_instr_name [preceding_instruction]* );
17455 //
17456 // peepconstraint %{
17457 // (instruction_number.operand_name relational_op instruction_number.operand_name
17458 // [, ...] );
17459 // // instruction numbers are zero-based using left to right order in peepmatch
17460 //
17461 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17462 // // provide an instruction_number.operand_name for each operand that appears
17463 // // in the replacement instruction's match rule
17464 //
17465 // ---------VM FLAGS---------------------------------------------------------
17466 //
17467 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17468 //
17469 // Each peephole rule is given an identifying number starting with zero and
17470 // increasing by one in the order seen by the parser. An individual peephole
17471 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17472 // on the command-line.
17473 //
17474 // ---------CURRENT LIMITATIONS----------------------------------------------
17475 //
17476 // Only match adjacent instructions in same basic block
17477 // Only equality constraints
17478 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17479 // Only one replacement instruction
17480 //
17481 // ---------EXAMPLE----------------------------------------------------------
17482 //
17483 // // pertinent parts of existing instructions in architecture description
17484 // instruct movI(iRegINoSp dst, iRegI src)
17485 // %{
17486 // match(Set dst (CopyI src));
17487 // %}
17488 //
17489 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17490 // %{
17491 // match(Set dst (AddI dst src));
17492 // effect(KILL cr);
17493 // %}
17494 //
17495 // // Change (inc mov) to lea
17496 // peephole %{
17497 // // increment preceded by register-register move
17498 // peepmatch ( incI_iReg movI );
17499 // // require that the destination register of the increment
17500 // // match the destination register of the move
17501 // peepconstraint ( 0.dst == 1.dst );
17502 // // construct a replacement instruction that sets
17503 // // the destination to ( move's source register + one )
17504 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17505 // %}
17506 //
17507
17508 // Implementation no longer uses movX instructions since
17509 // machine-independent system no longer uses CopyX nodes.
17510 //
17511 // peephole
17512 // %{
17513 // peepmatch (incI_iReg movI);
17514 // peepconstraint (0.dst == 1.dst);
17515 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17516 // %}
17517
17518 // peephole
17519 // %{
17520 // peepmatch (decI_iReg movI);
17521 // peepconstraint (0.dst == 1.dst);
17522 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17523 // %}
17524
17525 // peephole
17526 // %{
17527 // peepmatch (addI_iReg_imm movI);
17528 // peepconstraint (0.dst == 1.dst);
17529 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17530 // %}
17531
17532 // peephole
17533 // %{
17534 // peepmatch (incL_iReg movL);
17535 // peepconstraint (0.dst == 1.dst);
17536 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17537 // %}
17538
17539 // peephole
17540 // %{
17541 // peepmatch (decL_iReg movL);
17542 // peepconstraint (0.dst == 1.dst);
17543 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17544 // %}
17545
17546 // peephole
17547 // %{
17548 // peepmatch (addL_iReg_imm movL);
17549 // peepconstraint (0.dst == 1.dst);
17550 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17551 // %}
17552
17553 // peephole
17554 // %{
17555 // peepmatch (addP_iReg_imm movP);
17556 // peepconstraint (0.dst == 1.dst);
17557 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17558 // %}
17559
17560 // // Change load of spilled value to only a spill
17561 // instruct storeI(memory mem, iRegI src)
17562 // %{
17563 // match(Set mem (StoreI mem src));
17564 // %}
17565 //
17566 // instruct loadI(iRegINoSp dst, memory mem)
17567 // %{
17568 // match(Set dst (LoadI mem));
17569 // %}
17570 //
17571
17572 //----------SMARTSPILL RULES---------------------------------------------------
17573 // These must follow all instruction definitions as they use the names
17574 // defined in the instructions definitions.
17575
17576 // Local Variables:
17577 // mode: c++
17578 // End: