1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return MacroAssembler::max_trampoline_stub_size(); // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1652 // lea(rscratch1, RuntimeAddress(addr)
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else {
1658 return 6 * NativeInstruction::instruction_size;
1659 }
1660 }
1661
1662 //=============================================================================
1663
1664 #ifndef PRODUCT
1665 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1666 st->print("BREAKPOINT");
1667 }
1668 #endif
1669
1670 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1671 __ brk(0);
1672 }
1673
1674 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1675 return MachNode::size(ra_);
1676 }
1677
1678 //=============================================================================
1679
1680 #ifndef PRODUCT
1681 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1682 st->print("nop \t# %d bytes pad for loops and calls", _count);
1683 }
1684 #endif
1685
1686 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1687 for (int i = 0; i < _count; i++) {
1688 __ nop();
1689 }
1690 }
1691
1692 uint MachNopNode::size(PhaseRegAlloc*) const {
1693 return _count * NativeInstruction::instruction_size;
1694 }
1695
1696 //=============================================================================
1697 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1698
1699 int ConstantTable::calculate_table_base_offset() const {
1700 return 0; // absolute addressing, no offset
1701 }
1702
1703 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1704 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1705 ShouldNotReachHere();
1706 }
1707
1708 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1709 // Empty encoding
1710 }
1711
1712 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1713 return 0;
1714 }
1715
1716 #ifndef PRODUCT
1717 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1718 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1719 }
1720 #endif
1721
1722 #ifndef PRODUCT
1723 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1724 Compile* C = ra_->C;
1725
1726 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1727
1728 if (C->output()->need_stack_bang(framesize))
1729 st->print("# stack bang size=%d\n\t", framesize);
1730
1731 if (VM_Version::use_rop_protection()) {
1732 st->print("ldr zr, [lr]\n\t");
1733 st->print("paciaz\n\t");
1734 }
1735 if (framesize < ((1 << 9) + 2 * wordSize)) {
1736 st->print("sub sp, sp, #%d\n\t", framesize);
1737 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1738 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1739 } else {
1740 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1741 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1742 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1743 st->print("sub sp, sp, rscratch1");
1744 }
1745 if (C->stub_function() == nullptr) {
1746 st->print("\n\t");
1747 st->print("ldr rscratch1, [guard]\n\t");
1748 st->print("dmb ishld\n\t");
1749 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1750 st->print("cmp rscratch1, rscratch2\n\t");
1751 st->print("b.eq skip");
1752 st->print("\n\t");
1753 st->print("blr #nmethod_entry_barrier_stub\n\t");
1754 st->print("b skip\n\t");
1755 st->print("guard: int\n\t");
1756 st->print("\n\t");
1757 st->print("skip:\n\t");
1758 }
1759 }
1760 #endif
1761
1762 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1763 Compile* C = ra_->C;
1764
1765 // n.b. frame size includes space for return pc and rfp
1766 const int framesize = C->output()->frame_size_in_bytes();
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 if (C->clinit_barrier_on_entry()) {
1773 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1774
1775 Label L_skip_barrier;
1776
1777 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1778 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1779 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1780 __ bind(L_skip_barrier);
1781 }
1782
1783 if (C->max_vector_size() > 0) {
1784 __ reinitialize_ptrue();
1785 }
1786
1787 int bangsize = C->output()->bang_size_in_bytes();
1788 if (C->output()->need_stack_bang(bangsize))
1789 __ generate_stack_overflow_check(bangsize);
1790
1791 __ build_frame(framesize);
1792
1793 if (C->stub_function() == nullptr) {
1794 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1795 // Dummy labels for just measuring the code size
1796 Label dummy_slow_path;
1797 Label dummy_continuation;
1798 Label dummy_guard;
1799 Label* slow_path = &dummy_slow_path;
1800 Label* continuation = &dummy_continuation;
1801 Label* guard = &dummy_guard;
1802 if (!Compile::current()->output()->in_scratch_emit_size()) {
1803 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1804 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1805 Compile::current()->output()->add_stub(stub);
1806 slow_path = &stub->entry();
1807 continuation = &stub->continuation();
1808 guard = &stub->guard();
1809 }
1810 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1811 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1812 }
1813
1814 if (VerifyStackAtCalls) {
1815 Unimplemented();
1816 }
1817
1818 C->output()->set_frame_complete(__ offset());
1819
1820 if (C->has_mach_constant_base_node()) {
1821 // NOTE: We set the table base offset here because users might be
1822 // emitted before MachConstantBaseNode.
1823 ConstantTable& constant_table = C->output()->constant_table();
1824 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1825 }
1826 }
1827
1828 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1829 {
1830 return MachNode::size(ra_); // too many variables; just compute it
1831 // the hard way
1832 }
1833
1834 int MachPrologNode::reloc() const
1835 {
1836 return 0;
1837 }
1838
1839 //=============================================================================
1840
1841 #ifndef PRODUCT
1842 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1843 Compile* C = ra_->C;
1844 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1845
1846 st->print("# pop frame %d\n\t",framesize);
1847
1848 if (framesize == 0) {
1849 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1850 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1851 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1852 st->print("add sp, sp, #%d\n\t", framesize);
1853 } else {
1854 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1855 st->print("add sp, sp, rscratch1\n\t");
1856 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1857 }
1858 if (VM_Version::use_rop_protection()) {
1859 st->print("autiaz\n\t");
1860 st->print("ldr zr, [lr]\n\t");
1861 }
1862
1863 if (do_polling() && C->is_method_compilation()) {
1864 st->print("# test polling word\n\t");
1865 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1866 st->print("cmp sp, rscratch1\n\t");
1867 st->print("bhi #slow_path");
1868 }
1869 }
1870 #endif
1871
1872 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1873 Compile* C = ra_->C;
1874 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1875
1876 __ remove_frame(framesize);
1877
1878 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1879 __ reserved_stack_check();
1880 }
1881
1882 if (do_polling() && C->is_method_compilation()) {
1883 Label dummy_label;
1884 Label* code_stub = &dummy_label;
1885 if (!C->output()->in_scratch_emit_size()) {
1886 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1887 C->output()->add_stub(stub);
1888 code_stub = &stub->entry();
1889 }
1890 __ relocate(relocInfo::poll_return_type);
1891 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1892 }
1893 }
1894
1895 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1896 // Variable size. Determine dynamically.
1897 return MachNode::size(ra_);
1898 }
1899
1900 int MachEpilogNode::reloc() const {
1901 // Return number of relocatable values contained in this instruction.
1902 return 1; // 1 for polling page.
1903 }
1904
1905 const Pipeline * MachEpilogNode::pipeline() const {
1906 return MachNode::pipeline_class();
1907 }
1908
1909 //=============================================================================
1910
1911 static enum RC rc_class(OptoReg::Name reg) {
1912
1913 if (reg == OptoReg::Bad) {
1914 return rc_bad;
1915 }
1916
1917 // we have 32 int registers * 2 halves
1918 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1919
1920 if (reg < slots_of_int_registers) {
1921 return rc_int;
1922 }
1923
1924 // we have 32 float register * 8 halves
1925 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1926 if (reg < slots_of_int_registers + slots_of_float_registers) {
1927 return rc_float;
1928 }
1929
1930 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1931 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1932 return rc_predicate;
1933 }
1934
1935 // Between predicate regs & stack is the flags.
1936 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1937
1938 return rc_stack;
1939 }
1940
1941 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1942 Compile* C = ra_->C;
1943
1944 // Get registers to move.
1945 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1946 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1947 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1948 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1949
1950 enum RC src_hi_rc = rc_class(src_hi);
1951 enum RC src_lo_rc = rc_class(src_lo);
1952 enum RC dst_hi_rc = rc_class(dst_hi);
1953 enum RC dst_lo_rc = rc_class(dst_lo);
1954
1955 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1956
1957 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1958 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1959 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1960 "expected aligned-adjacent pairs");
1961 }
1962
1963 if (src_lo == dst_lo && src_hi == dst_hi) {
1964 return 0; // Self copy, no move.
1965 }
1966
1967 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1968 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1969 int src_offset = ra_->reg2offset(src_lo);
1970 int dst_offset = ra_->reg2offset(dst_lo);
1971
1972 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1973 uint ireg = ideal_reg();
1974 if (ireg == Op_VecA && masm) {
1975 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1976 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1977 // stack->stack
1978 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1979 sve_vector_reg_size_in_bytes);
1980 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1981 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1982 sve_vector_reg_size_in_bytes);
1983 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1984 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1985 sve_vector_reg_size_in_bytes);
1986 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1987 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1988 as_FloatRegister(Matcher::_regEncode[src_lo]),
1989 as_FloatRegister(Matcher::_regEncode[src_lo]));
1990 } else {
1991 ShouldNotReachHere();
1992 }
1993 } else if (masm) {
1994 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1995 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1996 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1997 // stack->stack
1998 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1999 if (ireg == Op_VecD) {
2000 __ unspill(rscratch1, true, src_offset);
2001 __ spill(rscratch1, true, dst_offset);
2002 } else {
2003 __ spill_copy128(src_offset, dst_offset);
2004 }
2005 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2006 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2007 ireg == Op_VecD ? __ T8B : __ T16B,
2008 as_FloatRegister(Matcher::_regEncode[src_lo]));
2009 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2010 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2011 ireg == Op_VecD ? __ D : __ Q,
2012 ra_->reg2offset(dst_lo));
2013 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2014 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2015 ireg == Op_VecD ? __ D : __ Q,
2016 ra_->reg2offset(src_lo));
2017 } else {
2018 ShouldNotReachHere();
2019 }
2020 }
2021 } else if (masm) {
2022 switch (src_lo_rc) {
2023 case rc_int:
2024 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2025 if (is64) {
2026 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2027 as_Register(Matcher::_regEncode[src_lo]));
2028 } else {
2029 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2030 as_Register(Matcher::_regEncode[src_lo]));
2031 }
2032 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2033 if (is64) {
2034 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2035 as_Register(Matcher::_regEncode[src_lo]));
2036 } else {
2037 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2038 as_Register(Matcher::_regEncode[src_lo]));
2039 }
2040 } else { // gpr --> stack spill
2041 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2042 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2043 }
2044 break;
2045 case rc_float:
2046 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2047 if (is64) {
2048 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2049 as_FloatRegister(Matcher::_regEncode[src_lo]));
2050 } else {
2051 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2052 as_FloatRegister(Matcher::_regEncode[src_lo]));
2053 }
2054 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2055 if (is64) {
2056 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2057 as_FloatRegister(Matcher::_regEncode[src_lo]));
2058 } else {
2059 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2060 as_FloatRegister(Matcher::_regEncode[src_lo]));
2061 }
2062 } else { // fpr --> stack spill
2063 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2064 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2065 is64 ? __ D : __ S, dst_offset);
2066 }
2067 break;
2068 case rc_stack:
2069 if (dst_lo_rc == rc_int) { // stack --> gpr load
2070 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2071 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2072 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2073 is64 ? __ D : __ S, src_offset);
2074 } else if (dst_lo_rc == rc_predicate) {
2075 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2076 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2077 } else { // stack --> stack copy
2078 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2079 if (ideal_reg() == Op_RegVectMask) {
2080 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2081 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2082 } else {
2083 __ unspill(rscratch1, is64, src_offset);
2084 __ spill(rscratch1, is64, dst_offset);
2085 }
2086 }
2087 break;
2088 case rc_predicate:
2089 if (dst_lo_rc == rc_predicate) {
2090 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2091 } else if (dst_lo_rc == rc_stack) {
2092 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2093 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2094 } else {
2095 assert(false, "bad src and dst rc_class combination.");
2096 ShouldNotReachHere();
2097 }
2098 break;
2099 default:
2100 assert(false, "bad rc_class for spill");
2101 ShouldNotReachHere();
2102 }
2103 }
2104
2105 if (st) {
2106 st->print("spill ");
2107 if (src_lo_rc == rc_stack) {
2108 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2109 } else {
2110 st->print("%s -> ", Matcher::regName[src_lo]);
2111 }
2112 if (dst_lo_rc == rc_stack) {
2113 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2114 } else {
2115 st->print("%s", Matcher::regName[dst_lo]);
2116 }
2117 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2118 int vsize = 0;
2119 switch (ideal_reg()) {
2120 case Op_VecD:
2121 vsize = 64;
2122 break;
2123 case Op_VecX:
2124 vsize = 128;
2125 break;
2126 case Op_VecA:
2127 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2128 break;
2129 default:
2130 assert(false, "bad register type for spill");
2131 ShouldNotReachHere();
2132 }
2133 st->print("\t# vector spill size = %d", vsize);
2134 } else if (ideal_reg() == Op_RegVectMask) {
2135 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2136 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2137 st->print("\t# predicate spill size = %d", vsize);
2138 } else {
2139 st->print("\t# spill size = %d", is64 ? 64 : 32);
2140 }
2141 }
2142
2143 return 0;
2144
2145 }
2146
2147 #ifndef PRODUCT
2148 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2149 if (!ra_)
2150 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2151 else
2152 implementation(nullptr, ra_, false, st);
2153 }
2154 #endif
2155
2156 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2157 implementation(masm, ra_, false, nullptr);
2158 }
2159
2160 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2161 return MachNode::size(ra_);
2162 }
2163
2164 //=============================================================================
2165
2166 #ifndef PRODUCT
2167 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2168 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2169 int reg = ra_->get_reg_first(this);
2170 st->print("add %s, rsp, #%d]\t# box lock",
2171 Matcher::regName[reg], offset);
2172 }
2173 #endif
2174
2175 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2176 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2177 int reg = ra_->get_encode(this);
2178
2179 // This add will handle any 24-bit signed offset. 24 bits allows an
2180 // 8 megabyte stack frame.
2181 __ add(as_Register(reg), sp, offset);
2182 }
2183
2184 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2185 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2186 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2187
2188 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2189 return NativeInstruction::instruction_size;
2190 } else {
2191 return 2 * NativeInstruction::instruction_size;
2192 }
2193 }
2194
2195 //=============================================================================
2196
2197 #ifndef PRODUCT
2198 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2199 {
2200 st->print_cr("# MachUEPNode");
2201 if (UseCompressedClassPointers) {
2202 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2203 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2204 st->print_cr("\tcmpw rscratch1, r10");
2205 } else {
2206 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2207 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2208 st->print_cr("\tcmp rscratch1, r10");
2209 }
2210 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2211 }
2212 #endif
2213
2214 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2215 {
2216 __ ic_check(InteriorEntryAlignment);
2217 }
2218
2219 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2220 {
2221 return MachNode::size(ra_);
2222 }
2223
2224 // REQUIRED EMIT CODE
2225
2226 //=============================================================================
2227
2228 // Emit exception handler code.
2229 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2230 {
2231 // mov rscratch1 #exception_blob_entry_point
2232 // br rscratch1
2233 // Note that the code buffer's insts_mark is always relative to insts.
2234 // That's why we must use the macroassembler to generate a handler.
2235 address base = __ start_a_stub(size_exception_handler());
2236 if (base == nullptr) {
2237 ciEnv::current()->record_failure("CodeCache is full");
2238 return 0; // CodeBuffer::expand failed
2239 }
2240 int offset = __ offset();
2241 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2242 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2243 __ end_a_stub();
2244 return offset;
2245 }
2246
2247 // Emit deopt handler code.
2248 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2249 {
2250 // Note that the code buffer's insts_mark is always relative to insts.
2251 // That's why we must use the macroassembler to generate a handler.
2252 address base = __ start_a_stub(size_deopt_handler());
2253 if (base == nullptr) {
2254 ciEnv::current()->record_failure("CodeCache is full");
2255 return 0; // CodeBuffer::expand failed
2256 }
2257 int offset = __ offset();
2258
2259 __ adr(lr, __ pc());
2260 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2261
2262 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2263 __ end_a_stub();
2264 return offset;
2265 }
2266
2267 // REQUIRED MATCHER CODE
2268
2269 //=============================================================================
2270
2271 bool Matcher::match_rule_supported(int opcode) {
2272 if (!has_match_rule(opcode))
2273 return false;
2274
2275 switch (opcode) {
2276 case Op_OnSpinWait:
2277 return VM_Version::supports_on_spin_wait();
2278 case Op_CacheWB:
2279 case Op_CacheWBPreSync:
2280 case Op_CacheWBPostSync:
2281 if (!VM_Version::supports_data_cache_line_flush()) {
2282 return false;
2283 }
2284 break;
2285 case Op_ExpandBits:
2286 case Op_CompressBits:
2287 if (!VM_Version::supports_svebitperm()) {
2288 return false;
2289 }
2290 break;
2291 case Op_FmaF:
2292 case Op_FmaD:
2293 case Op_FmaVF:
2294 case Op_FmaVD:
2295 if (!UseFMA) {
2296 return false;
2297 }
2298 break;
2299 case Op_FmaHF:
2300 // UseFMA flag also needs to be checked along with FEAT_FP16
2301 if (!UseFMA || !is_feat_fp16_supported()) {
2302 return false;
2303 }
2304 break;
2305 case Op_AddHF:
2306 case Op_SubHF:
2307 case Op_MulHF:
2308 case Op_DivHF:
2309 case Op_MinHF:
2310 case Op_MaxHF:
2311 case Op_SqrtHF:
2312 // Half-precision floating point scalar operations require FEAT_FP16
2313 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2314 // features are supported.
2315 if (!is_feat_fp16_supported()) {
2316 return false;
2317 }
2318 break;
2319 }
2320
2321 return true; // Per default match rules are supported.
2322 }
2323
2324 const RegMask* Matcher::predicate_reg_mask(void) {
2325 return &_PR_REG_mask;
2326 }
2327
2328 bool Matcher::supports_vector_calling_convention(void) {
2329 return EnableVectorSupport;
2330 }
2331
2332 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2333 assert(EnableVectorSupport, "sanity");
2334 int lo = V0_num;
2335 int hi = V0_H_num;
2336 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2337 hi = V0_K_num;
2338 }
2339 return OptoRegPair(hi, lo);
2340 }
2341
2342 // Is this branch offset short enough that a short branch can be used?
2343 //
2344 // NOTE: If the platform does not provide any short branch variants, then
2345 // this method should return false for offset 0.
2346 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2347 // The passed offset is relative to address of the branch.
2348
2349 return (-32768 <= offset && offset < 32768);
2350 }
2351
2352 // Vector width in bytes.
2353 int Matcher::vector_width_in_bytes(BasicType bt) {
2354 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2355 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2356 // Minimum 2 values in vector
2357 if (size < 2*type2aelembytes(bt)) size = 0;
2358 // But never < 4
2359 if (size < 4) size = 0;
2360 return size;
2361 }
2362
2363 // Limits on vector size (number of elements) loaded into vector.
2364 int Matcher::max_vector_size(const BasicType bt) {
2365 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2366 }
2367
2368 int Matcher::min_vector_size(const BasicType bt) {
2369 int max_size = max_vector_size(bt);
2370 // Limit the min vector size to 8 bytes.
2371 int size = 8 / type2aelembytes(bt);
2372 if (bt == T_BYTE) {
2373 // To support vector api shuffle/rearrange.
2374 size = 4;
2375 } else if (bt == T_BOOLEAN) {
2376 // To support vector api load/store mask.
2377 size = 2;
2378 }
2379 if (size < 2) size = 2;
2380 return MIN2(size, max_size);
2381 }
2382
2383 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2384 return Matcher::max_vector_size(bt);
2385 }
2386
2387 // Actual max scalable vector register length.
2388 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2389 return Matcher::max_vector_size(bt);
2390 }
2391
2392 // Vector ideal reg.
2393 uint Matcher::vector_ideal_reg(int len) {
2394 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2395 return Op_VecA;
2396 }
2397 switch(len) {
2398 // For 16-bit/32-bit mask vector, reuse VecD.
2399 case 2:
2400 case 4:
2401 case 8: return Op_VecD;
2402 case 16: return Op_VecX;
2403 }
2404 ShouldNotReachHere();
2405 return 0;
2406 }
2407
2408 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2409 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2410 switch (ideal_reg) {
2411 case Op_VecA: return new vecAOper();
2412 case Op_VecD: return new vecDOper();
2413 case Op_VecX: return new vecXOper();
2414 }
2415 ShouldNotReachHere();
2416 return nullptr;
2417 }
2418
2419 bool Matcher::is_reg2reg_move(MachNode* m) {
2420 return false;
2421 }
2422
2423 bool Matcher::is_generic_vector(MachOper* opnd) {
2424 return opnd->opcode() == VREG;
2425 }
2426
2427 // Return whether or not this register is ever used as an argument.
2428 // This function is used on startup to build the trampoline stubs in
2429 // generateOptoStub. Registers not mentioned will be killed by the VM
2430 // call in the trampoline, and arguments in those registers not be
2431 // available to the callee.
2432 bool Matcher::can_be_java_arg(int reg)
2433 {
2434 return
2435 reg == R0_num || reg == R0_H_num ||
2436 reg == R1_num || reg == R1_H_num ||
2437 reg == R2_num || reg == R2_H_num ||
2438 reg == R3_num || reg == R3_H_num ||
2439 reg == R4_num || reg == R4_H_num ||
2440 reg == R5_num || reg == R5_H_num ||
2441 reg == R6_num || reg == R6_H_num ||
2442 reg == R7_num || reg == R7_H_num ||
2443 reg == V0_num || reg == V0_H_num ||
2444 reg == V1_num || reg == V1_H_num ||
2445 reg == V2_num || reg == V2_H_num ||
2446 reg == V3_num || reg == V3_H_num ||
2447 reg == V4_num || reg == V4_H_num ||
2448 reg == V5_num || reg == V5_H_num ||
2449 reg == V6_num || reg == V6_H_num ||
2450 reg == V7_num || reg == V7_H_num;
2451 }
2452
2453 bool Matcher::is_spillable_arg(int reg)
2454 {
2455 return can_be_java_arg(reg);
2456 }
2457
2458 uint Matcher::int_pressure_limit()
2459 {
2460 // JDK-8183543: When taking the number of available registers as int
2461 // register pressure threshold, the jtreg test:
2462 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2463 // failed due to C2 compilation failure with
2464 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2465 //
2466 // A derived pointer is live at CallNode and then is flagged by RA
2467 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2468 // derived pointers and lastly fail to spill after reaching maximum
2469 // number of iterations. Lowering the default pressure threshold to
2470 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2471 // a high register pressure area of the code so that split_DEF can
2472 // generate DefinitionSpillCopy for the derived pointer.
2473 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2474 if (!PreserveFramePointer) {
2475 // When PreserveFramePointer is off, frame pointer is allocatable,
2476 // but different from other SOC registers, it is excluded from
2477 // fatproj's mask because its save type is No-Save. Decrease 1 to
2478 // ensure high pressure at fatproj when PreserveFramePointer is off.
2479 // See check_pressure_at_fatproj().
2480 default_int_pressure_threshold--;
2481 }
2482 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2483 }
2484
2485 uint Matcher::float_pressure_limit()
2486 {
2487 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2488 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2489 }
2490
2491 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2492 return false;
2493 }
2494
2495 RegMask Matcher::divI_proj_mask() {
2496 ShouldNotReachHere();
2497 return RegMask();
2498 }
2499
2500 // Register for MODI projection of divmodI.
2501 RegMask Matcher::modI_proj_mask() {
2502 ShouldNotReachHere();
2503 return RegMask();
2504 }
2505
2506 // Register for DIVL projection of divmodL.
2507 RegMask Matcher::divL_proj_mask() {
2508 ShouldNotReachHere();
2509 return RegMask();
2510 }
2511
2512 // Register for MODL projection of divmodL.
2513 RegMask Matcher::modL_proj_mask() {
2514 ShouldNotReachHere();
2515 return RegMask();
2516 }
2517
2518 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2519 return FP_REG_mask();
2520 }
2521
2522 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2523 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2524 Node* u = addp->fast_out(i);
2525 if (u->is_LoadStore()) {
2526 // On AArch64, LoadStoreNodes (i.e. compare and swap
2527 // instructions) only take register indirect as an operand, so
2528 // any attempt to use an AddPNode as an input to a LoadStoreNode
2529 // must fail.
2530 return false;
2531 }
2532 if (u->is_Mem()) {
2533 int opsize = u->as_Mem()->memory_size();
2534 assert(opsize > 0, "unexpected memory operand size");
2535 if (u->as_Mem()->memory_size() != (1<<shift)) {
2536 return false;
2537 }
2538 }
2539 }
2540 return true;
2541 }
2542
2543 // Convert BootTest condition to Assembler condition.
2544 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2545 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2546 Assembler::Condition result;
2547 switch(cond) {
2548 case BoolTest::eq:
2549 result = Assembler::EQ; break;
2550 case BoolTest::ne:
2551 result = Assembler::NE; break;
2552 case BoolTest::le:
2553 result = Assembler::LE; break;
2554 case BoolTest::ge:
2555 result = Assembler::GE; break;
2556 case BoolTest::lt:
2557 result = Assembler::LT; break;
2558 case BoolTest::gt:
2559 result = Assembler::GT; break;
2560 case BoolTest::ule:
2561 result = Assembler::LS; break;
2562 case BoolTest::uge:
2563 result = Assembler::HS; break;
2564 case BoolTest::ult:
2565 result = Assembler::LO; break;
2566 case BoolTest::ugt:
2567 result = Assembler::HI; break;
2568 case BoolTest::overflow:
2569 result = Assembler::VS; break;
2570 case BoolTest::no_overflow:
2571 result = Assembler::VC; break;
2572 default:
2573 ShouldNotReachHere();
2574 return Assembler::Condition(-1);
2575 }
2576
2577 // Check conversion
2578 if (cond & BoolTest::unsigned_compare) {
2579 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2580 } else {
2581 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2582 }
2583
2584 return result;
2585 }
2586
2587 // Binary src (Replicate con)
2588 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2589 if (n == nullptr || m == nullptr) {
2590 return false;
2591 }
2592
2593 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2594 return false;
2595 }
2596
2597 Node* imm_node = m->in(1);
2598 if (!imm_node->is_Con()) {
2599 return false;
2600 }
2601
2602 const Type* t = imm_node->bottom_type();
2603 if (!(t->isa_int() || t->isa_long())) {
2604 return false;
2605 }
2606
2607 switch (n->Opcode()) {
2608 case Op_AndV:
2609 case Op_OrV:
2610 case Op_XorV: {
2611 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2612 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2613 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2614 }
2615 case Op_AddVB:
2616 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2617 case Op_AddVS:
2618 case Op_AddVI:
2619 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2620 case Op_AddVL:
2621 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2622 default:
2623 return false;
2624 }
2625 }
2626
2627 // (XorV src (Replicate m1))
2628 // (XorVMask src (MaskAll m1))
2629 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2630 if (n != nullptr && m != nullptr) {
2631 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2632 VectorNode::is_all_ones_vector(m);
2633 }
2634 return false;
2635 }
2636
2637 // Should the matcher clone input 'm' of node 'n'?
2638 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2639 if (is_vshift_con_pattern(n, m) ||
2640 is_vector_bitwise_not_pattern(n, m) ||
2641 is_valid_sve_arith_imm_pattern(n, m) ||
2642 is_encode_and_store_pattern(n, m)) {
2643 mstack.push(m, Visit);
2644 return true;
2645 }
2646 return false;
2647 }
2648
2649 // Should the Matcher clone shifts on addressing modes, expecting them
2650 // to be subsumed into complex addressing expressions or compute them
2651 // into registers?
2652 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2653
2654 // Loads and stores with indirect memory input (e.g., volatile loads and
2655 // stores) do not subsume the input into complex addressing expressions. If
2656 // the addressing expression is input to at least one such load or store, do
2657 // not clone the addressing expression. Query needs_acquiring_load and
2658 // needs_releasing_store as a proxy for indirect memory input, as it is not
2659 // possible to directly query for indirect memory input at this stage.
2660 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2661 Node* n = m->fast_out(i);
2662 if (n->is_Load() && needs_acquiring_load(n)) {
2663 return false;
2664 }
2665 if (n->is_Store() && needs_releasing_store(n)) {
2666 return false;
2667 }
2668 }
2669
2670 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2671 return true;
2672 }
2673
2674 Node *off = m->in(AddPNode::Offset);
2675 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2676 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2677 // Are there other uses besides address expressions?
2678 !is_visited(off)) {
2679 address_visited.set(off->_idx); // Flag as address_visited
2680 mstack.push(off->in(2), Visit);
2681 Node *conv = off->in(1);
2682 if (conv->Opcode() == Op_ConvI2L &&
2683 // Are there other uses besides address expressions?
2684 !is_visited(conv)) {
2685 address_visited.set(conv->_idx); // Flag as address_visited
2686 mstack.push(conv->in(1), Pre_Visit);
2687 } else {
2688 mstack.push(conv, Pre_Visit);
2689 }
2690 address_visited.test_set(m->_idx); // Flag as address_visited
2691 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2692 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2693 return true;
2694 } else if (off->Opcode() == Op_ConvI2L &&
2695 // Are there other uses besides address expressions?
2696 !is_visited(off)) {
2697 address_visited.test_set(m->_idx); // Flag as address_visited
2698 address_visited.set(off->_idx); // Flag as address_visited
2699 mstack.push(off->in(1), Pre_Visit);
2700 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2701 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2702 return true;
2703 }
2704 return false;
2705 }
2706
2707 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2708 { \
2709 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2710 guarantee(DISP == 0, "mode not permitted for volatile"); \
2711 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2712 __ INSN(REG, as_Register(BASE)); \
2713 }
2714
2715
2716 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2717 {
2718 Address::extend scale;
2719
2720 // Hooboy, this is fugly. We need a way to communicate to the
2721 // encoder that the index needs to be sign extended, so we have to
2722 // enumerate all the cases.
2723 switch (opcode) {
2724 case INDINDEXSCALEDI2L:
2725 case INDINDEXSCALEDI2LN:
2726 case INDINDEXI2L:
2727 case INDINDEXI2LN:
2728 scale = Address::sxtw(size);
2729 break;
2730 default:
2731 scale = Address::lsl(size);
2732 }
2733
2734 if (index == -1) {
2735 return Address(base, disp);
2736 } else {
2737 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2738 return Address(base, as_Register(index), scale);
2739 }
2740 }
2741
2742
2743 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2744 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2745 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2746 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2747 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2748
2749 // Used for all non-volatile memory accesses. The use of
2750 // $mem->opcode() to discover whether this pattern uses sign-extended
2751 // offsets is something of a kludge.
2752 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2753 Register reg, int opcode,
2754 Register base, int index, int scale, int disp,
2755 int size_in_memory)
2756 {
2757 Address addr = mem2address(opcode, base, index, scale, disp);
2758 if (addr.getMode() == Address::base_plus_offset) {
2759 /* Fix up any out-of-range offsets. */
2760 assert_different_registers(rscratch1, base);
2761 assert_different_registers(rscratch1, reg);
2762 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2763 }
2764 (masm->*insn)(reg, addr);
2765 }
2766
2767 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2768 FloatRegister reg, int opcode,
2769 Register base, int index, int size, int disp,
2770 int size_in_memory)
2771 {
2772 Address::extend scale;
2773
2774 switch (opcode) {
2775 case INDINDEXSCALEDI2L:
2776 case INDINDEXSCALEDI2LN:
2777 scale = Address::sxtw(size);
2778 break;
2779 default:
2780 scale = Address::lsl(size);
2781 }
2782
2783 if (index == -1) {
2784 // Fix up any out-of-range offsets.
2785 assert_different_registers(rscratch1, base);
2786 Address addr = Address(base, disp);
2787 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2788 (masm->*insn)(reg, addr);
2789 } else {
2790 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2791 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2792 }
2793 }
2794
2795 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2796 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2797 int opcode, Register base, int index, int size, int disp)
2798 {
2799 if (index == -1) {
2800 (masm->*insn)(reg, T, Address(base, disp));
2801 } else {
2802 assert(disp == 0, "unsupported address mode");
2803 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2804 }
2805 }
2806
2807 %}
2808
2809
2810
2811 //----------ENCODING BLOCK-----------------------------------------------------
2812 // This block specifies the encoding classes used by the compiler to
2813 // output byte streams. Encoding classes are parameterized macros
2814 // used by Machine Instruction Nodes in order to generate the bit
2815 // encoding of the instruction. Operands specify their base encoding
2816 // interface with the interface keyword. There are currently
2817 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2818 // COND_INTER. REG_INTER causes an operand to generate a function
2819 // which returns its register number when queried. CONST_INTER causes
2820 // an operand to generate a function which returns the value of the
2821 // constant when queried. MEMORY_INTER causes an operand to generate
2822 // four functions which return the Base Register, the Index Register,
2823 // the Scale Value, and the Offset Value of the operand when queried.
2824 // COND_INTER causes an operand to generate six functions which return
2825 // the encoding code (ie - encoding bits for the instruction)
2826 // associated with each basic boolean condition for a conditional
2827 // instruction.
2828 //
2829 // Instructions specify two basic values for encoding. Again, a
2830 // function is available to check if the constant displacement is an
2831 // oop. They use the ins_encode keyword to specify their encoding
2832 // classes (which must be a sequence of enc_class names, and their
2833 // parameters, specified in the encoding block), and they use the
2834 // opcode keyword to specify, in order, their primary, secondary, and
2835 // tertiary opcode. Only the opcode sections which a particular
2836 // instruction needs for encoding need to be specified.
2837 encode %{
2838 // Build emit functions for each basic byte or larger field in the
2839 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2840 // from C++ code in the enc_class source block. Emit functions will
2841 // live in the main source block for now. In future, we can
2842 // generalize this by adding a syntax that specifies the sizes of
2843 // fields in an order, so that the adlc can build the emit functions
2844 // automagically
2845
2846 // catch all for unimplemented encodings
2847 enc_class enc_unimplemented %{
2848 __ unimplemented("C2 catch all");
2849 %}
2850
2851 // BEGIN Non-volatile memory access
2852
2853 // This encoding class is generated automatically from ad_encode.m4.
2854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2855 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2856 Register dst_reg = as_Register($dst$$reg);
2857 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2858 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2859 %}
2860
2861 // This encoding class is generated automatically from ad_encode.m4.
2862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2863 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2864 Register dst_reg = as_Register($dst$$reg);
2865 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2866 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2867 %}
2868
2869 // This encoding class is generated automatically from ad_encode.m4.
2870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2871 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2872 Register dst_reg = as_Register($dst$$reg);
2873 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2874 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2875 %}
2876
2877 // This encoding class is generated automatically from ad_encode.m4.
2878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2879 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2880 Register dst_reg = as_Register($dst$$reg);
2881 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2882 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2883 %}
2884
2885 // This encoding class is generated automatically from ad_encode.m4.
2886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2887 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2888 Register dst_reg = as_Register($dst$$reg);
2889 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2890 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2891 %}
2892
2893 // This encoding class is generated automatically from ad_encode.m4.
2894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2895 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2896 Register dst_reg = as_Register($dst$$reg);
2897 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2898 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2899 %}
2900
2901 // This encoding class is generated automatically from ad_encode.m4.
2902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2903 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2904 Register dst_reg = as_Register($dst$$reg);
2905 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2906 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2907 %}
2908
2909 // This encoding class is generated automatically from ad_encode.m4.
2910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2911 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2912 Register dst_reg = as_Register($dst$$reg);
2913 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2914 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2915 %}
2916
2917 // This encoding class is generated automatically from ad_encode.m4.
2918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2919 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2920 Register dst_reg = as_Register($dst$$reg);
2921 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2922 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2923 %}
2924
2925 // This encoding class is generated automatically from ad_encode.m4.
2926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2927 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2928 Register dst_reg = as_Register($dst$$reg);
2929 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2930 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2931 %}
2932
2933 // This encoding class is generated automatically from ad_encode.m4.
2934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2935 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2936 Register dst_reg = as_Register($dst$$reg);
2937 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2938 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2939 %}
2940
2941 // This encoding class is generated automatically from ad_encode.m4.
2942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2943 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2944 Register dst_reg = as_Register($dst$$reg);
2945 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2946 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2947 %}
2948
2949 // This encoding class is generated automatically from ad_encode.m4.
2950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2951 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2952 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2953 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2954 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2955 %}
2956
2957 // This encoding class is generated automatically from ad_encode.m4.
2958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2959 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2960 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2961 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2962 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2963 %}
2964
2965 // This encoding class is generated automatically from ad_encode.m4.
2966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2967 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2968 Register src_reg = as_Register($src$$reg);
2969 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2970 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2971 %}
2972
2973 // This encoding class is generated automatically from ad_encode.m4.
2974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2975 enc_class aarch64_enc_strb0(memory1 mem) %{
2976 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2977 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2978 %}
2979
2980 // This encoding class is generated automatically from ad_encode.m4.
2981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2982 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2983 Register src_reg = as_Register($src$$reg);
2984 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2985 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2986 %}
2987
2988 // This encoding class is generated automatically from ad_encode.m4.
2989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2990 enc_class aarch64_enc_strh0(memory2 mem) %{
2991 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2992 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2993 %}
2994
2995 // This encoding class is generated automatically from ad_encode.m4.
2996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2997 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2998 Register src_reg = as_Register($src$$reg);
2999 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3000 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3001 %}
3002
3003 // This encoding class is generated automatically from ad_encode.m4.
3004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3005 enc_class aarch64_enc_strw0(memory4 mem) %{
3006 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3007 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3008 %}
3009
3010 // This encoding class is generated automatically from ad_encode.m4.
3011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3012 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3013 Register src_reg = as_Register($src$$reg);
3014 // we sometimes get asked to store the stack pointer into the
3015 // current thread -- we cannot do that directly on AArch64
3016 if (src_reg == r31_sp) {
3017 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3018 __ mov(rscratch2, sp);
3019 src_reg = rscratch2;
3020 }
3021 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3022 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3023 %}
3024
3025 // This encoding class is generated automatically from ad_encode.m4.
3026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3027 enc_class aarch64_enc_str0(memory8 mem) %{
3028 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3029 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3030 %}
3031
3032 // This encoding class is generated automatically from ad_encode.m4.
3033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3034 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3035 FloatRegister src_reg = as_FloatRegister($src$$reg);
3036 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3037 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3038 %}
3039
3040 // This encoding class is generated automatically from ad_encode.m4.
3041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3042 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3043 FloatRegister src_reg = as_FloatRegister($src$$reg);
3044 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3045 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3046 %}
3047
3048 // This encoding class is generated automatically from ad_encode.m4.
3049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3050 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3051 __ membar(Assembler::StoreStore);
3052 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3053 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3054 %}
3055
3056 // END Non-volatile memory access
3057
3058 // Vector loads and stores
3059 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3060 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3061 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3062 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3063 %}
3064
3065 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3066 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3067 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3068 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3069 %}
3070
3071 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3072 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3073 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3074 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3075 %}
3076
3077 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3078 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3079 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3080 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3081 %}
3082
3083 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3084 FloatRegister src_reg = as_FloatRegister($src$$reg);
3085 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3086 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3087 %}
3088
3089 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3090 FloatRegister src_reg = as_FloatRegister($src$$reg);
3091 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3092 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3093 %}
3094
3095 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3096 FloatRegister src_reg = as_FloatRegister($src$$reg);
3097 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3098 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3099 %}
3100
3101 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3102 FloatRegister src_reg = as_FloatRegister($src$$reg);
3103 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3104 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3105 %}
3106
3107 // volatile loads and stores
3108
3109 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3110 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3111 rscratch1, stlrb);
3112 %}
3113
3114 enc_class aarch64_enc_stlrb0(memory mem) %{
3115 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3116 rscratch1, stlrb);
3117 %}
3118
3119 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3120 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3121 rscratch1, stlrh);
3122 %}
3123
3124 enc_class aarch64_enc_stlrh0(memory mem) %{
3125 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3126 rscratch1, stlrh);
3127 %}
3128
3129 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3130 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3131 rscratch1, stlrw);
3132 %}
3133
3134 enc_class aarch64_enc_stlrw0(memory mem) %{
3135 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3136 rscratch1, stlrw);
3137 %}
3138
3139 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3140 Register dst_reg = as_Register($dst$$reg);
3141 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3142 rscratch1, ldarb);
3143 __ sxtbw(dst_reg, dst_reg);
3144 %}
3145
3146 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3147 Register dst_reg = as_Register($dst$$reg);
3148 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3149 rscratch1, ldarb);
3150 __ sxtb(dst_reg, dst_reg);
3151 %}
3152
3153 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3154 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3155 rscratch1, ldarb);
3156 %}
3157
3158 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3159 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, ldarb);
3161 %}
3162
3163 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3164 Register dst_reg = as_Register($dst$$reg);
3165 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3166 rscratch1, ldarh);
3167 __ sxthw(dst_reg, dst_reg);
3168 %}
3169
3170 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3171 Register dst_reg = as_Register($dst$$reg);
3172 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3173 rscratch1, ldarh);
3174 __ sxth(dst_reg, dst_reg);
3175 %}
3176
3177 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3178 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3179 rscratch1, ldarh);
3180 %}
3181
3182 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3183 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3184 rscratch1, ldarh);
3185 %}
3186
3187 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3188 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3189 rscratch1, ldarw);
3190 %}
3191
3192 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3193 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3194 rscratch1, ldarw);
3195 %}
3196
3197 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3198 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3199 rscratch1, ldar);
3200 %}
3201
3202 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3203 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3204 rscratch1, ldarw);
3205 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3206 %}
3207
3208 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3209 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3210 rscratch1, ldar);
3211 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3212 %}
3213
3214 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3215 Register src_reg = as_Register($src$$reg);
3216 // we sometimes get asked to store the stack pointer into the
3217 // current thread -- we cannot do that directly on AArch64
3218 if (src_reg == r31_sp) {
3219 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3220 __ mov(rscratch2, sp);
3221 src_reg = rscratch2;
3222 }
3223 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3224 rscratch1, stlr);
3225 %}
3226
3227 enc_class aarch64_enc_stlr0(memory mem) %{
3228 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3229 rscratch1, stlr);
3230 %}
3231
3232 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3233 {
3234 FloatRegister src_reg = as_FloatRegister($src$$reg);
3235 __ fmovs(rscratch2, src_reg);
3236 }
3237 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3238 rscratch1, stlrw);
3239 %}
3240
3241 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3242 {
3243 FloatRegister src_reg = as_FloatRegister($src$$reg);
3244 __ fmovd(rscratch2, src_reg);
3245 }
3246 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3247 rscratch1, stlr);
3248 %}
3249
3250 // synchronized read/update encodings
3251
3252 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3253 Register dst_reg = as_Register($dst$$reg);
3254 Register base = as_Register($mem$$base);
3255 int index = $mem$$index;
3256 int scale = $mem$$scale;
3257 int disp = $mem$$disp;
3258 if (index == -1) {
3259 if (disp != 0) {
3260 __ lea(rscratch1, Address(base, disp));
3261 __ ldaxr(dst_reg, rscratch1);
3262 } else {
3263 // TODO
3264 // should we ever get anything other than this case?
3265 __ ldaxr(dst_reg, base);
3266 }
3267 } else {
3268 Register index_reg = as_Register(index);
3269 if (disp == 0) {
3270 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3271 __ ldaxr(dst_reg, rscratch1);
3272 } else {
3273 __ lea(rscratch1, Address(base, disp));
3274 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3275 __ ldaxr(dst_reg, rscratch1);
3276 }
3277 }
3278 %}
3279
3280 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3281 Register src_reg = as_Register($src$$reg);
3282 Register base = as_Register($mem$$base);
3283 int index = $mem$$index;
3284 int scale = $mem$$scale;
3285 int disp = $mem$$disp;
3286 if (index == -1) {
3287 if (disp != 0) {
3288 __ lea(rscratch2, Address(base, disp));
3289 __ stlxr(rscratch1, src_reg, rscratch2);
3290 } else {
3291 // TODO
3292 // should we ever get anything other than this case?
3293 __ stlxr(rscratch1, src_reg, base);
3294 }
3295 } else {
3296 Register index_reg = as_Register(index);
3297 if (disp == 0) {
3298 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3299 __ stlxr(rscratch1, src_reg, rscratch2);
3300 } else {
3301 __ lea(rscratch2, Address(base, disp));
3302 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3303 __ stlxr(rscratch1, src_reg, rscratch2);
3304 }
3305 }
3306 __ cmpw(rscratch1, zr);
3307 %}
3308
3309 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3310 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3311 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3312 Assembler::xword, /*acquire*/ false, /*release*/ true,
3313 /*weak*/ false, noreg);
3314 %}
3315
3316 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3317 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3318 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3319 Assembler::word, /*acquire*/ false, /*release*/ true,
3320 /*weak*/ false, noreg);
3321 %}
3322
3323 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3324 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3325 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3326 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3327 /*weak*/ false, noreg);
3328 %}
3329
3330 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3331 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3332 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3333 Assembler::byte, /*acquire*/ false, /*release*/ true,
3334 /*weak*/ false, noreg);
3335 %}
3336
3337
3338 // The only difference between aarch64_enc_cmpxchg and
3339 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3340 // CompareAndSwap sequence to serve as a barrier on acquiring a
3341 // lock.
3342 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3343 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3344 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3345 Assembler::xword, /*acquire*/ true, /*release*/ true,
3346 /*weak*/ false, noreg);
3347 %}
3348
3349 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3350 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3351 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3352 Assembler::word, /*acquire*/ true, /*release*/ true,
3353 /*weak*/ false, noreg);
3354 %}
3355
3356 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3357 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3358 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3359 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3360 /*weak*/ false, noreg);
3361 %}
3362
3363 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3364 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3365 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3366 Assembler::byte, /*acquire*/ true, /*release*/ true,
3367 /*weak*/ false, noreg);
3368 %}
3369
3370 // auxiliary used for CompareAndSwapX to set result register
3371 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3372 Register res_reg = as_Register($res$$reg);
3373 __ cset(res_reg, Assembler::EQ);
3374 %}
3375
3376 // prefetch encodings
3377
3378 enc_class aarch64_enc_prefetchw(memory mem) %{
3379 Register base = as_Register($mem$$base);
3380 int index = $mem$$index;
3381 int scale = $mem$$scale;
3382 int disp = $mem$$disp;
3383 if (index == -1) {
3384 // Fix up any out-of-range offsets.
3385 assert_different_registers(rscratch1, base);
3386 Address addr = Address(base, disp);
3387 addr = __ legitimize_address(addr, 8, rscratch1);
3388 __ prfm(addr, PSTL1KEEP);
3389 } else {
3390 Register index_reg = as_Register(index);
3391 if (disp == 0) {
3392 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3393 } else {
3394 __ lea(rscratch1, Address(base, disp));
3395 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3396 }
3397 }
3398 %}
3399
3400 // mov encodings
3401
3402 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3403 uint32_t con = (uint32_t)$src$$constant;
3404 Register dst_reg = as_Register($dst$$reg);
3405 if (con == 0) {
3406 __ movw(dst_reg, zr);
3407 } else {
3408 __ movw(dst_reg, con);
3409 }
3410 %}
3411
3412 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3413 Register dst_reg = as_Register($dst$$reg);
3414 uint64_t con = (uint64_t)$src$$constant;
3415 if (con == 0) {
3416 __ mov(dst_reg, zr);
3417 } else {
3418 __ mov(dst_reg, con);
3419 }
3420 %}
3421
3422 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3423 Register dst_reg = as_Register($dst$$reg);
3424 address con = (address)$src$$constant;
3425 if (con == nullptr || con == (address)1) {
3426 ShouldNotReachHere();
3427 } else {
3428 relocInfo::relocType rtype = $src->constant_reloc();
3429 if (rtype == relocInfo::oop_type) {
3430 __ movoop(dst_reg, (jobject)con);
3431 } else if (rtype == relocInfo::metadata_type) {
3432 __ mov_metadata(dst_reg, (Metadata*)con);
3433 } else {
3434 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3435 if (! __ is_valid_AArch64_address(con) ||
3436 con < (address)(uintptr_t)os::vm_page_size()) {
3437 __ mov(dst_reg, con);
3438 } else {
3439 uint64_t offset;
3440 __ adrp(dst_reg, con, offset);
3441 __ add(dst_reg, dst_reg, offset);
3442 }
3443 }
3444 }
3445 %}
3446
3447 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3448 Register dst_reg = as_Register($dst$$reg);
3449 __ mov(dst_reg, zr);
3450 %}
3451
3452 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3453 Register dst_reg = as_Register($dst$$reg);
3454 __ mov(dst_reg, (uint64_t)1);
3455 %}
3456
3457 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3458 Register dst_reg = as_Register($dst$$reg);
3459 address con = (address)$src$$constant;
3460 if (con == nullptr) {
3461 ShouldNotReachHere();
3462 } else {
3463 relocInfo::relocType rtype = $src->constant_reloc();
3464 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3465 __ set_narrow_oop(dst_reg, (jobject)con);
3466 }
3467 %}
3468
3469 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3470 Register dst_reg = as_Register($dst$$reg);
3471 __ mov(dst_reg, zr);
3472 %}
3473
3474 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3475 Register dst_reg = as_Register($dst$$reg);
3476 address con = (address)$src$$constant;
3477 if (con == nullptr) {
3478 ShouldNotReachHere();
3479 } else {
3480 relocInfo::relocType rtype = $src->constant_reloc();
3481 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3482 __ set_narrow_klass(dst_reg, (Klass *)con);
3483 }
3484 %}
3485
3486 // arithmetic encodings
3487
3488 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3489 Register dst_reg = as_Register($dst$$reg);
3490 Register src_reg = as_Register($src1$$reg);
3491 int32_t con = (int32_t)$src2$$constant;
3492 // add has primary == 0, subtract has primary == 1
3493 if ($primary) { con = -con; }
3494 if (con < 0) {
3495 __ subw(dst_reg, src_reg, -con);
3496 } else {
3497 __ addw(dst_reg, src_reg, con);
3498 }
3499 %}
3500
3501 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3502 Register dst_reg = as_Register($dst$$reg);
3503 Register src_reg = as_Register($src1$$reg);
3504 int32_t con = (int32_t)$src2$$constant;
3505 // add has primary == 0, subtract has primary == 1
3506 if ($primary) { con = -con; }
3507 if (con < 0) {
3508 __ sub(dst_reg, src_reg, -con);
3509 } else {
3510 __ add(dst_reg, src_reg, con);
3511 }
3512 %}
3513
3514 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3515 Register dst_reg = as_Register($dst$$reg);
3516 Register src1_reg = as_Register($src1$$reg);
3517 Register src2_reg = as_Register($src2$$reg);
3518 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3519 %}
3520
3521 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3522 Register dst_reg = as_Register($dst$$reg);
3523 Register src1_reg = as_Register($src1$$reg);
3524 Register src2_reg = as_Register($src2$$reg);
3525 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3526 %}
3527
3528 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3529 Register dst_reg = as_Register($dst$$reg);
3530 Register src1_reg = as_Register($src1$$reg);
3531 Register src2_reg = as_Register($src2$$reg);
3532 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3533 %}
3534
3535 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3536 Register dst_reg = as_Register($dst$$reg);
3537 Register src1_reg = as_Register($src1$$reg);
3538 Register src2_reg = as_Register($src2$$reg);
3539 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3540 %}
3541
3542 // compare instruction encodings
3543
3544 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3545 Register reg1 = as_Register($src1$$reg);
3546 Register reg2 = as_Register($src2$$reg);
3547 __ cmpw(reg1, reg2);
3548 %}
3549
3550 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3551 Register reg = as_Register($src1$$reg);
3552 int32_t val = $src2$$constant;
3553 if (val >= 0) {
3554 __ subsw(zr, reg, val);
3555 } else {
3556 __ addsw(zr, reg, -val);
3557 }
3558 %}
3559
3560 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3561 Register reg1 = as_Register($src1$$reg);
3562 uint32_t val = (uint32_t)$src2$$constant;
3563 __ movw(rscratch1, val);
3564 __ cmpw(reg1, rscratch1);
3565 %}
3566
3567 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3568 Register reg1 = as_Register($src1$$reg);
3569 Register reg2 = as_Register($src2$$reg);
3570 __ cmp(reg1, reg2);
3571 %}
3572
3573 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3574 Register reg = as_Register($src1$$reg);
3575 int64_t val = $src2$$constant;
3576 if (val >= 0) {
3577 __ subs(zr, reg, val);
3578 } else if (val != -val) {
3579 __ adds(zr, reg, -val);
3580 } else {
3581 // aargh, Long.MIN_VALUE is a special case
3582 __ orr(rscratch1, zr, (uint64_t)val);
3583 __ subs(zr, reg, rscratch1);
3584 }
3585 %}
3586
3587 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3588 Register reg1 = as_Register($src1$$reg);
3589 uint64_t val = (uint64_t)$src2$$constant;
3590 __ mov(rscratch1, val);
3591 __ cmp(reg1, rscratch1);
3592 %}
3593
3594 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3595 Register reg1 = as_Register($src1$$reg);
3596 Register reg2 = as_Register($src2$$reg);
3597 __ cmp(reg1, reg2);
3598 %}
3599
3600 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3601 Register reg1 = as_Register($src1$$reg);
3602 Register reg2 = as_Register($src2$$reg);
3603 __ cmpw(reg1, reg2);
3604 %}
3605
3606 enc_class aarch64_enc_testp(iRegP src) %{
3607 Register reg = as_Register($src$$reg);
3608 __ cmp(reg, zr);
3609 %}
3610
3611 enc_class aarch64_enc_testn(iRegN src) %{
3612 Register reg = as_Register($src$$reg);
3613 __ cmpw(reg, zr);
3614 %}
3615
3616 enc_class aarch64_enc_b(label lbl) %{
3617 Label *L = $lbl$$label;
3618 __ b(*L);
3619 %}
3620
3621 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3622 Label *L = $lbl$$label;
3623 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3624 %}
3625
3626 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3627 Label *L = $lbl$$label;
3628 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3629 %}
3630
3631 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3632 %{
3633 Register sub_reg = as_Register($sub$$reg);
3634 Register super_reg = as_Register($super$$reg);
3635 Register temp_reg = as_Register($temp$$reg);
3636 Register result_reg = as_Register($result$$reg);
3637
3638 Label miss;
3639 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3640 nullptr, &miss,
3641 /*set_cond_codes:*/ true);
3642 if ($primary) {
3643 __ mov(result_reg, zr);
3644 }
3645 __ bind(miss);
3646 %}
3647
3648 enc_class aarch64_enc_java_static_call(method meth) %{
3649 address addr = (address)$meth$$method;
3650 address call;
3651 if (!_method) {
3652 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3653 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3654 if (call == nullptr) {
3655 ciEnv::current()->record_failure("CodeCache is full");
3656 return;
3657 }
3658 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3659 // The NOP here is purely to ensure that eliding a call to
3660 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3661 __ nop();
3662 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3663 } else {
3664 int method_index = resolved_method_index(masm);
3665 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3666 : static_call_Relocation::spec(method_index);
3667 call = __ trampoline_call(Address(addr, rspec));
3668 if (call == nullptr) {
3669 ciEnv::current()->record_failure("CodeCache is full");
3670 return;
3671 }
3672 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3673 // Calls of the same statically bound method can share
3674 // a stub to the interpreter.
3675 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3676 } else {
3677 // Emit stub for static call
3678 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3679 if (stub == nullptr) {
3680 ciEnv::current()->record_failure("CodeCache is full");
3681 return;
3682 }
3683 }
3684 }
3685
3686 __ post_call_nop();
3687
3688 // Only non uncommon_trap calls need to reinitialize ptrue.
3689 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3690 __ reinitialize_ptrue();
3691 }
3692 %}
3693
3694 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3695 int method_index = resolved_method_index(masm);
3696 address call = __ ic_call((address)$meth$$method, method_index);
3697 if (call == nullptr) {
3698 ciEnv::current()->record_failure("CodeCache is full");
3699 return;
3700 }
3701 __ post_call_nop();
3702 if (Compile::current()->max_vector_size() > 0) {
3703 __ reinitialize_ptrue();
3704 }
3705 %}
3706
3707 enc_class aarch64_enc_call_epilog() %{
3708 if (VerifyStackAtCalls) {
3709 // Check that stack depth is unchanged: find majik cookie on stack
3710 __ call_Unimplemented();
3711 }
3712 %}
3713
3714 enc_class aarch64_enc_java_to_runtime(method meth) %{
3715 // some calls to generated routines (arraycopy code) are scheduled
3716 // by C2 as runtime calls. if so we can call them using a br (they
3717 // will be in a reachable segment) otherwise we have to use a blr
3718 // which loads the absolute address into a register.
3719 address entry = (address)$meth$$method;
3720 CodeBlob *cb = CodeCache::find_blob(entry);
3721 if (cb) {
3722 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3723 if (call == nullptr) {
3724 ciEnv::current()->record_failure("CodeCache is full");
3725 return;
3726 }
3727 __ post_call_nop();
3728 } else {
3729 Label retaddr;
3730 // Make the anchor frame walkable
3731 __ adr(rscratch2, retaddr);
3732 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3733 __ lea(rscratch1, RuntimeAddress(entry));
3734 __ blr(rscratch1);
3735 __ bind(retaddr);
3736 __ post_call_nop();
3737 }
3738 if (Compile::current()->max_vector_size() > 0) {
3739 __ reinitialize_ptrue();
3740 }
3741 %}
3742
3743 enc_class aarch64_enc_rethrow() %{
3744 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3745 %}
3746
3747 enc_class aarch64_enc_ret() %{
3748 #ifdef ASSERT
3749 if (Compile::current()->max_vector_size() > 0) {
3750 __ verify_ptrue();
3751 }
3752 #endif
3753 __ ret(lr);
3754 %}
3755
3756 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3757 Register target_reg = as_Register($jump_target$$reg);
3758 __ br(target_reg);
3759 %}
3760
3761 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3762 Register target_reg = as_Register($jump_target$$reg);
3763 // exception oop should be in r0
3764 // ret addr has been popped into lr
3765 // callee expects it in r3
3766 __ mov(r3, lr);
3767 __ br(target_reg);
3768 %}
3769
3770 %}
3771
3772 //----------FRAME--------------------------------------------------------------
3773 // Definition of frame structure and management information.
3774 //
3775 // S T A C K L A Y O U T Allocators stack-slot number
3776 // | (to get allocators register number
3777 // G Owned by | | v add OptoReg::stack0())
3778 // r CALLER | |
3779 // o | +--------+ pad to even-align allocators stack-slot
3780 // w V | pad0 | numbers; owned by CALLER
3781 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3782 // h ^ | in | 5
3783 // | | args | 4 Holes in incoming args owned by SELF
3784 // | | | | 3
3785 // | | +--------+
3786 // V | | old out| Empty on Intel, window on Sparc
3787 // | old |preserve| Must be even aligned.
3788 // | SP-+--------+----> Matcher::_old_SP, even aligned
3789 // | | in | 3 area for Intel ret address
3790 // Owned by |preserve| Empty on Sparc.
3791 // SELF +--------+
3792 // | | pad2 | 2 pad to align old SP
3793 // | +--------+ 1
3794 // | | locks | 0
3795 // | +--------+----> OptoReg::stack0(), even aligned
3796 // | | pad1 | 11 pad to align new SP
3797 // | +--------+
3798 // | | | 10
3799 // | | spills | 9 spills
3800 // V | | 8 (pad0 slot for callee)
3801 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3802 // ^ | out | 7
3803 // | | args | 6 Holes in outgoing args owned by CALLEE
3804 // Owned by +--------+
3805 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3806 // | new |preserve| Must be even-aligned.
3807 // | SP-+--------+----> Matcher::_new_SP, even aligned
3808 // | | |
3809 //
3810 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3811 // known from SELF's arguments and the Java calling convention.
3812 // Region 6-7 is determined per call site.
3813 // Note 2: If the calling convention leaves holes in the incoming argument
3814 // area, those holes are owned by SELF. Holes in the outgoing area
3815 // are owned by the CALLEE. Holes should not be necessary in the
3816 // incoming area, as the Java calling convention is completely under
3817 // the control of the AD file. Doubles can be sorted and packed to
3818 // avoid holes. Holes in the outgoing arguments may be necessary for
3819 // varargs C calling conventions.
3820 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3821 // even aligned with pad0 as needed.
3822 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3823 // (the latter is true on Intel but is it false on AArch64?)
3824 // region 6-11 is even aligned; it may be padded out more so that
3825 // the region from SP to FP meets the minimum stack alignment.
3826 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3827 // alignment. Region 11, pad1, may be dynamically extended so that
3828 // SP meets the minimum alignment.
3829
3830 frame %{
3831 // These three registers define part of the calling convention
3832 // between compiled code and the interpreter.
3833
3834 // Inline Cache Register or Method for I2C.
3835 inline_cache_reg(R12);
3836
3837 // Number of stack slots consumed by locking an object
3838 sync_stack_slots(2);
3839
3840 // Compiled code's Frame Pointer
3841 frame_pointer(R31);
3842
3843 // Interpreter stores its frame pointer in a register which is
3844 // stored to the stack by I2CAdaptors.
3845 // I2CAdaptors convert from interpreted java to compiled java.
3846 interpreter_frame_pointer(R29);
3847
3848 // Stack alignment requirement
3849 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3850
3851 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3852 // for calls to C. Supports the var-args backing area for register parms.
3853 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3854
3855 // The after-PROLOG location of the return address. Location of
3856 // return address specifies a type (REG or STACK) and a number
3857 // representing the register number (i.e. - use a register name) or
3858 // stack slot.
3859 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3860 // Otherwise, it is above the locks and verification slot and alignment word
3861 // TODO this may well be correct but need to check why that - 2 is there
3862 // ppc port uses 0 but we definitely need to allow for fixed_slots
3863 // which folds in the space used for monitors
3864 return_addr(STACK - 2 +
3865 align_up((Compile::current()->in_preserve_stack_slots() +
3866 Compile::current()->fixed_slots()),
3867 stack_alignment_in_slots()));
3868
3869 // Location of compiled Java return values. Same as C for now.
3870 return_value
3871 %{
3872 // TODO do we allow ideal_reg == Op_RegN???
3873 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3874 "only return normal values");
3875
3876 static const int lo[Op_RegL + 1] = { // enum name
3877 0, // Op_Node
3878 0, // Op_Set
3879 R0_num, // Op_RegN
3880 R0_num, // Op_RegI
3881 R0_num, // Op_RegP
3882 V0_num, // Op_RegF
3883 V0_num, // Op_RegD
3884 R0_num // Op_RegL
3885 };
3886
3887 static const int hi[Op_RegL + 1] = { // enum name
3888 0, // Op_Node
3889 0, // Op_Set
3890 OptoReg::Bad, // Op_RegN
3891 OptoReg::Bad, // Op_RegI
3892 R0_H_num, // Op_RegP
3893 OptoReg::Bad, // Op_RegF
3894 V0_H_num, // Op_RegD
3895 R0_H_num // Op_RegL
3896 };
3897
3898 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3899 %}
3900 %}
3901
3902 //----------ATTRIBUTES---------------------------------------------------------
3903 //----------Operand Attributes-------------------------------------------------
3904 op_attrib op_cost(1); // Required cost attribute
3905
3906 //----------Instruction Attributes---------------------------------------------
3907 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3908 ins_attrib ins_size(32); // Required size attribute (in bits)
3909 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3910 // a non-matching short branch variant
3911 // of some long branch?
3912 ins_attrib ins_alignment(4); // Required alignment attribute (must
3913 // be a power of 2) specifies the
3914 // alignment that some part of the
3915 // instruction (not necessarily the
3916 // start) requires. If > 1, a
3917 // compute_padding() function must be
3918 // provided for the instruction
3919
3920 //----------OPERANDS-----------------------------------------------------------
3921 // Operand definitions must precede instruction definitions for correct parsing
3922 // in the ADLC because operands constitute user defined types which are used in
3923 // instruction definitions.
3924
3925 //----------Simple Operands----------------------------------------------------
3926
3927 // Integer operands 32 bit
3928 // 32 bit immediate
3929 operand immI()
3930 %{
3931 match(ConI);
3932
3933 op_cost(0);
3934 format %{ %}
3935 interface(CONST_INTER);
3936 %}
3937
3938 // 32 bit zero
3939 operand immI0()
3940 %{
3941 predicate(n->get_int() == 0);
3942 match(ConI);
3943
3944 op_cost(0);
3945 format %{ %}
3946 interface(CONST_INTER);
3947 %}
3948
3949 // 32 bit unit increment
3950 operand immI_1()
3951 %{
3952 predicate(n->get_int() == 1);
3953 match(ConI);
3954
3955 op_cost(0);
3956 format %{ %}
3957 interface(CONST_INTER);
3958 %}
3959
3960 // 32 bit unit decrement
3961 operand immI_M1()
3962 %{
3963 predicate(n->get_int() == -1);
3964 match(ConI);
3965
3966 op_cost(0);
3967 format %{ %}
3968 interface(CONST_INTER);
3969 %}
3970
3971 // Shift values for add/sub extension shift
3972 operand immIExt()
3973 %{
3974 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3975 match(ConI);
3976
3977 op_cost(0);
3978 format %{ %}
3979 interface(CONST_INTER);
3980 %}
3981
3982 operand immI_gt_1()
3983 %{
3984 predicate(n->get_int() > 1);
3985 match(ConI);
3986
3987 op_cost(0);
3988 format %{ %}
3989 interface(CONST_INTER);
3990 %}
3991
3992 operand immI_le_4()
3993 %{
3994 predicate(n->get_int() <= 4);
3995 match(ConI);
3996
3997 op_cost(0);
3998 format %{ %}
3999 interface(CONST_INTER);
4000 %}
4001
4002 operand immI_16()
4003 %{
4004 predicate(n->get_int() == 16);
4005 match(ConI);
4006
4007 op_cost(0);
4008 format %{ %}
4009 interface(CONST_INTER);
4010 %}
4011
4012 operand immI_24()
4013 %{
4014 predicate(n->get_int() == 24);
4015 match(ConI);
4016
4017 op_cost(0);
4018 format %{ %}
4019 interface(CONST_INTER);
4020 %}
4021
4022 operand immI_32()
4023 %{
4024 predicate(n->get_int() == 32);
4025 match(ConI);
4026
4027 op_cost(0);
4028 format %{ %}
4029 interface(CONST_INTER);
4030 %}
4031
4032 operand immI_48()
4033 %{
4034 predicate(n->get_int() == 48);
4035 match(ConI);
4036
4037 op_cost(0);
4038 format %{ %}
4039 interface(CONST_INTER);
4040 %}
4041
4042 operand immI_56()
4043 %{
4044 predicate(n->get_int() == 56);
4045 match(ConI);
4046
4047 op_cost(0);
4048 format %{ %}
4049 interface(CONST_INTER);
4050 %}
4051
4052 operand immI_255()
4053 %{
4054 predicate(n->get_int() == 255);
4055 match(ConI);
4056
4057 op_cost(0);
4058 format %{ %}
4059 interface(CONST_INTER);
4060 %}
4061
4062 operand immI_65535()
4063 %{
4064 predicate(n->get_int() == 65535);
4065 match(ConI);
4066
4067 op_cost(0);
4068 format %{ %}
4069 interface(CONST_INTER);
4070 %}
4071
4072 operand immI_positive()
4073 %{
4074 predicate(n->get_int() > 0);
4075 match(ConI);
4076
4077 op_cost(0);
4078 format %{ %}
4079 interface(CONST_INTER);
4080 %}
4081
4082 // BoolTest condition for signed compare
4083 operand immI_cmp_cond()
4084 %{
4085 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4086 match(ConI);
4087
4088 op_cost(0);
4089 format %{ %}
4090 interface(CONST_INTER);
4091 %}
4092
4093 // BoolTest condition for unsigned compare
4094 operand immI_cmpU_cond()
4095 %{
4096 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4097 match(ConI);
4098
4099 op_cost(0);
4100 format %{ %}
4101 interface(CONST_INTER);
4102 %}
4103
4104 operand immL_255()
4105 %{
4106 predicate(n->get_long() == 255L);
4107 match(ConL);
4108
4109 op_cost(0);
4110 format %{ %}
4111 interface(CONST_INTER);
4112 %}
4113
4114 operand immL_65535()
4115 %{
4116 predicate(n->get_long() == 65535L);
4117 match(ConL);
4118
4119 op_cost(0);
4120 format %{ %}
4121 interface(CONST_INTER);
4122 %}
4123
4124 operand immL_4294967295()
4125 %{
4126 predicate(n->get_long() == 4294967295L);
4127 match(ConL);
4128
4129 op_cost(0);
4130 format %{ %}
4131 interface(CONST_INTER);
4132 %}
4133
4134 operand immL_bitmask()
4135 %{
4136 predicate((n->get_long() != 0)
4137 && ((n->get_long() & 0xc000000000000000l) == 0)
4138 && is_power_of_2(n->get_long() + 1));
4139 match(ConL);
4140
4141 op_cost(0);
4142 format %{ %}
4143 interface(CONST_INTER);
4144 %}
4145
4146 operand immI_bitmask()
4147 %{
4148 predicate((n->get_int() != 0)
4149 && ((n->get_int() & 0xc0000000) == 0)
4150 && is_power_of_2(n->get_int() + 1));
4151 match(ConI);
4152
4153 op_cost(0);
4154 format %{ %}
4155 interface(CONST_INTER);
4156 %}
4157
4158 operand immL_positive_bitmaskI()
4159 %{
4160 predicate((n->get_long() != 0)
4161 && ((julong)n->get_long() < 0x80000000ULL)
4162 && is_power_of_2(n->get_long() + 1));
4163 match(ConL);
4164
4165 op_cost(0);
4166 format %{ %}
4167 interface(CONST_INTER);
4168 %}
4169
4170 // Scale values for scaled offset addressing modes (up to long but not quad)
4171 operand immIScale()
4172 %{
4173 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4174 match(ConI);
4175
4176 op_cost(0);
4177 format %{ %}
4178 interface(CONST_INTER);
4179 %}
4180
4181 // 5 bit signed integer
4182 operand immI5()
4183 %{
4184 predicate(Assembler::is_simm(n->get_int(), 5));
4185 match(ConI);
4186
4187 op_cost(0);
4188 format %{ %}
4189 interface(CONST_INTER);
4190 %}
4191
4192 // 7 bit unsigned integer
4193 operand immIU7()
4194 %{
4195 predicate(Assembler::is_uimm(n->get_int(), 7));
4196 match(ConI);
4197
4198 op_cost(0);
4199 format %{ %}
4200 interface(CONST_INTER);
4201 %}
4202
4203 // Offset for scaled or unscaled immediate loads and stores
4204 operand immIOffset()
4205 %{
4206 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4207 match(ConI);
4208
4209 op_cost(0);
4210 format %{ %}
4211 interface(CONST_INTER);
4212 %}
4213
4214 operand immIOffset1()
4215 %{
4216 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4217 match(ConI);
4218
4219 op_cost(0);
4220 format %{ %}
4221 interface(CONST_INTER);
4222 %}
4223
4224 operand immIOffset2()
4225 %{
4226 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4227 match(ConI);
4228
4229 op_cost(0);
4230 format %{ %}
4231 interface(CONST_INTER);
4232 %}
4233
4234 operand immIOffset4()
4235 %{
4236 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4237 match(ConI);
4238
4239 op_cost(0);
4240 format %{ %}
4241 interface(CONST_INTER);
4242 %}
4243
4244 operand immIOffset8()
4245 %{
4246 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4247 match(ConI);
4248
4249 op_cost(0);
4250 format %{ %}
4251 interface(CONST_INTER);
4252 %}
4253
4254 operand immIOffset16()
4255 %{
4256 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4257 match(ConI);
4258
4259 op_cost(0);
4260 format %{ %}
4261 interface(CONST_INTER);
4262 %}
4263
4264 operand immLOffset()
4265 %{
4266 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4267 match(ConL);
4268
4269 op_cost(0);
4270 format %{ %}
4271 interface(CONST_INTER);
4272 %}
4273
4274 operand immLoffset1()
4275 %{
4276 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4277 match(ConL);
4278
4279 op_cost(0);
4280 format %{ %}
4281 interface(CONST_INTER);
4282 %}
4283
4284 operand immLoffset2()
4285 %{
4286 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4287 match(ConL);
4288
4289 op_cost(0);
4290 format %{ %}
4291 interface(CONST_INTER);
4292 %}
4293
4294 operand immLoffset4()
4295 %{
4296 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4297 match(ConL);
4298
4299 op_cost(0);
4300 format %{ %}
4301 interface(CONST_INTER);
4302 %}
4303
4304 operand immLoffset8()
4305 %{
4306 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4307 match(ConL);
4308
4309 op_cost(0);
4310 format %{ %}
4311 interface(CONST_INTER);
4312 %}
4313
4314 operand immLoffset16()
4315 %{
4316 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4317 match(ConL);
4318
4319 op_cost(0);
4320 format %{ %}
4321 interface(CONST_INTER);
4322 %}
4323
4324 // 5 bit signed long integer
4325 operand immL5()
4326 %{
4327 predicate(Assembler::is_simm(n->get_long(), 5));
4328 match(ConL);
4329
4330 op_cost(0);
4331 format %{ %}
4332 interface(CONST_INTER);
4333 %}
4334
4335 // 7 bit unsigned long integer
4336 operand immLU7()
4337 %{
4338 predicate(Assembler::is_uimm(n->get_long(), 7));
4339 match(ConL);
4340
4341 op_cost(0);
4342 format %{ %}
4343 interface(CONST_INTER);
4344 %}
4345
4346 // 8 bit signed value.
4347 operand immI8()
4348 %{
4349 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4350 match(ConI);
4351
4352 op_cost(0);
4353 format %{ %}
4354 interface(CONST_INTER);
4355 %}
4356
4357 // 8 bit signed value (simm8), or #simm8 LSL 8.
4358 operand immI8_shift8()
4359 %{
4360 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4361 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4362 match(ConI);
4363
4364 op_cost(0);
4365 format %{ %}
4366 interface(CONST_INTER);
4367 %}
4368
4369 // 8 bit signed value (simm8), or #simm8 LSL 8.
4370 operand immL8_shift8()
4371 %{
4372 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4373 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4374 match(ConL);
4375
4376 op_cost(0);
4377 format %{ %}
4378 interface(CONST_INTER);
4379 %}
4380
4381 // 8 bit integer valid for vector add sub immediate
4382 operand immBAddSubV()
4383 %{
4384 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4385 match(ConI);
4386
4387 op_cost(0);
4388 format %{ %}
4389 interface(CONST_INTER);
4390 %}
4391
4392 // 32 bit integer valid for add sub immediate
4393 operand immIAddSub()
4394 %{
4395 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4396 match(ConI);
4397 op_cost(0);
4398 format %{ %}
4399 interface(CONST_INTER);
4400 %}
4401
4402 // 32 bit integer valid for vector add sub immediate
4403 operand immIAddSubV()
4404 %{
4405 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4406 match(ConI);
4407
4408 op_cost(0);
4409 format %{ %}
4410 interface(CONST_INTER);
4411 %}
4412
4413 // 32 bit unsigned integer valid for logical immediate
4414
4415 operand immBLog()
4416 %{
4417 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4418 match(ConI);
4419
4420 op_cost(0);
4421 format %{ %}
4422 interface(CONST_INTER);
4423 %}
4424
4425 operand immSLog()
4426 %{
4427 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4428 match(ConI);
4429
4430 op_cost(0);
4431 format %{ %}
4432 interface(CONST_INTER);
4433 %}
4434
4435 operand immILog()
4436 %{
4437 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4438 match(ConI);
4439
4440 op_cost(0);
4441 format %{ %}
4442 interface(CONST_INTER);
4443 %}
4444
4445 // Integer operands 64 bit
4446 // 64 bit immediate
4447 operand immL()
4448 %{
4449 match(ConL);
4450
4451 op_cost(0);
4452 format %{ %}
4453 interface(CONST_INTER);
4454 %}
4455
4456 // 64 bit zero
4457 operand immL0()
4458 %{
4459 predicate(n->get_long() == 0);
4460 match(ConL);
4461
4462 op_cost(0);
4463 format %{ %}
4464 interface(CONST_INTER);
4465 %}
4466
4467 // 64 bit unit decrement
4468 operand immL_M1()
4469 %{
4470 predicate(n->get_long() == -1);
4471 match(ConL);
4472
4473 op_cost(0);
4474 format %{ %}
4475 interface(CONST_INTER);
4476 %}
4477
4478 // 64 bit integer valid for add sub immediate
4479 operand immLAddSub()
4480 %{
4481 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4482 match(ConL);
4483 op_cost(0);
4484 format %{ %}
4485 interface(CONST_INTER);
4486 %}
4487
4488 // 64 bit integer valid for addv subv immediate
4489 operand immLAddSubV()
4490 %{
4491 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4492 match(ConL);
4493
4494 op_cost(0);
4495 format %{ %}
4496 interface(CONST_INTER);
4497 %}
4498
4499 // 64 bit integer valid for logical immediate
4500 operand immLLog()
4501 %{
4502 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4503 match(ConL);
4504 op_cost(0);
4505 format %{ %}
4506 interface(CONST_INTER);
4507 %}
4508
4509 // Long Immediate: low 32-bit mask
4510 operand immL_32bits()
4511 %{
4512 predicate(n->get_long() == 0xFFFFFFFFL);
4513 match(ConL);
4514 op_cost(0);
4515 format %{ %}
4516 interface(CONST_INTER);
4517 %}
4518
4519 // Pointer operands
4520 // Pointer Immediate
4521 operand immP()
4522 %{
4523 match(ConP);
4524
4525 op_cost(0);
4526 format %{ %}
4527 interface(CONST_INTER);
4528 %}
4529
4530 // nullptr Pointer Immediate
4531 operand immP0()
4532 %{
4533 predicate(n->get_ptr() == 0);
4534 match(ConP);
4535
4536 op_cost(0);
4537 format %{ %}
4538 interface(CONST_INTER);
4539 %}
4540
4541 // Pointer Immediate One
4542 // this is used in object initialization (initial object header)
4543 operand immP_1()
4544 %{
4545 predicate(n->get_ptr() == 1);
4546 match(ConP);
4547
4548 op_cost(0);
4549 format %{ %}
4550 interface(CONST_INTER);
4551 %}
4552
4553 // Card Table Byte Map Base
4554 operand immByteMapBase()
4555 %{
4556 // Get base of card map
4557 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4558 is_card_table_address((address)(n->get_ptr())));
4559 match(ConP);
4560
4561 op_cost(0);
4562 format %{ %}
4563 interface(CONST_INTER);
4564 %}
4565
4566 // AOT Runtime Constants Address
4567 operand immAOTRuntimeConstantsAddress()
4568 %{
4569 // Check if the address is in the range of AOT Runtime Constants
4570 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4571 match(ConP);
4572
4573 op_cost(0);
4574 format %{ %}
4575 interface(CONST_INTER);
4576 %}
4577
4578 // Float and Double operands
4579 // Double Immediate
4580 operand immD()
4581 %{
4582 match(ConD);
4583 op_cost(0);
4584 format %{ %}
4585 interface(CONST_INTER);
4586 %}
4587
4588 // Double Immediate: +0.0d
4589 operand immD0()
4590 %{
4591 predicate(jlong_cast(n->getd()) == 0);
4592 match(ConD);
4593
4594 op_cost(0);
4595 format %{ %}
4596 interface(CONST_INTER);
4597 %}
4598
4599 // constant 'double +0.0'.
4600 operand immDPacked()
4601 %{
4602 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4603 match(ConD);
4604 op_cost(0);
4605 format %{ %}
4606 interface(CONST_INTER);
4607 %}
4608
4609 // Float Immediate
4610 operand immF()
4611 %{
4612 match(ConF);
4613 op_cost(0);
4614 format %{ %}
4615 interface(CONST_INTER);
4616 %}
4617
4618 // Float Immediate: +0.0f.
4619 operand immF0()
4620 %{
4621 predicate(jint_cast(n->getf()) == 0);
4622 match(ConF);
4623
4624 op_cost(0);
4625 format %{ %}
4626 interface(CONST_INTER);
4627 %}
4628
4629 // Half Float (FP16) Immediate
4630 operand immH()
4631 %{
4632 match(ConH);
4633 op_cost(0);
4634 format %{ %}
4635 interface(CONST_INTER);
4636 %}
4637
4638 //
4639 operand immFPacked()
4640 %{
4641 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4642 match(ConF);
4643 op_cost(0);
4644 format %{ %}
4645 interface(CONST_INTER);
4646 %}
4647
4648 // Narrow pointer operands
4649 // Narrow Pointer Immediate
4650 operand immN()
4651 %{
4652 match(ConN);
4653
4654 op_cost(0);
4655 format %{ %}
4656 interface(CONST_INTER);
4657 %}
4658
4659 // Narrow nullptr Pointer Immediate
4660 operand immN0()
4661 %{
4662 predicate(n->get_narrowcon() == 0);
4663 match(ConN);
4664
4665 op_cost(0);
4666 format %{ %}
4667 interface(CONST_INTER);
4668 %}
4669
4670 operand immNKlass()
4671 %{
4672 match(ConNKlass);
4673
4674 op_cost(0);
4675 format %{ %}
4676 interface(CONST_INTER);
4677 %}
4678
4679 // Integer 32 bit Register Operands
4680 // Integer 32 bitRegister (excludes SP)
4681 operand iRegI()
4682 %{
4683 constraint(ALLOC_IN_RC(any_reg32));
4684 match(RegI);
4685 match(iRegINoSp);
4686 op_cost(0);
4687 format %{ %}
4688 interface(REG_INTER);
4689 %}
4690
4691 // Integer 32 bit Register not Special
4692 operand iRegINoSp()
4693 %{
4694 constraint(ALLOC_IN_RC(no_special_reg32));
4695 match(RegI);
4696 op_cost(0);
4697 format %{ %}
4698 interface(REG_INTER);
4699 %}
4700
4701 // Integer 64 bit Register Operands
4702 // Integer 64 bit Register (includes SP)
4703 operand iRegL()
4704 %{
4705 constraint(ALLOC_IN_RC(any_reg));
4706 match(RegL);
4707 match(iRegLNoSp);
4708 op_cost(0);
4709 format %{ %}
4710 interface(REG_INTER);
4711 %}
4712
4713 // Integer 64 bit Register not Special
4714 operand iRegLNoSp()
4715 %{
4716 constraint(ALLOC_IN_RC(no_special_reg));
4717 match(RegL);
4718 match(iRegL_R0);
4719 format %{ %}
4720 interface(REG_INTER);
4721 %}
4722
4723 // Pointer Register Operands
4724 // Pointer Register
4725 operand iRegP()
4726 %{
4727 constraint(ALLOC_IN_RC(ptr_reg));
4728 match(RegP);
4729 match(iRegPNoSp);
4730 match(iRegP_R0);
4731 //match(iRegP_R2);
4732 //match(iRegP_R4);
4733 match(iRegP_R5);
4734 match(thread_RegP);
4735 op_cost(0);
4736 format %{ %}
4737 interface(REG_INTER);
4738 %}
4739
4740 // Pointer 64 bit Register not Special
4741 operand iRegPNoSp()
4742 %{
4743 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4744 match(RegP);
4745 // match(iRegP);
4746 // match(iRegP_R0);
4747 // match(iRegP_R2);
4748 // match(iRegP_R4);
4749 // match(iRegP_R5);
4750 // match(thread_RegP);
4751 op_cost(0);
4752 format %{ %}
4753 interface(REG_INTER);
4754 %}
4755
4756 // This operand is not allowed to use rfp even if
4757 // rfp is not used to hold the frame pointer.
4758 operand iRegPNoSpNoRfp()
4759 %{
4760 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4761 match(RegP);
4762 match(iRegPNoSp);
4763 op_cost(0);
4764 format %{ %}
4765 interface(REG_INTER);
4766 %}
4767
4768 // Pointer 64 bit Register R0 only
4769 operand iRegP_R0()
4770 %{
4771 constraint(ALLOC_IN_RC(r0_reg));
4772 match(RegP);
4773 // match(iRegP);
4774 match(iRegPNoSp);
4775 op_cost(0);
4776 format %{ %}
4777 interface(REG_INTER);
4778 %}
4779
4780 // Pointer 64 bit Register R1 only
4781 operand iRegP_R1()
4782 %{
4783 constraint(ALLOC_IN_RC(r1_reg));
4784 match(RegP);
4785 // match(iRegP);
4786 match(iRegPNoSp);
4787 op_cost(0);
4788 format %{ %}
4789 interface(REG_INTER);
4790 %}
4791
4792 // Pointer 64 bit Register R2 only
4793 operand iRegP_R2()
4794 %{
4795 constraint(ALLOC_IN_RC(r2_reg));
4796 match(RegP);
4797 // match(iRegP);
4798 match(iRegPNoSp);
4799 op_cost(0);
4800 format %{ %}
4801 interface(REG_INTER);
4802 %}
4803
4804 // Pointer 64 bit Register R3 only
4805 operand iRegP_R3()
4806 %{
4807 constraint(ALLOC_IN_RC(r3_reg));
4808 match(RegP);
4809 // match(iRegP);
4810 match(iRegPNoSp);
4811 op_cost(0);
4812 format %{ %}
4813 interface(REG_INTER);
4814 %}
4815
4816 // Pointer 64 bit Register R4 only
4817 operand iRegP_R4()
4818 %{
4819 constraint(ALLOC_IN_RC(r4_reg));
4820 match(RegP);
4821 // match(iRegP);
4822 match(iRegPNoSp);
4823 op_cost(0);
4824 format %{ %}
4825 interface(REG_INTER);
4826 %}
4827
4828 // Pointer 64 bit Register R5 only
4829 operand iRegP_R5()
4830 %{
4831 constraint(ALLOC_IN_RC(r5_reg));
4832 match(RegP);
4833 // match(iRegP);
4834 match(iRegPNoSp);
4835 op_cost(0);
4836 format %{ %}
4837 interface(REG_INTER);
4838 %}
4839
4840 // Pointer 64 bit Register R10 only
4841 operand iRegP_R10()
4842 %{
4843 constraint(ALLOC_IN_RC(r10_reg));
4844 match(RegP);
4845 // match(iRegP);
4846 match(iRegPNoSp);
4847 op_cost(0);
4848 format %{ %}
4849 interface(REG_INTER);
4850 %}
4851
4852 // Long 64 bit Register R0 only
4853 operand iRegL_R0()
4854 %{
4855 constraint(ALLOC_IN_RC(r0_reg));
4856 match(RegL);
4857 match(iRegLNoSp);
4858 op_cost(0);
4859 format %{ %}
4860 interface(REG_INTER);
4861 %}
4862
4863 // Long 64 bit Register R11 only
4864 operand iRegL_R11()
4865 %{
4866 constraint(ALLOC_IN_RC(r11_reg));
4867 match(RegL);
4868 match(iRegLNoSp);
4869 op_cost(0);
4870 format %{ %}
4871 interface(REG_INTER);
4872 %}
4873
4874 // Register R0 only
4875 operand iRegI_R0()
4876 %{
4877 constraint(ALLOC_IN_RC(int_r0_reg));
4878 match(RegI);
4879 match(iRegINoSp);
4880 op_cost(0);
4881 format %{ %}
4882 interface(REG_INTER);
4883 %}
4884
4885 // Register R2 only
4886 operand iRegI_R2()
4887 %{
4888 constraint(ALLOC_IN_RC(int_r2_reg));
4889 match(RegI);
4890 match(iRegINoSp);
4891 op_cost(0);
4892 format %{ %}
4893 interface(REG_INTER);
4894 %}
4895
4896 // Register R3 only
4897 operand iRegI_R3()
4898 %{
4899 constraint(ALLOC_IN_RC(int_r3_reg));
4900 match(RegI);
4901 match(iRegINoSp);
4902 op_cost(0);
4903 format %{ %}
4904 interface(REG_INTER);
4905 %}
4906
4907
4908 // Register R4 only
4909 operand iRegI_R4()
4910 %{
4911 constraint(ALLOC_IN_RC(int_r4_reg));
4912 match(RegI);
4913 match(iRegINoSp);
4914 op_cost(0);
4915 format %{ %}
4916 interface(REG_INTER);
4917 %}
4918
4919
4920 // Pointer Register Operands
4921 // Narrow Pointer Register
4922 operand iRegN()
4923 %{
4924 constraint(ALLOC_IN_RC(any_reg32));
4925 match(RegN);
4926 match(iRegNNoSp);
4927 op_cost(0);
4928 format %{ %}
4929 interface(REG_INTER);
4930 %}
4931
4932 // Integer 64 bit Register not Special
4933 operand iRegNNoSp()
4934 %{
4935 constraint(ALLOC_IN_RC(no_special_reg32));
4936 match(RegN);
4937 op_cost(0);
4938 format %{ %}
4939 interface(REG_INTER);
4940 %}
4941
4942 // Float Register
4943 // Float register operands
4944 operand vRegF()
4945 %{
4946 constraint(ALLOC_IN_RC(float_reg));
4947 match(RegF);
4948
4949 op_cost(0);
4950 format %{ %}
4951 interface(REG_INTER);
4952 %}
4953
4954 // Double Register
4955 // Double register operands
4956 operand vRegD()
4957 %{
4958 constraint(ALLOC_IN_RC(double_reg));
4959 match(RegD);
4960
4961 op_cost(0);
4962 format %{ %}
4963 interface(REG_INTER);
4964 %}
4965
4966 // Generic vector class. This will be used for
4967 // all vector operands, including NEON and SVE.
4968 operand vReg()
4969 %{
4970 constraint(ALLOC_IN_RC(dynamic));
4971 match(VecA);
4972 match(VecD);
4973 match(VecX);
4974
4975 op_cost(0);
4976 format %{ %}
4977 interface(REG_INTER);
4978 %}
4979
4980 operand vecA()
4981 %{
4982 constraint(ALLOC_IN_RC(vectora_reg));
4983 match(VecA);
4984
4985 op_cost(0);
4986 format %{ %}
4987 interface(REG_INTER);
4988 %}
4989
4990 operand vecD()
4991 %{
4992 constraint(ALLOC_IN_RC(vectord_reg));
4993 match(VecD);
4994
4995 op_cost(0);
4996 format %{ %}
4997 interface(REG_INTER);
4998 %}
4999
5000 operand vecX()
5001 %{
5002 constraint(ALLOC_IN_RC(vectorx_reg));
5003 match(VecX);
5004
5005 op_cost(0);
5006 format %{ %}
5007 interface(REG_INTER);
5008 %}
5009
5010 operand vRegD_V0()
5011 %{
5012 constraint(ALLOC_IN_RC(v0_reg));
5013 match(RegD);
5014 op_cost(0);
5015 format %{ %}
5016 interface(REG_INTER);
5017 %}
5018
5019 operand vRegD_V1()
5020 %{
5021 constraint(ALLOC_IN_RC(v1_reg));
5022 match(RegD);
5023 op_cost(0);
5024 format %{ %}
5025 interface(REG_INTER);
5026 %}
5027
5028 operand vRegD_V2()
5029 %{
5030 constraint(ALLOC_IN_RC(v2_reg));
5031 match(RegD);
5032 op_cost(0);
5033 format %{ %}
5034 interface(REG_INTER);
5035 %}
5036
5037 operand vRegD_V3()
5038 %{
5039 constraint(ALLOC_IN_RC(v3_reg));
5040 match(RegD);
5041 op_cost(0);
5042 format %{ %}
5043 interface(REG_INTER);
5044 %}
5045
5046 operand vRegD_V4()
5047 %{
5048 constraint(ALLOC_IN_RC(v4_reg));
5049 match(RegD);
5050 op_cost(0);
5051 format %{ %}
5052 interface(REG_INTER);
5053 %}
5054
5055 operand vRegD_V5()
5056 %{
5057 constraint(ALLOC_IN_RC(v5_reg));
5058 match(RegD);
5059 op_cost(0);
5060 format %{ %}
5061 interface(REG_INTER);
5062 %}
5063
5064 operand vRegD_V6()
5065 %{
5066 constraint(ALLOC_IN_RC(v6_reg));
5067 match(RegD);
5068 op_cost(0);
5069 format %{ %}
5070 interface(REG_INTER);
5071 %}
5072
5073 operand vRegD_V7()
5074 %{
5075 constraint(ALLOC_IN_RC(v7_reg));
5076 match(RegD);
5077 op_cost(0);
5078 format %{ %}
5079 interface(REG_INTER);
5080 %}
5081
5082 operand vRegD_V12()
5083 %{
5084 constraint(ALLOC_IN_RC(v12_reg));
5085 match(RegD);
5086 op_cost(0);
5087 format %{ %}
5088 interface(REG_INTER);
5089 %}
5090
5091 operand vRegD_V13()
5092 %{
5093 constraint(ALLOC_IN_RC(v13_reg));
5094 match(RegD);
5095 op_cost(0);
5096 format %{ %}
5097 interface(REG_INTER);
5098 %}
5099
5100 operand pReg()
5101 %{
5102 constraint(ALLOC_IN_RC(pr_reg));
5103 match(RegVectMask);
5104 match(pRegGov);
5105 op_cost(0);
5106 format %{ %}
5107 interface(REG_INTER);
5108 %}
5109
5110 operand pRegGov()
5111 %{
5112 constraint(ALLOC_IN_RC(gov_pr));
5113 match(RegVectMask);
5114 match(pReg);
5115 op_cost(0);
5116 format %{ %}
5117 interface(REG_INTER);
5118 %}
5119
5120 operand pRegGov_P0()
5121 %{
5122 constraint(ALLOC_IN_RC(p0_reg));
5123 match(RegVectMask);
5124 op_cost(0);
5125 format %{ %}
5126 interface(REG_INTER);
5127 %}
5128
5129 operand pRegGov_P1()
5130 %{
5131 constraint(ALLOC_IN_RC(p1_reg));
5132 match(RegVectMask);
5133 op_cost(0);
5134 format %{ %}
5135 interface(REG_INTER);
5136 %}
5137
5138 // Flags register, used as output of signed compare instructions
5139
5140 // note that on AArch64 we also use this register as the output for
5141 // for floating point compare instructions (CmpF CmpD). this ensures
5142 // that ordered inequality tests use GT, GE, LT or LE none of which
5143 // pass through cases where the result is unordered i.e. one or both
5144 // inputs to the compare is a NaN. this means that the ideal code can
5145 // replace e.g. a GT with an LE and not end up capturing the NaN case
5146 // (where the comparison should always fail). EQ and NE tests are
5147 // always generated in ideal code so that unordered folds into the NE
5148 // case, matching the behaviour of AArch64 NE.
5149 //
5150 // This differs from x86 where the outputs of FP compares use a
5151 // special FP flags registers and where compares based on this
5152 // register are distinguished into ordered inequalities (cmpOpUCF) and
5153 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5154 // to explicitly handle the unordered case in branches. x86 also has
5155 // to include extra CMoveX rules to accept a cmpOpUCF input.
5156
5157 operand rFlagsReg()
5158 %{
5159 constraint(ALLOC_IN_RC(int_flags));
5160 match(RegFlags);
5161
5162 op_cost(0);
5163 format %{ "RFLAGS" %}
5164 interface(REG_INTER);
5165 %}
5166
5167 // Flags register, used as output of unsigned compare instructions
5168 operand rFlagsRegU()
5169 %{
5170 constraint(ALLOC_IN_RC(int_flags));
5171 match(RegFlags);
5172
5173 op_cost(0);
5174 format %{ "RFLAGSU" %}
5175 interface(REG_INTER);
5176 %}
5177
5178 // Special Registers
5179
5180 // Method Register
5181 operand inline_cache_RegP(iRegP reg)
5182 %{
5183 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5184 match(reg);
5185 match(iRegPNoSp);
5186 op_cost(0);
5187 format %{ %}
5188 interface(REG_INTER);
5189 %}
5190
5191 // Thread Register
5192 operand thread_RegP(iRegP reg)
5193 %{
5194 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5195 match(reg);
5196 op_cost(0);
5197 format %{ %}
5198 interface(REG_INTER);
5199 %}
5200
5201 //----------Memory Operands----------------------------------------------------
5202
5203 operand indirect(iRegP reg)
5204 %{
5205 constraint(ALLOC_IN_RC(ptr_reg));
5206 match(reg);
5207 op_cost(0);
5208 format %{ "[$reg]" %}
5209 interface(MEMORY_INTER) %{
5210 base($reg);
5211 index(0xffffffff);
5212 scale(0x0);
5213 disp(0x0);
5214 %}
5215 %}
5216
5217 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5218 %{
5219 constraint(ALLOC_IN_RC(ptr_reg));
5220 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5221 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5222 op_cost(0);
5223 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5224 interface(MEMORY_INTER) %{
5225 base($reg);
5226 index($ireg);
5227 scale($scale);
5228 disp(0x0);
5229 %}
5230 %}
5231
5232 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5233 %{
5234 constraint(ALLOC_IN_RC(ptr_reg));
5235 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5236 match(AddP reg (LShiftL lreg scale));
5237 op_cost(0);
5238 format %{ "$reg, $lreg lsl($scale)" %}
5239 interface(MEMORY_INTER) %{
5240 base($reg);
5241 index($lreg);
5242 scale($scale);
5243 disp(0x0);
5244 %}
5245 %}
5246
5247 operand indIndexI2L(iRegP reg, iRegI ireg)
5248 %{
5249 constraint(ALLOC_IN_RC(ptr_reg));
5250 match(AddP reg (ConvI2L ireg));
5251 op_cost(0);
5252 format %{ "$reg, $ireg, 0, I2L" %}
5253 interface(MEMORY_INTER) %{
5254 base($reg);
5255 index($ireg);
5256 scale(0x0);
5257 disp(0x0);
5258 %}
5259 %}
5260
5261 operand indIndex(iRegP reg, iRegL lreg)
5262 %{
5263 constraint(ALLOC_IN_RC(ptr_reg));
5264 match(AddP reg lreg);
5265 op_cost(0);
5266 format %{ "$reg, $lreg" %}
5267 interface(MEMORY_INTER) %{
5268 base($reg);
5269 index($lreg);
5270 scale(0x0);
5271 disp(0x0);
5272 %}
5273 %}
5274
5275 operand indOffI1(iRegP reg, immIOffset1 off)
5276 %{
5277 constraint(ALLOC_IN_RC(ptr_reg));
5278 match(AddP reg off);
5279 op_cost(0);
5280 format %{ "[$reg, $off]" %}
5281 interface(MEMORY_INTER) %{
5282 base($reg);
5283 index(0xffffffff);
5284 scale(0x0);
5285 disp($off);
5286 %}
5287 %}
5288
5289 operand indOffI2(iRegP reg, immIOffset2 off)
5290 %{
5291 constraint(ALLOC_IN_RC(ptr_reg));
5292 match(AddP reg off);
5293 op_cost(0);
5294 format %{ "[$reg, $off]" %}
5295 interface(MEMORY_INTER) %{
5296 base($reg);
5297 index(0xffffffff);
5298 scale(0x0);
5299 disp($off);
5300 %}
5301 %}
5302
5303 operand indOffI4(iRegP reg, immIOffset4 off)
5304 %{
5305 constraint(ALLOC_IN_RC(ptr_reg));
5306 match(AddP reg off);
5307 op_cost(0);
5308 format %{ "[$reg, $off]" %}
5309 interface(MEMORY_INTER) %{
5310 base($reg);
5311 index(0xffffffff);
5312 scale(0x0);
5313 disp($off);
5314 %}
5315 %}
5316
5317 operand indOffI8(iRegP reg, immIOffset8 off)
5318 %{
5319 constraint(ALLOC_IN_RC(ptr_reg));
5320 match(AddP reg off);
5321 op_cost(0);
5322 format %{ "[$reg, $off]" %}
5323 interface(MEMORY_INTER) %{
5324 base($reg);
5325 index(0xffffffff);
5326 scale(0x0);
5327 disp($off);
5328 %}
5329 %}
5330
5331 operand indOffI16(iRegP reg, immIOffset16 off)
5332 %{
5333 constraint(ALLOC_IN_RC(ptr_reg));
5334 match(AddP reg off);
5335 op_cost(0);
5336 format %{ "[$reg, $off]" %}
5337 interface(MEMORY_INTER) %{
5338 base($reg);
5339 index(0xffffffff);
5340 scale(0x0);
5341 disp($off);
5342 %}
5343 %}
5344
5345 operand indOffL1(iRegP reg, immLoffset1 off)
5346 %{
5347 constraint(ALLOC_IN_RC(ptr_reg));
5348 match(AddP reg off);
5349 op_cost(0);
5350 format %{ "[$reg, $off]" %}
5351 interface(MEMORY_INTER) %{
5352 base($reg);
5353 index(0xffffffff);
5354 scale(0x0);
5355 disp($off);
5356 %}
5357 %}
5358
5359 operand indOffL2(iRegP reg, immLoffset2 off)
5360 %{
5361 constraint(ALLOC_IN_RC(ptr_reg));
5362 match(AddP reg off);
5363 op_cost(0);
5364 format %{ "[$reg, $off]" %}
5365 interface(MEMORY_INTER) %{
5366 base($reg);
5367 index(0xffffffff);
5368 scale(0x0);
5369 disp($off);
5370 %}
5371 %}
5372
5373 operand indOffL4(iRegP reg, immLoffset4 off)
5374 %{
5375 constraint(ALLOC_IN_RC(ptr_reg));
5376 match(AddP reg off);
5377 op_cost(0);
5378 format %{ "[$reg, $off]" %}
5379 interface(MEMORY_INTER) %{
5380 base($reg);
5381 index(0xffffffff);
5382 scale(0x0);
5383 disp($off);
5384 %}
5385 %}
5386
5387 operand indOffL8(iRegP reg, immLoffset8 off)
5388 %{
5389 constraint(ALLOC_IN_RC(ptr_reg));
5390 match(AddP reg off);
5391 op_cost(0);
5392 format %{ "[$reg, $off]" %}
5393 interface(MEMORY_INTER) %{
5394 base($reg);
5395 index(0xffffffff);
5396 scale(0x0);
5397 disp($off);
5398 %}
5399 %}
5400
5401 operand indOffL16(iRegP reg, immLoffset16 off)
5402 %{
5403 constraint(ALLOC_IN_RC(ptr_reg));
5404 match(AddP reg off);
5405 op_cost(0);
5406 format %{ "[$reg, $off]" %}
5407 interface(MEMORY_INTER) %{
5408 base($reg);
5409 index(0xffffffff);
5410 scale(0x0);
5411 disp($off);
5412 %}
5413 %}
5414
5415 operand indirectX2P(iRegL reg)
5416 %{
5417 constraint(ALLOC_IN_RC(ptr_reg));
5418 match(CastX2P reg);
5419 op_cost(0);
5420 format %{ "[$reg]\t# long -> ptr" %}
5421 interface(MEMORY_INTER) %{
5422 base($reg);
5423 index(0xffffffff);
5424 scale(0x0);
5425 disp(0x0);
5426 %}
5427 %}
5428
5429 operand indOffX2P(iRegL reg, immLOffset off)
5430 %{
5431 constraint(ALLOC_IN_RC(ptr_reg));
5432 match(AddP (CastX2P reg) off);
5433 op_cost(0);
5434 format %{ "[$reg, $off]\t# long -> ptr" %}
5435 interface(MEMORY_INTER) %{
5436 base($reg);
5437 index(0xffffffff);
5438 scale(0x0);
5439 disp($off);
5440 %}
5441 %}
5442
5443 operand indirectN(iRegN reg)
5444 %{
5445 predicate(CompressedOops::shift() == 0);
5446 constraint(ALLOC_IN_RC(ptr_reg));
5447 match(DecodeN reg);
5448 op_cost(0);
5449 format %{ "[$reg]\t# narrow" %}
5450 interface(MEMORY_INTER) %{
5451 base($reg);
5452 index(0xffffffff);
5453 scale(0x0);
5454 disp(0x0);
5455 %}
5456 %}
5457
5458 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5459 %{
5460 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5461 constraint(ALLOC_IN_RC(ptr_reg));
5462 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5463 op_cost(0);
5464 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5465 interface(MEMORY_INTER) %{
5466 base($reg);
5467 index($ireg);
5468 scale($scale);
5469 disp(0x0);
5470 %}
5471 %}
5472
5473 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5474 %{
5475 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5476 constraint(ALLOC_IN_RC(ptr_reg));
5477 match(AddP (DecodeN reg) (LShiftL lreg scale));
5478 op_cost(0);
5479 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5480 interface(MEMORY_INTER) %{
5481 base($reg);
5482 index($lreg);
5483 scale($scale);
5484 disp(0x0);
5485 %}
5486 %}
5487
5488 operand indIndexI2LN(iRegN reg, iRegI ireg)
5489 %{
5490 predicate(CompressedOops::shift() == 0);
5491 constraint(ALLOC_IN_RC(ptr_reg));
5492 match(AddP (DecodeN reg) (ConvI2L ireg));
5493 op_cost(0);
5494 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5495 interface(MEMORY_INTER) %{
5496 base($reg);
5497 index($ireg);
5498 scale(0x0);
5499 disp(0x0);
5500 %}
5501 %}
5502
5503 operand indIndexN(iRegN reg, iRegL lreg)
5504 %{
5505 predicate(CompressedOops::shift() == 0);
5506 constraint(ALLOC_IN_RC(ptr_reg));
5507 match(AddP (DecodeN reg) lreg);
5508 op_cost(0);
5509 format %{ "$reg, $lreg\t# narrow" %}
5510 interface(MEMORY_INTER) %{
5511 base($reg);
5512 index($lreg);
5513 scale(0x0);
5514 disp(0x0);
5515 %}
5516 %}
5517
5518 operand indOffIN(iRegN reg, immIOffset off)
5519 %{
5520 predicate(CompressedOops::shift() == 0);
5521 constraint(ALLOC_IN_RC(ptr_reg));
5522 match(AddP (DecodeN reg) off);
5523 op_cost(0);
5524 format %{ "[$reg, $off]\t# narrow" %}
5525 interface(MEMORY_INTER) %{
5526 base($reg);
5527 index(0xffffffff);
5528 scale(0x0);
5529 disp($off);
5530 %}
5531 %}
5532
5533 operand indOffLN(iRegN reg, immLOffset off)
5534 %{
5535 predicate(CompressedOops::shift() == 0);
5536 constraint(ALLOC_IN_RC(ptr_reg));
5537 match(AddP (DecodeN reg) off);
5538 op_cost(0);
5539 format %{ "[$reg, $off]\t# narrow" %}
5540 interface(MEMORY_INTER) %{
5541 base($reg);
5542 index(0xffffffff);
5543 scale(0x0);
5544 disp($off);
5545 %}
5546 %}
5547
5548
5549 //----------Special Memory Operands--------------------------------------------
5550 // Stack Slot Operand - This operand is used for loading and storing temporary
5551 // values on the stack where a match requires a value to
5552 // flow through memory.
5553 operand stackSlotP(sRegP reg)
5554 %{
5555 constraint(ALLOC_IN_RC(stack_slots));
5556 op_cost(100);
5557 // No match rule because this operand is only generated in matching
5558 // match(RegP);
5559 format %{ "[$reg]" %}
5560 interface(MEMORY_INTER) %{
5561 base(0x1e); // RSP
5562 index(0x0); // No Index
5563 scale(0x0); // No Scale
5564 disp($reg); // Stack Offset
5565 %}
5566 %}
5567
5568 operand stackSlotI(sRegI reg)
5569 %{
5570 constraint(ALLOC_IN_RC(stack_slots));
5571 // No match rule because this operand is only generated in matching
5572 // match(RegI);
5573 format %{ "[$reg]" %}
5574 interface(MEMORY_INTER) %{
5575 base(0x1e); // RSP
5576 index(0x0); // No Index
5577 scale(0x0); // No Scale
5578 disp($reg); // Stack Offset
5579 %}
5580 %}
5581
5582 operand stackSlotF(sRegF reg)
5583 %{
5584 constraint(ALLOC_IN_RC(stack_slots));
5585 // No match rule because this operand is only generated in matching
5586 // match(RegF);
5587 format %{ "[$reg]" %}
5588 interface(MEMORY_INTER) %{
5589 base(0x1e); // RSP
5590 index(0x0); // No Index
5591 scale(0x0); // No Scale
5592 disp($reg); // Stack Offset
5593 %}
5594 %}
5595
5596 operand stackSlotD(sRegD reg)
5597 %{
5598 constraint(ALLOC_IN_RC(stack_slots));
5599 // No match rule because this operand is only generated in matching
5600 // match(RegD);
5601 format %{ "[$reg]" %}
5602 interface(MEMORY_INTER) %{
5603 base(0x1e); // RSP
5604 index(0x0); // No Index
5605 scale(0x0); // No Scale
5606 disp($reg); // Stack Offset
5607 %}
5608 %}
5609
5610 operand stackSlotL(sRegL reg)
5611 %{
5612 constraint(ALLOC_IN_RC(stack_slots));
5613 // No match rule because this operand is only generated in matching
5614 // match(RegL);
5615 format %{ "[$reg]" %}
5616 interface(MEMORY_INTER) %{
5617 base(0x1e); // RSP
5618 index(0x0); // No Index
5619 scale(0x0); // No Scale
5620 disp($reg); // Stack Offset
5621 %}
5622 %}
5623
5624 // Operands for expressing Control Flow
5625 // NOTE: Label is a predefined operand which should not be redefined in
5626 // the AD file. It is generically handled within the ADLC.
5627
5628 //----------Conditional Branch Operands----------------------------------------
5629 // Comparison Op - This is the operation of the comparison, and is limited to
5630 // the following set of codes:
5631 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5632 //
5633 // Other attributes of the comparison, such as unsignedness, are specified
5634 // by the comparison instruction that sets a condition code flags register.
5635 // That result is represented by a flags operand whose subtype is appropriate
5636 // to the unsignedness (etc.) of the comparison.
5637 //
5638 // Later, the instruction which matches both the Comparison Op (a Bool) and
5639 // the flags (produced by the Cmp) specifies the coding of the comparison op
5640 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5641
5642 // used for signed integral comparisons and fp comparisons
5643
5644 operand cmpOp()
5645 %{
5646 match(Bool);
5647
5648 format %{ "" %}
5649 interface(COND_INTER) %{
5650 equal(0x0, "eq");
5651 not_equal(0x1, "ne");
5652 less(0xb, "lt");
5653 greater_equal(0xa, "ge");
5654 less_equal(0xd, "le");
5655 greater(0xc, "gt");
5656 overflow(0x6, "vs");
5657 no_overflow(0x7, "vc");
5658 %}
5659 %}
5660
5661 // used for unsigned integral comparisons
5662
5663 operand cmpOpU()
5664 %{
5665 match(Bool);
5666
5667 format %{ "" %}
5668 interface(COND_INTER) %{
5669 equal(0x0, "eq");
5670 not_equal(0x1, "ne");
5671 less(0x3, "lo");
5672 greater_equal(0x2, "hs");
5673 less_equal(0x9, "ls");
5674 greater(0x8, "hi");
5675 overflow(0x6, "vs");
5676 no_overflow(0x7, "vc");
5677 %}
5678 %}
5679
5680 // used for certain integral comparisons which can be
5681 // converted to cbxx or tbxx instructions
5682
5683 operand cmpOpEqNe()
5684 %{
5685 match(Bool);
5686 op_cost(0);
5687 predicate(n->as_Bool()->_test._test == BoolTest::ne
5688 || n->as_Bool()->_test._test == BoolTest::eq);
5689
5690 format %{ "" %}
5691 interface(COND_INTER) %{
5692 equal(0x0, "eq");
5693 not_equal(0x1, "ne");
5694 less(0xb, "lt");
5695 greater_equal(0xa, "ge");
5696 less_equal(0xd, "le");
5697 greater(0xc, "gt");
5698 overflow(0x6, "vs");
5699 no_overflow(0x7, "vc");
5700 %}
5701 %}
5702
5703 // used for certain integral comparisons which can be
5704 // converted to cbxx or tbxx instructions
5705
5706 operand cmpOpLtGe()
5707 %{
5708 match(Bool);
5709 op_cost(0);
5710
5711 predicate(n->as_Bool()->_test._test == BoolTest::lt
5712 || n->as_Bool()->_test._test == BoolTest::ge);
5713
5714 format %{ "" %}
5715 interface(COND_INTER) %{
5716 equal(0x0, "eq");
5717 not_equal(0x1, "ne");
5718 less(0xb, "lt");
5719 greater_equal(0xa, "ge");
5720 less_equal(0xd, "le");
5721 greater(0xc, "gt");
5722 overflow(0x6, "vs");
5723 no_overflow(0x7, "vc");
5724 %}
5725 %}
5726
5727 // used for certain unsigned integral comparisons which can be
5728 // converted to cbxx or tbxx instructions
5729
5730 operand cmpOpUEqNeLeGt()
5731 %{
5732 match(Bool);
5733 op_cost(0);
5734
5735 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5736 n->as_Bool()->_test._test == BoolTest::ne ||
5737 n->as_Bool()->_test._test == BoolTest::le ||
5738 n->as_Bool()->_test._test == BoolTest::gt);
5739
5740 format %{ "" %}
5741 interface(COND_INTER) %{
5742 equal(0x0, "eq");
5743 not_equal(0x1, "ne");
5744 less(0x3, "lo");
5745 greater_equal(0x2, "hs");
5746 less_equal(0x9, "ls");
5747 greater(0x8, "hi");
5748 overflow(0x6, "vs");
5749 no_overflow(0x7, "vc");
5750 %}
5751 %}
5752
5753 // Special operand allowing long args to int ops to be truncated for free
5754
5755 operand iRegL2I(iRegL reg) %{
5756
5757 op_cost(0);
5758
5759 match(ConvL2I reg);
5760
5761 format %{ "l2i($reg)" %}
5762
5763 interface(REG_INTER)
5764 %}
5765
5766 operand iRegL2P(iRegL reg) %{
5767
5768 op_cost(0);
5769
5770 match(CastX2P reg);
5771
5772 format %{ "l2p($reg)" %}
5773
5774 interface(REG_INTER)
5775 %}
5776
5777 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5778 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5779 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5780 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5781
5782 //----------OPERAND CLASSES----------------------------------------------------
5783 // Operand Classes are groups of operands that are used as to simplify
5784 // instruction definitions by not requiring the AD writer to specify
5785 // separate instructions for every form of operand when the
5786 // instruction accepts multiple operand types with the same basic
5787 // encoding and format. The classic case of this is memory operands.
5788
5789 // memory is used to define read/write location for load/store
5790 // instruction defs. we can turn a memory op into an Address
5791
5792 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5793 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5794
5795 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5796 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5797
5798 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5799 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5800
5801 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5802 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5803
5804 // All of the memory operands. For the pipeline description.
5805 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5806 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5807 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5808
5809
5810 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5811 // operations. it allows the src to be either an iRegI or a (ConvL2I
5812 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5813 // can be elided because the 32-bit instruction will just employ the
5814 // lower 32 bits anyway.
5815 //
5816 // n.b. this does not elide all L2I conversions. if the truncated
5817 // value is consumed by more than one operation then the ConvL2I
5818 // cannot be bundled into the consuming nodes so an l2i gets planted
5819 // (actually a movw $dst $src) and the downstream instructions consume
5820 // the result of the l2i as an iRegI input. That's a shame since the
5821 // movw is actually redundant but its not too costly.
5822
5823 opclass iRegIorL2I(iRegI, iRegL2I);
5824 opclass iRegPorL2P(iRegP, iRegL2P);
5825
5826 //----------PIPELINE-----------------------------------------------------------
5827 // Rules which define the behavior of the target architectures pipeline.
5828
5829 // For specific pipelines, eg A53, define the stages of that pipeline
5830 //pipe_desc(ISS, EX1, EX2, WR);
5831 #define ISS S0
5832 #define EX1 S1
5833 #define EX2 S2
5834 #define WR S3
5835
5836 // Integer ALU reg operation
5837 pipeline %{
5838
5839 attributes %{
5840 // ARM instructions are of fixed length
5841 fixed_size_instructions; // Fixed size instructions TODO does
5842 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5843 // ARM instructions come in 32-bit word units
5844 instruction_unit_size = 4; // An instruction is 4 bytes long
5845 instruction_fetch_unit_size = 64; // The processor fetches one line
5846 instruction_fetch_units = 1; // of 64 bytes
5847
5848 // List of nop instructions
5849 nops( MachNop );
5850 %}
5851
5852 // We don't use an actual pipeline model so don't care about resources
5853 // or description. we do use pipeline classes to introduce fixed
5854 // latencies
5855
5856 //----------RESOURCES----------------------------------------------------------
5857 // Resources are the functional units available to the machine
5858
5859 resources( INS0, INS1, INS01 = INS0 | INS1,
5860 ALU0, ALU1, ALU = ALU0 | ALU1,
5861 MAC,
5862 DIV,
5863 BRANCH,
5864 LDST,
5865 NEON_FP);
5866
5867 //----------PIPELINE DESCRIPTION-----------------------------------------------
5868 // Pipeline Description specifies the stages in the machine's pipeline
5869
5870 // Define the pipeline as a generic 6 stage pipeline
5871 pipe_desc(S0, S1, S2, S3, S4, S5);
5872
5873 //----------PIPELINE CLASSES---------------------------------------------------
5874 // Pipeline Classes describe the stages in which input and output are
5875 // referenced by the hardware pipeline.
5876
5877 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5878 %{
5879 single_instruction;
5880 src1 : S1(read);
5881 src2 : S2(read);
5882 dst : S5(write);
5883 INS01 : ISS;
5884 NEON_FP : S5;
5885 %}
5886
5887 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5888 %{
5889 single_instruction;
5890 src1 : S1(read);
5891 src2 : S2(read);
5892 dst : S5(write);
5893 INS01 : ISS;
5894 NEON_FP : S5;
5895 %}
5896
5897 pipe_class fp_uop_s(vRegF dst, vRegF src)
5898 %{
5899 single_instruction;
5900 src : S1(read);
5901 dst : S5(write);
5902 INS01 : ISS;
5903 NEON_FP : S5;
5904 %}
5905
5906 pipe_class fp_uop_d(vRegD dst, vRegD src)
5907 %{
5908 single_instruction;
5909 src : S1(read);
5910 dst : S5(write);
5911 INS01 : ISS;
5912 NEON_FP : S5;
5913 %}
5914
5915 pipe_class fp_d2f(vRegF dst, vRegD src)
5916 %{
5917 single_instruction;
5918 src : S1(read);
5919 dst : S5(write);
5920 INS01 : ISS;
5921 NEON_FP : S5;
5922 %}
5923
5924 pipe_class fp_f2d(vRegD dst, vRegF src)
5925 %{
5926 single_instruction;
5927 src : S1(read);
5928 dst : S5(write);
5929 INS01 : ISS;
5930 NEON_FP : S5;
5931 %}
5932
5933 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5934 %{
5935 single_instruction;
5936 src : S1(read);
5937 dst : S5(write);
5938 INS01 : ISS;
5939 NEON_FP : S5;
5940 %}
5941
5942 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5943 %{
5944 single_instruction;
5945 src : S1(read);
5946 dst : S5(write);
5947 INS01 : ISS;
5948 NEON_FP : S5;
5949 %}
5950
5951 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5952 %{
5953 single_instruction;
5954 src : S1(read);
5955 dst : S5(write);
5956 INS01 : ISS;
5957 NEON_FP : S5;
5958 %}
5959
5960 pipe_class fp_l2f(vRegF dst, iRegL src)
5961 %{
5962 single_instruction;
5963 src : S1(read);
5964 dst : S5(write);
5965 INS01 : ISS;
5966 NEON_FP : S5;
5967 %}
5968
5969 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5970 %{
5971 single_instruction;
5972 src : S1(read);
5973 dst : S5(write);
5974 INS01 : ISS;
5975 NEON_FP : S5;
5976 %}
5977
5978 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5979 %{
5980 single_instruction;
5981 src : S1(read);
5982 dst : S5(write);
5983 INS01 : ISS;
5984 NEON_FP : S5;
5985 %}
5986
5987 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5988 %{
5989 single_instruction;
5990 src : S1(read);
5991 dst : S5(write);
5992 INS01 : ISS;
5993 NEON_FP : S5;
5994 %}
5995
5996 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5997 %{
5998 single_instruction;
5999 src : S1(read);
6000 dst : S5(write);
6001 INS01 : ISS;
6002 NEON_FP : S5;
6003 %}
6004
6005 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6006 %{
6007 single_instruction;
6008 src1 : S1(read);
6009 src2 : S2(read);
6010 dst : S5(write);
6011 INS0 : ISS;
6012 NEON_FP : S5;
6013 %}
6014
6015 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6016 %{
6017 single_instruction;
6018 src1 : S1(read);
6019 src2 : S2(read);
6020 dst : S5(write);
6021 INS0 : ISS;
6022 NEON_FP : S5;
6023 %}
6024
6025 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6026 %{
6027 single_instruction;
6028 cr : S1(read);
6029 src1 : S1(read);
6030 src2 : S1(read);
6031 dst : S3(write);
6032 INS01 : ISS;
6033 NEON_FP : S3;
6034 %}
6035
6036 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6037 %{
6038 single_instruction;
6039 cr : S1(read);
6040 src1 : S1(read);
6041 src2 : S1(read);
6042 dst : S3(write);
6043 INS01 : ISS;
6044 NEON_FP : S3;
6045 %}
6046
6047 pipe_class fp_imm_s(vRegF dst)
6048 %{
6049 single_instruction;
6050 dst : S3(write);
6051 INS01 : ISS;
6052 NEON_FP : S3;
6053 %}
6054
6055 pipe_class fp_imm_d(vRegD dst)
6056 %{
6057 single_instruction;
6058 dst : S3(write);
6059 INS01 : ISS;
6060 NEON_FP : S3;
6061 %}
6062
6063 pipe_class fp_load_constant_s(vRegF dst)
6064 %{
6065 single_instruction;
6066 dst : S4(write);
6067 INS01 : ISS;
6068 NEON_FP : S4;
6069 %}
6070
6071 pipe_class fp_load_constant_d(vRegD dst)
6072 %{
6073 single_instruction;
6074 dst : S4(write);
6075 INS01 : ISS;
6076 NEON_FP : S4;
6077 %}
6078
6079 //------- Integer ALU operations --------------------------
6080
6081 // Integer ALU reg-reg operation
6082 // Operands needed in EX1, result generated in EX2
6083 // Eg. ADD x0, x1, x2
6084 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6085 %{
6086 single_instruction;
6087 dst : EX2(write);
6088 src1 : EX1(read);
6089 src2 : EX1(read);
6090 INS01 : ISS; // Dual issue as instruction 0 or 1
6091 ALU : EX2;
6092 %}
6093
6094 // Integer ALU reg-reg operation with constant shift
6095 // Shifted register must be available in LATE_ISS instead of EX1
6096 // Eg. ADD x0, x1, x2, LSL #2
6097 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6098 %{
6099 single_instruction;
6100 dst : EX2(write);
6101 src1 : EX1(read);
6102 src2 : ISS(read);
6103 INS01 : ISS;
6104 ALU : EX2;
6105 %}
6106
6107 // Integer ALU reg operation with constant shift
6108 // Eg. LSL x0, x1, #shift
6109 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6110 %{
6111 single_instruction;
6112 dst : EX2(write);
6113 src1 : ISS(read);
6114 INS01 : ISS;
6115 ALU : EX2;
6116 %}
6117
6118 // Integer ALU reg-reg operation with variable shift
6119 // Both operands must be available in LATE_ISS instead of EX1
6120 // Result is available in EX1 instead of EX2
6121 // Eg. LSLV x0, x1, x2
6122 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6123 %{
6124 single_instruction;
6125 dst : EX1(write);
6126 src1 : ISS(read);
6127 src2 : ISS(read);
6128 INS01 : ISS;
6129 ALU : EX1;
6130 %}
6131
6132 // Integer ALU reg-reg operation with extract
6133 // As for _vshift above, but result generated in EX2
6134 // Eg. EXTR x0, x1, x2, #N
6135 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6136 %{
6137 single_instruction;
6138 dst : EX2(write);
6139 src1 : ISS(read);
6140 src2 : ISS(read);
6141 INS1 : ISS; // Can only dual issue as Instruction 1
6142 ALU : EX1;
6143 %}
6144
6145 // Integer ALU reg operation
6146 // Eg. NEG x0, x1
6147 pipe_class ialu_reg(iRegI dst, iRegI src)
6148 %{
6149 single_instruction;
6150 dst : EX2(write);
6151 src : EX1(read);
6152 INS01 : ISS;
6153 ALU : EX2;
6154 %}
6155
6156 // Integer ALU reg mmediate operation
6157 // Eg. ADD x0, x1, #N
6158 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6159 %{
6160 single_instruction;
6161 dst : EX2(write);
6162 src1 : EX1(read);
6163 INS01 : ISS;
6164 ALU : EX2;
6165 %}
6166
6167 // Integer ALU immediate operation (no source operands)
6168 // Eg. MOV x0, #N
6169 pipe_class ialu_imm(iRegI dst)
6170 %{
6171 single_instruction;
6172 dst : EX1(write);
6173 INS01 : ISS;
6174 ALU : EX1;
6175 %}
6176
6177 //------- Compare operation -------------------------------
6178
6179 // Compare reg-reg
6180 // Eg. CMP x0, x1
6181 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6182 %{
6183 single_instruction;
6184 // fixed_latency(16);
6185 cr : EX2(write);
6186 op1 : EX1(read);
6187 op2 : EX1(read);
6188 INS01 : ISS;
6189 ALU : EX2;
6190 %}
6191
6192 // Compare reg-reg
6193 // Eg. CMP x0, #N
6194 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6195 %{
6196 single_instruction;
6197 // fixed_latency(16);
6198 cr : EX2(write);
6199 op1 : EX1(read);
6200 INS01 : ISS;
6201 ALU : EX2;
6202 %}
6203
6204 //------- Conditional instructions ------------------------
6205
6206 // Conditional no operands
6207 // Eg. CSINC x0, zr, zr, <cond>
6208 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6209 %{
6210 single_instruction;
6211 cr : EX1(read);
6212 dst : EX2(write);
6213 INS01 : ISS;
6214 ALU : EX2;
6215 %}
6216
6217 // Conditional 2 operand
6218 // EG. CSEL X0, X1, X2, <cond>
6219 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6220 %{
6221 single_instruction;
6222 cr : EX1(read);
6223 src1 : EX1(read);
6224 src2 : EX1(read);
6225 dst : EX2(write);
6226 INS01 : ISS;
6227 ALU : EX2;
6228 %}
6229
6230 // Conditional 2 operand
6231 // EG. CSEL X0, X1, X2, <cond>
6232 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6233 %{
6234 single_instruction;
6235 cr : EX1(read);
6236 src : EX1(read);
6237 dst : EX2(write);
6238 INS01 : ISS;
6239 ALU : EX2;
6240 %}
6241
6242 //------- Multiply pipeline operations --------------------
6243
6244 // Multiply reg-reg
6245 // Eg. MUL w0, w1, w2
6246 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6247 %{
6248 single_instruction;
6249 dst : WR(write);
6250 src1 : ISS(read);
6251 src2 : ISS(read);
6252 INS01 : ISS;
6253 MAC : WR;
6254 %}
6255
6256 // Multiply accumulate
6257 // Eg. MADD w0, w1, w2, w3
6258 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6259 %{
6260 single_instruction;
6261 dst : WR(write);
6262 src1 : ISS(read);
6263 src2 : ISS(read);
6264 src3 : ISS(read);
6265 INS01 : ISS;
6266 MAC : WR;
6267 %}
6268
6269 // Eg. MUL w0, w1, w2
6270 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6271 %{
6272 single_instruction;
6273 fixed_latency(3); // Maximum latency for 64 bit mul
6274 dst : WR(write);
6275 src1 : ISS(read);
6276 src2 : ISS(read);
6277 INS01 : ISS;
6278 MAC : WR;
6279 %}
6280
6281 // Multiply accumulate
6282 // Eg. MADD w0, w1, w2, w3
6283 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6284 %{
6285 single_instruction;
6286 fixed_latency(3); // Maximum latency for 64 bit mul
6287 dst : WR(write);
6288 src1 : ISS(read);
6289 src2 : ISS(read);
6290 src3 : ISS(read);
6291 INS01 : ISS;
6292 MAC : WR;
6293 %}
6294
6295 //------- Divide pipeline operations --------------------
6296
6297 // Eg. SDIV w0, w1, w2
6298 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6299 %{
6300 single_instruction;
6301 fixed_latency(8); // Maximum latency for 32 bit divide
6302 dst : WR(write);
6303 src1 : ISS(read);
6304 src2 : ISS(read);
6305 INS0 : ISS; // Can only dual issue as instruction 0
6306 DIV : WR;
6307 %}
6308
6309 // Eg. SDIV x0, x1, x2
6310 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6311 %{
6312 single_instruction;
6313 fixed_latency(16); // Maximum latency for 64 bit divide
6314 dst : WR(write);
6315 src1 : ISS(read);
6316 src2 : ISS(read);
6317 INS0 : ISS; // Can only dual issue as instruction 0
6318 DIV : WR;
6319 %}
6320
6321 //------- Load pipeline operations ------------------------
6322
6323 // Load - prefetch
6324 // Eg. PFRM <mem>
6325 pipe_class iload_prefetch(memory mem)
6326 %{
6327 single_instruction;
6328 mem : ISS(read);
6329 INS01 : ISS;
6330 LDST : WR;
6331 %}
6332
6333 // Load - reg, mem
6334 // Eg. LDR x0, <mem>
6335 pipe_class iload_reg_mem(iRegI dst, memory mem)
6336 %{
6337 single_instruction;
6338 dst : WR(write);
6339 mem : ISS(read);
6340 INS01 : ISS;
6341 LDST : WR;
6342 %}
6343
6344 // Load - reg, reg
6345 // Eg. LDR x0, [sp, x1]
6346 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6347 %{
6348 single_instruction;
6349 dst : WR(write);
6350 src : ISS(read);
6351 INS01 : ISS;
6352 LDST : WR;
6353 %}
6354
6355 //------- Store pipeline operations -----------------------
6356
6357 // Store - zr, mem
6358 // Eg. STR zr, <mem>
6359 pipe_class istore_mem(memory mem)
6360 %{
6361 single_instruction;
6362 mem : ISS(read);
6363 INS01 : ISS;
6364 LDST : WR;
6365 %}
6366
6367 // Store - reg, mem
6368 // Eg. STR x0, <mem>
6369 pipe_class istore_reg_mem(iRegI src, memory mem)
6370 %{
6371 single_instruction;
6372 mem : ISS(read);
6373 src : EX2(read);
6374 INS01 : ISS;
6375 LDST : WR;
6376 %}
6377
6378 // Store - reg, reg
6379 // Eg. STR x0, [sp, x1]
6380 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6381 %{
6382 single_instruction;
6383 dst : ISS(read);
6384 src : EX2(read);
6385 INS01 : ISS;
6386 LDST : WR;
6387 %}
6388
6389 //------- Store pipeline operations -----------------------
6390
6391 // Branch
6392 pipe_class pipe_branch()
6393 %{
6394 single_instruction;
6395 INS01 : ISS;
6396 BRANCH : EX1;
6397 %}
6398
6399 // Conditional branch
6400 pipe_class pipe_branch_cond(rFlagsReg cr)
6401 %{
6402 single_instruction;
6403 cr : EX1(read);
6404 INS01 : ISS;
6405 BRANCH : EX1;
6406 %}
6407
6408 // Compare & Branch
6409 // EG. CBZ/CBNZ
6410 pipe_class pipe_cmp_branch(iRegI op1)
6411 %{
6412 single_instruction;
6413 op1 : EX1(read);
6414 INS01 : ISS;
6415 BRANCH : EX1;
6416 %}
6417
6418 //------- Synchronisation operations ----------------------
6419
6420 // Any operation requiring serialization.
6421 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6422 pipe_class pipe_serial()
6423 %{
6424 single_instruction;
6425 force_serialization;
6426 fixed_latency(16);
6427 INS01 : ISS(2); // Cannot dual issue with any other instruction
6428 LDST : WR;
6429 %}
6430
6431 // Generic big/slow expanded idiom - also serialized
6432 pipe_class pipe_slow()
6433 %{
6434 instruction_count(10);
6435 multiple_bundles;
6436 force_serialization;
6437 fixed_latency(16);
6438 INS01 : ISS(2); // Cannot dual issue with any other instruction
6439 LDST : WR;
6440 %}
6441
6442 // Empty pipeline class
6443 pipe_class pipe_class_empty()
6444 %{
6445 single_instruction;
6446 fixed_latency(0);
6447 %}
6448
6449 // Default pipeline class.
6450 pipe_class pipe_class_default()
6451 %{
6452 single_instruction;
6453 fixed_latency(2);
6454 %}
6455
6456 // Pipeline class for compares.
6457 pipe_class pipe_class_compare()
6458 %{
6459 single_instruction;
6460 fixed_latency(16);
6461 %}
6462
6463 // Pipeline class for memory operations.
6464 pipe_class pipe_class_memory()
6465 %{
6466 single_instruction;
6467 fixed_latency(16);
6468 %}
6469
6470 // Pipeline class for call.
6471 pipe_class pipe_class_call()
6472 %{
6473 single_instruction;
6474 fixed_latency(100);
6475 %}
6476
6477 // Define the class for the Nop node.
6478 define %{
6479 MachNop = pipe_class_empty;
6480 %}
6481
6482 %}
6483 //----------INSTRUCTIONS-------------------------------------------------------
6484 //
6485 // match -- States which machine-independent subtree may be replaced
6486 // by this instruction.
6487 // ins_cost -- The estimated cost of this instruction is used by instruction
6488 // selection to identify a minimum cost tree of machine
6489 // instructions that matches a tree of machine-independent
6490 // instructions.
6491 // format -- A string providing the disassembly for this instruction.
6492 // The value of an instruction's operand may be inserted
6493 // by referring to it with a '$' prefix.
6494 // opcode -- Three instruction opcodes may be provided. These are referred
6495 // to within an encode class as $primary, $secondary, and $tertiary
6496 // rrspectively. The primary opcode is commonly used to
6497 // indicate the type of machine instruction, while secondary
6498 // and tertiary are often used for prefix options or addressing
6499 // modes.
6500 // ins_encode -- A list of encode classes with parameters. The encode class
6501 // name must have been defined in an 'enc_class' specification
6502 // in the encode section of the architecture description.
6503
6504 // ============================================================================
6505 // Memory (Load/Store) Instructions
6506
6507 // Load Instructions
6508
6509 // Load Byte (8 bit signed)
6510 instruct loadB(iRegINoSp dst, memory1 mem)
6511 %{
6512 match(Set dst (LoadB mem));
6513 predicate(!needs_acquiring_load(n));
6514
6515 ins_cost(4 * INSN_COST);
6516 format %{ "ldrsbw $dst, $mem\t# byte" %}
6517
6518 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6519
6520 ins_pipe(iload_reg_mem);
6521 %}
6522
6523 // Load Byte (8 bit signed) into long
6524 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6525 %{
6526 match(Set dst (ConvI2L (LoadB mem)));
6527 predicate(!needs_acquiring_load(n->in(1)));
6528
6529 ins_cost(4 * INSN_COST);
6530 format %{ "ldrsb $dst, $mem\t# byte" %}
6531
6532 ins_encode(aarch64_enc_ldrsb(dst, mem));
6533
6534 ins_pipe(iload_reg_mem);
6535 %}
6536
6537 // Load Byte (8 bit unsigned)
6538 instruct loadUB(iRegINoSp dst, memory1 mem)
6539 %{
6540 match(Set dst (LoadUB mem));
6541 predicate(!needs_acquiring_load(n));
6542
6543 ins_cost(4 * INSN_COST);
6544 format %{ "ldrbw $dst, $mem\t# byte" %}
6545
6546 ins_encode(aarch64_enc_ldrb(dst, mem));
6547
6548 ins_pipe(iload_reg_mem);
6549 %}
6550
6551 // Load Byte (8 bit unsigned) into long
6552 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6553 %{
6554 match(Set dst (ConvI2L (LoadUB mem)));
6555 predicate(!needs_acquiring_load(n->in(1)));
6556
6557 ins_cost(4 * INSN_COST);
6558 format %{ "ldrb $dst, $mem\t# byte" %}
6559
6560 ins_encode(aarch64_enc_ldrb(dst, mem));
6561
6562 ins_pipe(iload_reg_mem);
6563 %}
6564
6565 // Load Short (16 bit signed)
6566 instruct loadS(iRegINoSp dst, memory2 mem)
6567 %{
6568 match(Set dst (LoadS mem));
6569 predicate(!needs_acquiring_load(n));
6570
6571 ins_cost(4 * INSN_COST);
6572 format %{ "ldrshw $dst, $mem\t# short" %}
6573
6574 ins_encode(aarch64_enc_ldrshw(dst, mem));
6575
6576 ins_pipe(iload_reg_mem);
6577 %}
6578
6579 // Load Short (16 bit signed) into long
6580 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6581 %{
6582 match(Set dst (ConvI2L (LoadS mem)));
6583 predicate(!needs_acquiring_load(n->in(1)));
6584
6585 ins_cost(4 * INSN_COST);
6586 format %{ "ldrsh $dst, $mem\t# short" %}
6587
6588 ins_encode(aarch64_enc_ldrsh(dst, mem));
6589
6590 ins_pipe(iload_reg_mem);
6591 %}
6592
6593 // Load Char (16 bit unsigned)
6594 instruct loadUS(iRegINoSp dst, memory2 mem)
6595 %{
6596 match(Set dst (LoadUS mem));
6597 predicate(!needs_acquiring_load(n));
6598
6599 ins_cost(4 * INSN_COST);
6600 format %{ "ldrh $dst, $mem\t# short" %}
6601
6602 ins_encode(aarch64_enc_ldrh(dst, mem));
6603
6604 ins_pipe(iload_reg_mem);
6605 %}
6606
6607 // Load Short/Char (16 bit unsigned) into long
6608 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6609 %{
6610 match(Set dst (ConvI2L (LoadUS mem)));
6611 predicate(!needs_acquiring_load(n->in(1)));
6612
6613 ins_cost(4 * INSN_COST);
6614 format %{ "ldrh $dst, $mem\t# short" %}
6615
6616 ins_encode(aarch64_enc_ldrh(dst, mem));
6617
6618 ins_pipe(iload_reg_mem);
6619 %}
6620
6621 // Load Integer (32 bit signed)
6622 instruct loadI(iRegINoSp dst, memory4 mem)
6623 %{
6624 match(Set dst (LoadI mem));
6625 predicate(!needs_acquiring_load(n));
6626
6627 ins_cost(4 * INSN_COST);
6628 format %{ "ldrw $dst, $mem\t# int" %}
6629
6630 ins_encode(aarch64_enc_ldrw(dst, mem));
6631
6632 ins_pipe(iload_reg_mem);
6633 %}
6634
6635 // Load Integer (32 bit signed) into long
6636 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6637 %{
6638 match(Set dst (ConvI2L (LoadI mem)));
6639 predicate(!needs_acquiring_load(n->in(1)));
6640
6641 ins_cost(4 * INSN_COST);
6642 format %{ "ldrsw $dst, $mem\t# int" %}
6643
6644 ins_encode(aarch64_enc_ldrsw(dst, mem));
6645
6646 ins_pipe(iload_reg_mem);
6647 %}
6648
6649 // Load Integer (32 bit unsigned) into long
6650 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6651 %{
6652 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6653 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6654
6655 ins_cost(4 * INSN_COST);
6656 format %{ "ldrw $dst, $mem\t# int" %}
6657
6658 ins_encode(aarch64_enc_ldrw(dst, mem));
6659
6660 ins_pipe(iload_reg_mem);
6661 %}
6662
6663 // Load Long (64 bit signed)
6664 instruct loadL(iRegLNoSp dst, memory8 mem)
6665 %{
6666 match(Set dst (LoadL mem));
6667 predicate(!needs_acquiring_load(n));
6668
6669 ins_cost(4 * INSN_COST);
6670 format %{ "ldr $dst, $mem\t# int" %}
6671
6672 ins_encode(aarch64_enc_ldr(dst, mem));
6673
6674 ins_pipe(iload_reg_mem);
6675 %}
6676
6677 // Load Range
6678 instruct loadRange(iRegINoSp dst, memory4 mem)
6679 %{
6680 match(Set dst (LoadRange mem));
6681
6682 ins_cost(4 * INSN_COST);
6683 format %{ "ldrw $dst, $mem\t# range" %}
6684
6685 ins_encode(aarch64_enc_ldrw(dst, mem));
6686
6687 ins_pipe(iload_reg_mem);
6688 %}
6689
6690 // Load Pointer
6691 instruct loadP(iRegPNoSp dst, memory8 mem)
6692 %{
6693 match(Set dst (LoadP mem));
6694 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6695
6696 ins_cost(4 * INSN_COST);
6697 format %{ "ldr $dst, $mem\t# ptr" %}
6698
6699 ins_encode(aarch64_enc_ldr(dst, mem));
6700
6701 ins_pipe(iload_reg_mem);
6702 %}
6703
6704 // Load Compressed Pointer
6705 instruct loadN(iRegNNoSp dst, memory4 mem)
6706 %{
6707 match(Set dst (LoadN mem));
6708 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6709
6710 ins_cost(4 * INSN_COST);
6711 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6712
6713 ins_encode(aarch64_enc_ldrw(dst, mem));
6714
6715 ins_pipe(iload_reg_mem);
6716 %}
6717
6718 // Load Klass Pointer
6719 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6720 %{
6721 match(Set dst (LoadKlass mem));
6722 predicate(!needs_acquiring_load(n));
6723
6724 ins_cost(4 * INSN_COST);
6725 format %{ "ldr $dst, $mem\t# class" %}
6726
6727 ins_encode(aarch64_enc_ldr(dst, mem));
6728
6729 ins_pipe(iload_reg_mem);
6730 %}
6731
6732 // Load Narrow Klass Pointer
6733 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6734 %{
6735 match(Set dst (LoadNKlass mem));
6736 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6737
6738 ins_cost(4 * INSN_COST);
6739 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6740
6741 ins_encode(aarch64_enc_ldrw(dst, mem));
6742
6743 ins_pipe(iload_reg_mem);
6744 %}
6745
6746 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6747 %{
6748 match(Set dst (LoadNKlass mem));
6749 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6750
6751 ins_cost(4 * INSN_COST);
6752 format %{
6753 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6754 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6755 %}
6756 ins_encode %{
6757 // inlined aarch64_enc_ldrw
6758 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6759 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6760 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6761 %}
6762 ins_pipe(iload_reg_mem);
6763 %}
6764
6765 // Load Float
6766 instruct loadF(vRegF dst, memory4 mem)
6767 %{
6768 match(Set dst (LoadF mem));
6769 predicate(!needs_acquiring_load(n));
6770
6771 ins_cost(4 * INSN_COST);
6772 format %{ "ldrs $dst, $mem\t# float" %}
6773
6774 ins_encode( aarch64_enc_ldrs(dst, mem) );
6775
6776 ins_pipe(pipe_class_memory);
6777 %}
6778
6779 // Load Double
6780 instruct loadD(vRegD dst, memory8 mem)
6781 %{
6782 match(Set dst (LoadD mem));
6783 predicate(!needs_acquiring_load(n));
6784
6785 ins_cost(4 * INSN_COST);
6786 format %{ "ldrd $dst, $mem\t# double" %}
6787
6788 ins_encode( aarch64_enc_ldrd(dst, mem) );
6789
6790 ins_pipe(pipe_class_memory);
6791 %}
6792
6793
6794 // Load Int Constant
6795 instruct loadConI(iRegINoSp dst, immI src)
6796 %{
6797 match(Set dst src);
6798
6799 ins_cost(INSN_COST);
6800 format %{ "mov $dst, $src\t# int" %}
6801
6802 ins_encode( aarch64_enc_movw_imm(dst, src) );
6803
6804 ins_pipe(ialu_imm);
6805 %}
6806
6807 // Load Long Constant
6808 instruct loadConL(iRegLNoSp dst, immL src)
6809 %{
6810 match(Set dst src);
6811
6812 ins_cost(INSN_COST);
6813 format %{ "mov $dst, $src\t# long" %}
6814
6815 ins_encode( aarch64_enc_mov_imm(dst, src) );
6816
6817 ins_pipe(ialu_imm);
6818 %}
6819
6820 // Load Pointer Constant
6821
6822 instruct loadConP(iRegPNoSp dst, immP con)
6823 %{
6824 match(Set dst con);
6825
6826 ins_cost(INSN_COST * 4);
6827 format %{
6828 "mov $dst, $con\t# ptr\n\t"
6829 %}
6830
6831 ins_encode(aarch64_enc_mov_p(dst, con));
6832
6833 ins_pipe(ialu_imm);
6834 %}
6835
6836 // Load Null Pointer Constant
6837
6838 instruct loadConP0(iRegPNoSp dst, immP0 con)
6839 %{
6840 match(Set dst con);
6841
6842 ins_cost(INSN_COST);
6843 format %{ "mov $dst, $con\t# nullptr ptr" %}
6844
6845 ins_encode(aarch64_enc_mov_p0(dst, con));
6846
6847 ins_pipe(ialu_imm);
6848 %}
6849
6850 // Load Pointer Constant One
6851
6852 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6853 %{
6854 match(Set dst con);
6855
6856 ins_cost(INSN_COST);
6857 format %{ "mov $dst, $con\t# nullptr ptr" %}
6858
6859 ins_encode(aarch64_enc_mov_p1(dst, con));
6860
6861 ins_pipe(ialu_imm);
6862 %}
6863
6864 // Load Byte Map Base Constant
6865
6866 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6867 %{
6868 match(Set dst con);
6869
6870 ins_cost(INSN_COST);
6871 format %{ "adr $dst, $con\t# Byte Map Base" %}
6872
6873 ins_encode %{
6874 __ load_byte_map_base($dst$$Register);
6875 %}
6876
6877 ins_pipe(ialu_imm);
6878 %}
6879
6880 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6881 %{
6882 match(Set dst con);
6883
6884 ins_cost(INSN_COST);
6885 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6886
6887 ins_encode %{
6888 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6889 %}
6890
6891 ins_pipe(ialu_imm);
6892 %}
6893
6894 // Load Narrow Pointer Constant
6895
6896 instruct loadConN(iRegNNoSp dst, immN con)
6897 %{
6898 match(Set dst con);
6899
6900 ins_cost(INSN_COST * 4);
6901 format %{ "mov $dst, $con\t# compressed ptr" %}
6902
6903 ins_encode(aarch64_enc_mov_n(dst, con));
6904
6905 ins_pipe(ialu_imm);
6906 %}
6907
6908 // Load Narrow Null Pointer Constant
6909
6910 instruct loadConN0(iRegNNoSp dst, immN0 con)
6911 %{
6912 match(Set dst con);
6913
6914 ins_cost(INSN_COST);
6915 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6916
6917 ins_encode(aarch64_enc_mov_n0(dst, con));
6918
6919 ins_pipe(ialu_imm);
6920 %}
6921
6922 // Load Narrow Klass Constant
6923
6924 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6925 %{
6926 match(Set dst con);
6927
6928 ins_cost(INSN_COST);
6929 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6930
6931 ins_encode(aarch64_enc_mov_nk(dst, con));
6932
6933 ins_pipe(ialu_imm);
6934 %}
6935
6936 // Load Packed Float Constant
6937
6938 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6939 match(Set dst con);
6940 ins_cost(INSN_COST * 4);
6941 format %{ "fmovs $dst, $con"%}
6942 ins_encode %{
6943 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6944 %}
6945
6946 ins_pipe(fp_imm_s);
6947 %}
6948
6949 // Load Float Constant
6950
6951 instruct loadConF(vRegF dst, immF con) %{
6952 match(Set dst con);
6953
6954 ins_cost(INSN_COST * 4);
6955
6956 format %{
6957 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6958 %}
6959
6960 ins_encode %{
6961 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6962 %}
6963
6964 ins_pipe(fp_load_constant_s);
6965 %}
6966
6967 // Load Packed Double Constant
6968
6969 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6970 match(Set dst con);
6971 ins_cost(INSN_COST);
6972 format %{ "fmovd $dst, $con"%}
6973 ins_encode %{
6974 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6975 %}
6976
6977 ins_pipe(fp_imm_d);
6978 %}
6979
6980 // Load Double Constant
6981
6982 instruct loadConD(vRegD dst, immD con) %{
6983 match(Set dst con);
6984
6985 ins_cost(INSN_COST * 5);
6986 format %{
6987 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6988 %}
6989
6990 ins_encode %{
6991 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6992 %}
6993
6994 ins_pipe(fp_load_constant_d);
6995 %}
6996
6997 // Load Half Float Constant
6998 // The "ldr" instruction loads a 32-bit word from the constant pool into a
6999 // 32-bit register but only the bottom half will be populated and the top
7000 // 16 bits are zero.
7001 instruct loadConH(vRegF dst, immH con) %{
7002 match(Set dst con);
7003 format %{
7004 "ldrs $dst, [$constantaddress]\t# load from constant table: half float=$con\n\t"
7005 %}
7006 ins_encode %{
7007 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7008 %}
7009 ins_pipe(fp_load_constant_s);
7010 %}
7011
7012 // Store Instructions
7013
7014 // Store Byte
7015 instruct storeB(iRegIorL2I src, memory1 mem)
7016 %{
7017 match(Set mem (StoreB mem src));
7018 predicate(!needs_releasing_store(n));
7019
7020 ins_cost(INSN_COST);
7021 format %{ "strb $src, $mem\t# byte" %}
7022
7023 ins_encode(aarch64_enc_strb(src, mem));
7024
7025 ins_pipe(istore_reg_mem);
7026 %}
7027
7028
7029 instruct storeimmB0(immI0 zero, memory1 mem)
7030 %{
7031 match(Set mem (StoreB mem zero));
7032 predicate(!needs_releasing_store(n));
7033
7034 ins_cost(INSN_COST);
7035 format %{ "strb rscractch2, $mem\t# byte" %}
7036
7037 ins_encode(aarch64_enc_strb0(mem));
7038
7039 ins_pipe(istore_mem);
7040 %}
7041
7042 // Store Char/Short
7043 instruct storeC(iRegIorL2I src, memory2 mem)
7044 %{
7045 match(Set mem (StoreC mem src));
7046 predicate(!needs_releasing_store(n));
7047
7048 ins_cost(INSN_COST);
7049 format %{ "strh $src, $mem\t# short" %}
7050
7051 ins_encode(aarch64_enc_strh(src, mem));
7052
7053 ins_pipe(istore_reg_mem);
7054 %}
7055
7056 instruct storeimmC0(immI0 zero, memory2 mem)
7057 %{
7058 match(Set mem (StoreC mem zero));
7059 predicate(!needs_releasing_store(n));
7060
7061 ins_cost(INSN_COST);
7062 format %{ "strh zr, $mem\t# short" %}
7063
7064 ins_encode(aarch64_enc_strh0(mem));
7065
7066 ins_pipe(istore_mem);
7067 %}
7068
7069 // Store Integer
7070
7071 instruct storeI(iRegIorL2I src, memory4 mem)
7072 %{
7073 match(Set mem(StoreI mem src));
7074 predicate(!needs_releasing_store(n));
7075
7076 ins_cost(INSN_COST);
7077 format %{ "strw $src, $mem\t# int" %}
7078
7079 ins_encode(aarch64_enc_strw(src, mem));
7080
7081 ins_pipe(istore_reg_mem);
7082 %}
7083
7084 instruct storeimmI0(immI0 zero, memory4 mem)
7085 %{
7086 match(Set mem(StoreI mem zero));
7087 predicate(!needs_releasing_store(n));
7088
7089 ins_cost(INSN_COST);
7090 format %{ "strw zr, $mem\t# int" %}
7091
7092 ins_encode(aarch64_enc_strw0(mem));
7093
7094 ins_pipe(istore_mem);
7095 %}
7096
7097 // Store Long (64 bit signed)
7098 instruct storeL(iRegL src, memory8 mem)
7099 %{
7100 match(Set mem (StoreL mem src));
7101 predicate(!needs_releasing_store(n));
7102
7103 ins_cost(INSN_COST);
7104 format %{ "str $src, $mem\t# int" %}
7105
7106 ins_encode(aarch64_enc_str(src, mem));
7107
7108 ins_pipe(istore_reg_mem);
7109 %}
7110
7111 // Store Long (64 bit signed)
7112 instruct storeimmL0(immL0 zero, memory8 mem)
7113 %{
7114 match(Set mem (StoreL mem zero));
7115 predicate(!needs_releasing_store(n));
7116
7117 ins_cost(INSN_COST);
7118 format %{ "str zr, $mem\t# int" %}
7119
7120 ins_encode(aarch64_enc_str0(mem));
7121
7122 ins_pipe(istore_mem);
7123 %}
7124
7125 // Store Pointer
7126 instruct storeP(iRegP src, memory8 mem)
7127 %{
7128 match(Set mem (StoreP mem src));
7129 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7130
7131 ins_cost(INSN_COST);
7132 format %{ "str $src, $mem\t# ptr" %}
7133
7134 ins_encode(aarch64_enc_str(src, mem));
7135
7136 ins_pipe(istore_reg_mem);
7137 %}
7138
7139 // Store Pointer
7140 instruct storeimmP0(immP0 zero, memory8 mem)
7141 %{
7142 match(Set mem (StoreP mem zero));
7143 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7144
7145 ins_cost(INSN_COST);
7146 format %{ "str zr, $mem\t# ptr" %}
7147
7148 ins_encode(aarch64_enc_str0(mem));
7149
7150 ins_pipe(istore_mem);
7151 %}
7152
7153 // Store Compressed Pointer
7154 instruct storeN(iRegN src, memory4 mem)
7155 %{
7156 match(Set mem (StoreN mem src));
7157 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7158
7159 ins_cost(INSN_COST);
7160 format %{ "strw $src, $mem\t# compressed ptr" %}
7161
7162 ins_encode(aarch64_enc_strw(src, mem));
7163
7164 ins_pipe(istore_reg_mem);
7165 %}
7166
7167 instruct storeImmN0(immN0 zero, memory4 mem)
7168 %{
7169 match(Set mem (StoreN mem zero));
7170 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7171
7172 ins_cost(INSN_COST);
7173 format %{ "strw zr, $mem\t# compressed ptr" %}
7174
7175 ins_encode(aarch64_enc_strw0(mem));
7176
7177 ins_pipe(istore_mem);
7178 %}
7179
7180 // Store Float
7181 instruct storeF(vRegF src, memory4 mem)
7182 %{
7183 match(Set mem (StoreF mem src));
7184 predicate(!needs_releasing_store(n));
7185
7186 ins_cost(INSN_COST);
7187 format %{ "strs $src, $mem\t# float" %}
7188
7189 ins_encode( aarch64_enc_strs(src, mem) );
7190
7191 ins_pipe(pipe_class_memory);
7192 %}
7193
7194 // TODO
7195 // implement storeImmF0 and storeFImmPacked
7196
7197 // Store Double
7198 instruct storeD(vRegD src, memory8 mem)
7199 %{
7200 match(Set mem (StoreD mem src));
7201 predicate(!needs_releasing_store(n));
7202
7203 ins_cost(INSN_COST);
7204 format %{ "strd $src, $mem\t# double" %}
7205
7206 ins_encode( aarch64_enc_strd(src, mem) );
7207
7208 ins_pipe(pipe_class_memory);
7209 %}
7210
7211 // Store Compressed Klass Pointer
7212 instruct storeNKlass(iRegN src, memory4 mem)
7213 %{
7214 predicate(!needs_releasing_store(n));
7215 match(Set mem (StoreNKlass mem src));
7216
7217 ins_cost(INSN_COST);
7218 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7219
7220 ins_encode(aarch64_enc_strw(src, mem));
7221
7222 ins_pipe(istore_reg_mem);
7223 %}
7224
7225 // TODO
7226 // implement storeImmD0 and storeDImmPacked
7227
7228 // prefetch instructions
7229 // Must be safe to execute with invalid address (cannot fault).
7230
7231 instruct prefetchalloc( memory8 mem ) %{
7232 match(PrefetchAllocation mem);
7233
7234 ins_cost(INSN_COST);
7235 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7236
7237 ins_encode( aarch64_enc_prefetchw(mem) );
7238
7239 ins_pipe(iload_prefetch);
7240 %}
7241
7242 // ---------------- volatile loads and stores ----------------
7243
7244 // Load Byte (8 bit signed)
7245 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7246 %{
7247 match(Set dst (LoadB mem));
7248
7249 ins_cost(VOLATILE_REF_COST);
7250 format %{ "ldarsb $dst, $mem\t# byte" %}
7251
7252 ins_encode(aarch64_enc_ldarsb(dst, mem));
7253
7254 ins_pipe(pipe_serial);
7255 %}
7256
7257 // Load Byte (8 bit signed) into long
7258 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7259 %{
7260 match(Set dst (ConvI2L (LoadB mem)));
7261
7262 ins_cost(VOLATILE_REF_COST);
7263 format %{ "ldarsb $dst, $mem\t# byte" %}
7264
7265 ins_encode(aarch64_enc_ldarsb(dst, mem));
7266
7267 ins_pipe(pipe_serial);
7268 %}
7269
7270 // Load Byte (8 bit unsigned)
7271 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7272 %{
7273 match(Set dst (LoadUB mem));
7274
7275 ins_cost(VOLATILE_REF_COST);
7276 format %{ "ldarb $dst, $mem\t# byte" %}
7277
7278 ins_encode(aarch64_enc_ldarb(dst, mem));
7279
7280 ins_pipe(pipe_serial);
7281 %}
7282
7283 // Load Byte (8 bit unsigned) into long
7284 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7285 %{
7286 match(Set dst (ConvI2L (LoadUB mem)));
7287
7288 ins_cost(VOLATILE_REF_COST);
7289 format %{ "ldarb $dst, $mem\t# byte" %}
7290
7291 ins_encode(aarch64_enc_ldarb(dst, mem));
7292
7293 ins_pipe(pipe_serial);
7294 %}
7295
7296 // Load Short (16 bit signed)
7297 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7298 %{
7299 match(Set dst (LoadS mem));
7300
7301 ins_cost(VOLATILE_REF_COST);
7302 format %{ "ldarshw $dst, $mem\t# short" %}
7303
7304 ins_encode(aarch64_enc_ldarshw(dst, mem));
7305
7306 ins_pipe(pipe_serial);
7307 %}
7308
7309 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7310 %{
7311 match(Set dst (LoadUS mem));
7312
7313 ins_cost(VOLATILE_REF_COST);
7314 format %{ "ldarhw $dst, $mem\t# short" %}
7315
7316 ins_encode(aarch64_enc_ldarhw(dst, mem));
7317
7318 ins_pipe(pipe_serial);
7319 %}
7320
7321 // Load Short/Char (16 bit unsigned) into long
7322 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7323 %{
7324 match(Set dst (ConvI2L (LoadUS mem)));
7325
7326 ins_cost(VOLATILE_REF_COST);
7327 format %{ "ldarh $dst, $mem\t# short" %}
7328
7329 ins_encode(aarch64_enc_ldarh(dst, mem));
7330
7331 ins_pipe(pipe_serial);
7332 %}
7333
7334 // Load Short/Char (16 bit signed) into long
7335 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7336 %{
7337 match(Set dst (ConvI2L (LoadS mem)));
7338
7339 ins_cost(VOLATILE_REF_COST);
7340 format %{ "ldarh $dst, $mem\t# short" %}
7341
7342 ins_encode(aarch64_enc_ldarsh(dst, mem));
7343
7344 ins_pipe(pipe_serial);
7345 %}
7346
7347 // Load Integer (32 bit signed)
7348 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7349 %{
7350 match(Set dst (LoadI mem));
7351
7352 ins_cost(VOLATILE_REF_COST);
7353 format %{ "ldarw $dst, $mem\t# int" %}
7354
7355 ins_encode(aarch64_enc_ldarw(dst, mem));
7356
7357 ins_pipe(pipe_serial);
7358 %}
7359
7360 // Load Integer (32 bit unsigned) into long
7361 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7362 %{
7363 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7364
7365 ins_cost(VOLATILE_REF_COST);
7366 format %{ "ldarw $dst, $mem\t# int" %}
7367
7368 ins_encode(aarch64_enc_ldarw(dst, mem));
7369
7370 ins_pipe(pipe_serial);
7371 %}
7372
7373 // Load Long (64 bit signed)
7374 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7375 %{
7376 match(Set dst (LoadL mem));
7377
7378 ins_cost(VOLATILE_REF_COST);
7379 format %{ "ldar $dst, $mem\t# int" %}
7380
7381 ins_encode(aarch64_enc_ldar(dst, mem));
7382
7383 ins_pipe(pipe_serial);
7384 %}
7385
7386 // Load Pointer
7387 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7388 %{
7389 match(Set dst (LoadP mem));
7390 predicate(n->as_Load()->barrier_data() == 0);
7391
7392 ins_cost(VOLATILE_REF_COST);
7393 format %{ "ldar $dst, $mem\t# ptr" %}
7394
7395 ins_encode(aarch64_enc_ldar(dst, mem));
7396
7397 ins_pipe(pipe_serial);
7398 %}
7399
7400 // Load Compressed Pointer
7401 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7402 %{
7403 match(Set dst (LoadN mem));
7404 predicate(n->as_Load()->barrier_data() == 0);
7405
7406 ins_cost(VOLATILE_REF_COST);
7407 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7408
7409 ins_encode(aarch64_enc_ldarw(dst, mem));
7410
7411 ins_pipe(pipe_serial);
7412 %}
7413
7414 // Load Float
7415 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7416 %{
7417 match(Set dst (LoadF mem));
7418
7419 ins_cost(VOLATILE_REF_COST);
7420 format %{ "ldars $dst, $mem\t# float" %}
7421
7422 ins_encode( aarch64_enc_fldars(dst, mem) );
7423
7424 ins_pipe(pipe_serial);
7425 %}
7426
7427 // Load Double
7428 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7429 %{
7430 match(Set dst (LoadD mem));
7431
7432 ins_cost(VOLATILE_REF_COST);
7433 format %{ "ldard $dst, $mem\t# double" %}
7434
7435 ins_encode( aarch64_enc_fldard(dst, mem) );
7436
7437 ins_pipe(pipe_serial);
7438 %}
7439
7440 // Store Byte
7441 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7442 %{
7443 match(Set mem (StoreB mem src));
7444
7445 ins_cost(VOLATILE_REF_COST);
7446 format %{ "stlrb $src, $mem\t# byte" %}
7447
7448 ins_encode(aarch64_enc_stlrb(src, mem));
7449
7450 ins_pipe(pipe_class_memory);
7451 %}
7452
7453 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7454 %{
7455 match(Set mem (StoreB mem zero));
7456
7457 ins_cost(VOLATILE_REF_COST);
7458 format %{ "stlrb zr, $mem\t# byte" %}
7459
7460 ins_encode(aarch64_enc_stlrb0(mem));
7461
7462 ins_pipe(pipe_class_memory);
7463 %}
7464
7465 // Store Char/Short
7466 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7467 %{
7468 match(Set mem (StoreC mem src));
7469
7470 ins_cost(VOLATILE_REF_COST);
7471 format %{ "stlrh $src, $mem\t# short" %}
7472
7473 ins_encode(aarch64_enc_stlrh(src, mem));
7474
7475 ins_pipe(pipe_class_memory);
7476 %}
7477
7478 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7479 %{
7480 match(Set mem (StoreC mem zero));
7481
7482 ins_cost(VOLATILE_REF_COST);
7483 format %{ "stlrh zr, $mem\t# short" %}
7484
7485 ins_encode(aarch64_enc_stlrh0(mem));
7486
7487 ins_pipe(pipe_class_memory);
7488 %}
7489
7490 // Store Integer
7491
7492 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7493 %{
7494 match(Set mem(StoreI mem src));
7495
7496 ins_cost(VOLATILE_REF_COST);
7497 format %{ "stlrw $src, $mem\t# int" %}
7498
7499 ins_encode(aarch64_enc_stlrw(src, mem));
7500
7501 ins_pipe(pipe_class_memory);
7502 %}
7503
7504 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7505 %{
7506 match(Set mem(StoreI mem zero));
7507
7508 ins_cost(VOLATILE_REF_COST);
7509 format %{ "stlrw zr, $mem\t# int" %}
7510
7511 ins_encode(aarch64_enc_stlrw0(mem));
7512
7513 ins_pipe(pipe_class_memory);
7514 %}
7515
7516 // Store Long (64 bit signed)
7517 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7518 %{
7519 match(Set mem (StoreL mem src));
7520
7521 ins_cost(VOLATILE_REF_COST);
7522 format %{ "stlr $src, $mem\t# int" %}
7523
7524 ins_encode(aarch64_enc_stlr(src, mem));
7525
7526 ins_pipe(pipe_class_memory);
7527 %}
7528
7529 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7530 %{
7531 match(Set mem (StoreL mem zero));
7532
7533 ins_cost(VOLATILE_REF_COST);
7534 format %{ "stlr zr, $mem\t# int" %}
7535
7536 ins_encode(aarch64_enc_stlr0(mem));
7537
7538 ins_pipe(pipe_class_memory);
7539 %}
7540
7541 // Store Pointer
7542 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7543 %{
7544 match(Set mem (StoreP mem src));
7545 predicate(n->as_Store()->barrier_data() == 0);
7546
7547 ins_cost(VOLATILE_REF_COST);
7548 format %{ "stlr $src, $mem\t# ptr" %}
7549
7550 ins_encode(aarch64_enc_stlr(src, mem));
7551
7552 ins_pipe(pipe_class_memory);
7553 %}
7554
7555 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7556 %{
7557 match(Set mem (StoreP mem zero));
7558 predicate(n->as_Store()->barrier_data() == 0);
7559
7560 ins_cost(VOLATILE_REF_COST);
7561 format %{ "stlr zr, $mem\t# ptr" %}
7562
7563 ins_encode(aarch64_enc_stlr0(mem));
7564
7565 ins_pipe(pipe_class_memory);
7566 %}
7567
7568 // Store Compressed Pointer
7569 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7570 %{
7571 match(Set mem (StoreN mem src));
7572 predicate(n->as_Store()->barrier_data() == 0);
7573
7574 ins_cost(VOLATILE_REF_COST);
7575 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7576
7577 ins_encode(aarch64_enc_stlrw(src, mem));
7578
7579 ins_pipe(pipe_class_memory);
7580 %}
7581
7582 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7583 %{
7584 match(Set mem (StoreN mem zero));
7585 predicate(n->as_Store()->barrier_data() == 0);
7586
7587 ins_cost(VOLATILE_REF_COST);
7588 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7589
7590 ins_encode(aarch64_enc_stlrw0(mem));
7591
7592 ins_pipe(pipe_class_memory);
7593 %}
7594
7595 // Store Float
7596 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7597 %{
7598 match(Set mem (StoreF mem src));
7599
7600 ins_cost(VOLATILE_REF_COST);
7601 format %{ "stlrs $src, $mem\t# float" %}
7602
7603 ins_encode( aarch64_enc_fstlrs(src, mem) );
7604
7605 ins_pipe(pipe_class_memory);
7606 %}
7607
7608 // TODO
7609 // implement storeImmF0 and storeFImmPacked
7610
7611 // Store Double
7612 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7613 %{
7614 match(Set mem (StoreD mem src));
7615
7616 ins_cost(VOLATILE_REF_COST);
7617 format %{ "stlrd $src, $mem\t# double" %}
7618
7619 ins_encode( aarch64_enc_fstlrd(src, mem) );
7620
7621 ins_pipe(pipe_class_memory);
7622 %}
7623
7624 // ---------------- end of volatile loads and stores ----------------
7625
7626 instruct cacheWB(indirect addr)
7627 %{
7628 predicate(VM_Version::supports_data_cache_line_flush());
7629 match(CacheWB addr);
7630
7631 ins_cost(100);
7632 format %{"cache wb $addr" %}
7633 ins_encode %{
7634 assert($addr->index_position() < 0, "should be");
7635 assert($addr$$disp == 0, "should be");
7636 __ cache_wb(Address($addr$$base$$Register, 0));
7637 %}
7638 ins_pipe(pipe_slow); // XXX
7639 %}
7640
7641 instruct cacheWBPreSync()
7642 %{
7643 predicate(VM_Version::supports_data_cache_line_flush());
7644 match(CacheWBPreSync);
7645
7646 ins_cost(100);
7647 format %{"cache wb presync" %}
7648 ins_encode %{
7649 __ cache_wbsync(true);
7650 %}
7651 ins_pipe(pipe_slow); // XXX
7652 %}
7653
7654 instruct cacheWBPostSync()
7655 %{
7656 predicate(VM_Version::supports_data_cache_line_flush());
7657 match(CacheWBPostSync);
7658
7659 ins_cost(100);
7660 format %{"cache wb postsync" %}
7661 ins_encode %{
7662 __ cache_wbsync(false);
7663 %}
7664 ins_pipe(pipe_slow); // XXX
7665 %}
7666
7667 // ============================================================================
7668 // BSWAP Instructions
7669
7670 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7671 match(Set dst (ReverseBytesI src));
7672
7673 ins_cost(INSN_COST);
7674 format %{ "revw $dst, $src" %}
7675
7676 ins_encode %{
7677 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7678 %}
7679
7680 ins_pipe(ialu_reg);
7681 %}
7682
7683 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7684 match(Set dst (ReverseBytesL src));
7685
7686 ins_cost(INSN_COST);
7687 format %{ "rev $dst, $src" %}
7688
7689 ins_encode %{
7690 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7691 %}
7692
7693 ins_pipe(ialu_reg);
7694 %}
7695
7696 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7697 match(Set dst (ReverseBytesUS src));
7698
7699 ins_cost(INSN_COST);
7700 format %{ "rev16w $dst, $src" %}
7701
7702 ins_encode %{
7703 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7704 %}
7705
7706 ins_pipe(ialu_reg);
7707 %}
7708
7709 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7710 match(Set dst (ReverseBytesS src));
7711
7712 ins_cost(INSN_COST);
7713 format %{ "rev16w $dst, $src\n\t"
7714 "sbfmw $dst, $dst, #0, #15" %}
7715
7716 ins_encode %{
7717 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7718 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7719 %}
7720
7721 ins_pipe(ialu_reg);
7722 %}
7723
7724 // ============================================================================
7725 // Zero Count Instructions
7726
7727 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7728 match(Set dst (CountLeadingZerosI src));
7729
7730 ins_cost(INSN_COST);
7731 format %{ "clzw $dst, $src" %}
7732 ins_encode %{
7733 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7734 %}
7735
7736 ins_pipe(ialu_reg);
7737 %}
7738
7739 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7740 match(Set dst (CountLeadingZerosL src));
7741
7742 ins_cost(INSN_COST);
7743 format %{ "clz $dst, $src" %}
7744 ins_encode %{
7745 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7746 %}
7747
7748 ins_pipe(ialu_reg);
7749 %}
7750
7751 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7752 match(Set dst (CountTrailingZerosI src));
7753
7754 ins_cost(INSN_COST * 2);
7755 format %{ "rbitw $dst, $src\n\t"
7756 "clzw $dst, $dst" %}
7757 ins_encode %{
7758 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7759 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7760 %}
7761
7762 ins_pipe(ialu_reg);
7763 %}
7764
7765 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7766 match(Set dst (CountTrailingZerosL src));
7767
7768 ins_cost(INSN_COST * 2);
7769 format %{ "rbit $dst, $src\n\t"
7770 "clz $dst, $dst" %}
7771 ins_encode %{
7772 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7773 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7774 %}
7775
7776 ins_pipe(ialu_reg);
7777 %}
7778
7779 //---------- Population Count Instructions -------------------------------------
7780 //
7781
7782 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7783 match(Set dst (PopCountI src));
7784 effect(TEMP tmp);
7785 ins_cost(INSN_COST * 13);
7786
7787 format %{ "movw $src, $src\n\t"
7788 "mov $tmp, $src\t# vector (1D)\n\t"
7789 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7790 "addv $tmp, $tmp\t# vector (8B)\n\t"
7791 "mov $dst, $tmp\t# vector (1D)" %}
7792 ins_encode %{
7793 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7794 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7795 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7796 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7797 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7798 %}
7799
7800 ins_pipe(pipe_class_default);
7801 %}
7802
7803 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7804 match(Set dst (PopCountI (LoadI mem)));
7805 effect(TEMP tmp);
7806 ins_cost(INSN_COST * 13);
7807
7808 format %{ "ldrs $tmp, $mem\n\t"
7809 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7810 "addv $tmp, $tmp\t# vector (8B)\n\t"
7811 "mov $dst, $tmp\t# vector (1D)" %}
7812 ins_encode %{
7813 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7814 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7815 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7816 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7817 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7818 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7819 %}
7820
7821 ins_pipe(pipe_class_default);
7822 %}
7823
7824 // Note: Long.bitCount(long) returns an int.
7825 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7826 match(Set dst (PopCountL src));
7827 effect(TEMP tmp);
7828 ins_cost(INSN_COST * 13);
7829
7830 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7831 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7832 "addv $tmp, $tmp\t# vector (8B)\n\t"
7833 "mov $dst, $tmp\t# vector (1D)" %}
7834 ins_encode %{
7835 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7836 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7837 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7838 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7839 %}
7840
7841 ins_pipe(pipe_class_default);
7842 %}
7843
7844 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7845 match(Set dst (PopCountL (LoadL mem)));
7846 effect(TEMP tmp);
7847 ins_cost(INSN_COST * 13);
7848
7849 format %{ "ldrd $tmp, $mem\n\t"
7850 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7851 "addv $tmp, $tmp\t# vector (8B)\n\t"
7852 "mov $dst, $tmp\t# vector (1D)" %}
7853 ins_encode %{
7854 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7855 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7856 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7857 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7858 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7859 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7860 %}
7861
7862 ins_pipe(pipe_class_default);
7863 %}
7864
7865 // ============================================================================
7866 // VerifyVectorAlignment Instruction
7867
7868 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7869 match(Set addr (VerifyVectorAlignment addr mask));
7870 effect(KILL cr);
7871 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7872 ins_encode %{
7873 Label Lskip;
7874 // check if masked bits of addr are zero
7875 __ tst($addr$$Register, $mask$$constant);
7876 __ br(Assembler::EQ, Lskip);
7877 __ stop("verify_vector_alignment found a misaligned vector memory access");
7878 __ bind(Lskip);
7879 %}
7880 ins_pipe(pipe_slow);
7881 %}
7882
7883 // ============================================================================
7884 // MemBar Instruction
7885
7886 instruct load_fence() %{
7887 match(LoadFence);
7888 ins_cost(VOLATILE_REF_COST);
7889
7890 format %{ "load_fence" %}
7891
7892 ins_encode %{
7893 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7894 %}
7895 ins_pipe(pipe_serial);
7896 %}
7897
7898 instruct unnecessary_membar_acquire() %{
7899 predicate(unnecessary_acquire(n));
7900 match(MemBarAcquire);
7901 ins_cost(0);
7902
7903 format %{ "membar_acquire (elided)" %}
7904
7905 ins_encode %{
7906 __ block_comment("membar_acquire (elided)");
7907 %}
7908
7909 ins_pipe(pipe_class_empty);
7910 %}
7911
7912 instruct membar_acquire() %{
7913 match(MemBarAcquire);
7914 ins_cost(VOLATILE_REF_COST);
7915
7916 format %{ "membar_acquire\n\t"
7917 "dmb ishld" %}
7918
7919 ins_encode %{
7920 __ block_comment("membar_acquire");
7921 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7922 %}
7923
7924 ins_pipe(pipe_serial);
7925 %}
7926
7927
7928 instruct membar_acquire_lock() %{
7929 match(MemBarAcquireLock);
7930 ins_cost(VOLATILE_REF_COST);
7931
7932 format %{ "membar_acquire_lock (elided)" %}
7933
7934 ins_encode %{
7935 __ block_comment("membar_acquire_lock (elided)");
7936 %}
7937
7938 ins_pipe(pipe_serial);
7939 %}
7940
7941 instruct store_fence() %{
7942 match(StoreFence);
7943 ins_cost(VOLATILE_REF_COST);
7944
7945 format %{ "store_fence" %}
7946
7947 ins_encode %{
7948 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7949 %}
7950 ins_pipe(pipe_serial);
7951 %}
7952
7953 instruct unnecessary_membar_release() %{
7954 predicate(unnecessary_release(n));
7955 match(MemBarRelease);
7956 ins_cost(0);
7957
7958 format %{ "membar_release (elided)" %}
7959
7960 ins_encode %{
7961 __ block_comment("membar_release (elided)");
7962 %}
7963 ins_pipe(pipe_serial);
7964 %}
7965
7966 instruct membar_release() %{
7967 match(MemBarRelease);
7968 ins_cost(VOLATILE_REF_COST);
7969
7970 format %{ "membar_release\n\t"
7971 "dmb ishst\n\tdmb ishld" %}
7972
7973 ins_encode %{
7974 __ block_comment("membar_release");
7975 // These will be merged if AlwaysMergeDMB is enabled.
7976 __ membar(Assembler::StoreStore);
7977 __ membar(Assembler::LoadStore);
7978 %}
7979 ins_pipe(pipe_serial);
7980 %}
7981
7982 instruct membar_storestore() %{
7983 match(MemBarStoreStore);
7984 match(StoreStoreFence);
7985 ins_cost(VOLATILE_REF_COST);
7986
7987 format %{ "MEMBAR-store-store" %}
7988
7989 ins_encode %{
7990 __ membar(Assembler::StoreStore);
7991 %}
7992 ins_pipe(pipe_serial);
7993 %}
7994
7995 instruct membar_release_lock() %{
7996 match(MemBarReleaseLock);
7997 ins_cost(VOLATILE_REF_COST);
7998
7999 format %{ "membar_release_lock (elided)" %}
8000
8001 ins_encode %{
8002 __ block_comment("membar_release_lock (elided)");
8003 %}
8004
8005 ins_pipe(pipe_serial);
8006 %}
8007
8008 instruct unnecessary_membar_volatile() %{
8009 predicate(unnecessary_volatile(n));
8010 match(MemBarVolatile);
8011 ins_cost(0);
8012
8013 format %{ "membar_volatile (elided)" %}
8014
8015 ins_encode %{
8016 __ block_comment("membar_volatile (elided)");
8017 %}
8018
8019 ins_pipe(pipe_serial);
8020 %}
8021
8022 instruct membar_volatile() %{
8023 match(MemBarVolatile);
8024 ins_cost(VOLATILE_REF_COST*100);
8025
8026 format %{ "membar_volatile\n\t"
8027 "dmb ish"%}
8028
8029 ins_encode %{
8030 __ block_comment("membar_volatile");
8031 __ membar(Assembler::StoreLoad);
8032 %}
8033
8034 ins_pipe(pipe_serial);
8035 %}
8036
8037 // ============================================================================
8038 // Cast/Convert Instructions
8039
8040 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8041 match(Set dst (CastX2P src));
8042
8043 ins_cost(INSN_COST);
8044 format %{ "mov $dst, $src\t# long -> ptr" %}
8045
8046 ins_encode %{
8047 if ($dst$$reg != $src$$reg) {
8048 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8049 }
8050 %}
8051
8052 ins_pipe(ialu_reg);
8053 %}
8054
8055 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8056 match(Set dst (CastP2X src));
8057
8058 ins_cost(INSN_COST);
8059 format %{ "mov $dst, $src\t# ptr -> long" %}
8060
8061 ins_encode %{
8062 if ($dst$$reg != $src$$reg) {
8063 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8064 }
8065 %}
8066
8067 ins_pipe(ialu_reg);
8068 %}
8069
8070 // Convert oop into int for vectors alignment masking
8071 instruct convP2I(iRegINoSp dst, iRegP src) %{
8072 match(Set dst (ConvL2I (CastP2X src)));
8073
8074 ins_cost(INSN_COST);
8075 format %{ "movw $dst, $src\t# ptr -> int" %}
8076 ins_encode %{
8077 __ movw($dst$$Register, $src$$Register);
8078 %}
8079
8080 ins_pipe(ialu_reg);
8081 %}
8082
8083 // Convert compressed oop into int for vectors alignment masking
8084 // in case of 32bit oops (heap < 4Gb).
8085 instruct convN2I(iRegINoSp dst, iRegN src)
8086 %{
8087 predicate(CompressedOops::shift() == 0);
8088 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8089
8090 ins_cost(INSN_COST);
8091 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8092 ins_encode %{
8093 __ movw($dst$$Register, $src$$Register);
8094 %}
8095
8096 ins_pipe(ialu_reg);
8097 %}
8098
8099
8100 // Convert oop pointer into compressed form
8101 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8102 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8103 match(Set dst (EncodeP src));
8104 effect(KILL cr);
8105 ins_cost(INSN_COST * 3);
8106 format %{ "encode_heap_oop $dst, $src" %}
8107 ins_encode %{
8108 Register s = $src$$Register;
8109 Register d = $dst$$Register;
8110 __ encode_heap_oop(d, s);
8111 %}
8112 ins_pipe(ialu_reg);
8113 %}
8114
8115 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8116 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8117 match(Set dst (EncodeP src));
8118 ins_cost(INSN_COST * 3);
8119 format %{ "encode_heap_oop_not_null $dst, $src" %}
8120 ins_encode %{
8121 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8122 %}
8123 ins_pipe(ialu_reg);
8124 %}
8125
8126 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8127 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8128 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8129 match(Set dst (DecodeN src));
8130 ins_cost(INSN_COST * 3);
8131 format %{ "decode_heap_oop $dst, $src" %}
8132 ins_encode %{
8133 Register s = $src$$Register;
8134 Register d = $dst$$Register;
8135 __ decode_heap_oop(d, s);
8136 %}
8137 ins_pipe(ialu_reg);
8138 %}
8139
8140 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8141 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8142 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8143 match(Set dst (DecodeN src));
8144 ins_cost(INSN_COST * 3);
8145 format %{ "decode_heap_oop_not_null $dst, $src" %}
8146 ins_encode %{
8147 Register s = $src$$Register;
8148 Register d = $dst$$Register;
8149 __ decode_heap_oop_not_null(d, s);
8150 %}
8151 ins_pipe(ialu_reg);
8152 %}
8153
8154 // n.b. AArch64 implementations of encode_klass_not_null and
8155 // decode_klass_not_null do not modify the flags register so, unlike
8156 // Intel, we don't kill CR as a side effect here
8157
8158 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8159 match(Set dst (EncodePKlass src));
8160
8161 ins_cost(INSN_COST * 3);
8162 format %{ "encode_klass_not_null $dst,$src" %}
8163
8164 ins_encode %{
8165 Register src_reg = as_Register($src$$reg);
8166 Register dst_reg = as_Register($dst$$reg);
8167 __ encode_klass_not_null(dst_reg, src_reg);
8168 %}
8169
8170 ins_pipe(ialu_reg);
8171 %}
8172
8173 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8174 match(Set dst (DecodeNKlass src));
8175
8176 ins_cost(INSN_COST * 3);
8177 format %{ "decode_klass_not_null $dst,$src" %}
8178
8179 ins_encode %{
8180 Register src_reg = as_Register($src$$reg);
8181 Register dst_reg = as_Register($dst$$reg);
8182 if (dst_reg != src_reg) {
8183 __ decode_klass_not_null(dst_reg, src_reg);
8184 } else {
8185 __ decode_klass_not_null(dst_reg);
8186 }
8187 %}
8188
8189 ins_pipe(ialu_reg);
8190 %}
8191
8192 instruct checkCastPP(iRegPNoSp dst)
8193 %{
8194 match(Set dst (CheckCastPP dst));
8195
8196 size(0);
8197 format %{ "# checkcastPP of $dst" %}
8198 ins_encode(/* empty encoding */);
8199 ins_pipe(pipe_class_empty);
8200 %}
8201
8202 instruct castPP(iRegPNoSp dst)
8203 %{
8204 match(Set dst (CastPP dst));
8205
8206 size(0);
8207 format %{ "# castPP of $dst" %}
8208 ins_encode(/* empty encoding */);
8209 ins_pipe(pipe_class_empty);
8210 %}
8211
8212 instruct castII(iRegI dst)
8213 %{
8214 predicate(VerifyConstraintCasts == 0);
8215 match(Set dst (CastII dst));
8216
8217 size(0);
8218 format %{ "# castII of $dst" %}
8219 ins_encode(/* empty encoding */);
8220 ins_cost(0);
8221 ins_pipe(pipe_class_empty);
8222 %}
8223
8224 instruct castII_checked(iRegI dst, rFlagsReg cr)
8225 %{
8226 predicate(VerifyConstraintCasts > 0);
8227 match(Set dst (CastII dst));
8228 effect(KILL cr);
8229
8230 format %{ "# castII_checked of $dst" %}
8231 ins_encode %{
8232 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8233 %}
8234 ins_pipe(pipe_slow);
8235 %}
8236
8237 instruct castLL(iRegL dst)
8238 %{
8239 predicate(VerifyConstraintCasts == 0);
8240 match(Set dst (CastLL dst));
8241
8242 size(0);
8243 format %{ "# castLL of $dst" %}
8244 ins_encode(/* empty encoding */);
8245 ins_cost(0);
8246 ins_pipe(pipe_class_empty);
8247 %}
8248
8249 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8250 %{
8251 predicate(VerifyConstraintCasts > 0);
8252 match(Set dst (CastLL dst));
8253 effect(KILL cr);
8254
8255 format %{ "# castLL_checked of $dst" %}
8256 ins_encode %{
8257 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8258 %}
8259 ins_pipe(pipe_slow);
8260 %}
8261
8262 instruct castHH(vRegF dst)
8263 %{
8264 match(Set dst (CastHH dst));
8265 size(0);
8266 format %{ "# castHH of $dst" %}
8267 ins_encode(/* empty encoding */);
8268 ins_cost(0);
8269 ins_pipe(pipe_class_empty);
8270 %}
8271
8272 instruct castFF(vRegF dst)
8273 %{
8274 match(Set dst (CastFF dst));
8275
8276 size(0);
8277 format %{ "# castFF of $dst" %}
8278 ins_encode(/* empty encoding */);
8279 ins_cost(0);
8280 ins_pipe(pipe_class_empty);
8281 %}
8282
8283 instruct castDD(vRegD dst)
8284 %{
8285 match(Set dst (CastDD dst));
8286
8287 size(0);
8288 format %{ "# castDD of $dst" %}
8289 ins_encode(/* empty encoding */);
8290 ins_cost(0);
8291 ins_pipe(pipe_class_empty);
8292 %}
8293
8294 instruct castVV(vReg dst)
8295 %{
8296 match(Set dst (CastVV dst));
8297
8298 size(0);
8299 format %{ "# castVV of $dst" %}
8300 ins_encode(/* empty encoding */);
8301 ins_cost(0);
8302 ins_pipe(pipe_class_empty);
8303 %}
8304
8305 instruct castVVMask(pRegGov dst)
8306 %{
8307 match(Set dst (CastVV dst));
8308
8309 size(0);
8310 format %{ "# castVV of $dst" %}
8311 ins_encode(/* empty encoding */);
8312 ins_cost(0);
8313 ins_pipe(pipe_class_empty);
8314 %}
8315
8316 // ============================================================================
8317 // Atomic operation instructions
8318 //
8319
8320 // standard CompareAndSwapX when we are using barriers
8321 // these have higher priority than the rules selected by a predicate
8322
8323 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8324 // can't match them
8325
8326 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8327
8328 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8329 ins_cost(2 * VOLATILE_REF_COST);
8330
8331 effect(KILL cr);
8332
8333 format %{
8334 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8335 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8336 %}
8337
8338 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8339 aarch64_enc_cset_eq(res));
8340
8341 ins_pipe(pipe_slow);
8342 %}
8343
8344 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8345
8346 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8347 ins_cost(2 * VOLATILE_REF_COST);
8348
8349 effect(KILL cr);
8350
8351 format %{
8352 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8353 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8354 %}
8355
8356 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8357 aarch64_enc_cset_eq(res));
8358
8359 ins_pipe(pipe_slow);
8360 %}
8361
8362 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8363
8364 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8365 ins_cost(2 * VOLATILE_REF_COST);
8366
8367 effect(KILL cr);
8368
8369 format %{
8370 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8371 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8372 %}
8373
8374 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8375 aarch64_enc_cset_eq(res));
8376
8377 ins_pipe(pipe_slow);
8378 %}
8379
8380 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8381
8382 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8383 ins_cost(2 * VOLATILE_REF_COST);
8384
8385 effect(KILL cr);
8386
8387 format %{
8388 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8389 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8390 %}
8391
8392 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8393 aarch64_enc_cset_eq(res));
8394
8395 ins_pipe(pipe_slow);
8396 %}
8397
8398 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8399
8400 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8401 predicate(n->as_LoadStore()->barrier_data() == 0);
8402 ins_cost(2 * VOLATILE_REF_COST);
8403
8404 effect(KILL cr);
8405
8406 format %{
8407 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8408 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8409 %}
8410
8411 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8412 aarch64_enc_cset_eq(res));
8413
8414 ins_pipe(pipe_slow);
8415 %}
8416
8417 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8418
8419 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8420 predicate(n->as_LoadStore()->barrier_data() == 0);
8421 ins_cost(2 * VOLATILE_REF_COST);
8422
8423 effect(KILL cr);
8424
8425 format %{
8426 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8427 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8428 %}
8429
8430 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8431 aarch64_enc_cset_eq(res));
8432
8433 ins_pipe(pipe_slow);
8434 %}
8435
8436 // alternative CompareAndSwapX when we are eliding barriers
8437
8438 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8439
8440 predicate(needs_acquiring_load_exclusive(n));
8441 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8442 ins_cost(VOLATILE_REF_COST);
8443
8444 effect(KILL cr);
8445
8446 format %{
8447 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8448 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8449 %}
8450
8451 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8452 aarch64_enc_cset_eq(res));
8453
8454 ins_pipe(pipe_slow);
8455 %}
8456
8457 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8458
8459 predicate(needs_acquiring_load_exclusive(n));
8460 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8461 ins_cost(VOLATILE_REF_COST);
8462
8463 effect(KILL cr);
8464
8465 format %{
8466 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8467 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8468 %}
8469
8470 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8471 aarch64_enc_cset_eq(res));
8472
8473 ins_pipe(pipe_slow);
8474 %}
8475
8476 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8477
8478 predicate(needs_acquiring_load_exclusive(n));
8479 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8480 ins_cost(VOLATILE_REF_COST);
8481
8482 effect(KILL cr);
8483
8484 format %{
8485 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8486 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8487 %}
8488
8489 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8490 aarch64_enc_cset_eq(res));
8491
8492 ins_pipe(pipe_slow);
8493 %}
8494
8495 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8496
8497 predicate(needs_acquiring_load_exclusive(n));
8498 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8499 ins_cost(VOLATILE_REF_COST);
8500
8501 effect(KILL cr);
8502
8503 format %{
8504 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8505 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8506 %}
8507
8508 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8509 aarch64_enc_cset_eq(res));
8510
8511 ins_pipe(pipe_slow);
8512 %}
8513
8514 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8515
8516 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8517 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8518 ins_cost(VOLATILE_REF_COST);
8519
8520 effect(KILL cr);
8521
8522 format %{
8523 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8524 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8525 %}
8526
8527 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8528 aarch64_enc_cset_eq(res));
8529
8530 ins_pipe(pipe_slow);
8531 %}
8532
8533 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8534
8535 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8536 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8537 ins_cost(VOLATILE_REF_COST);
8538
8539 effect(KILL cr);
8540
8541 format %{
8542 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8543 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8544 %}
8545
8546 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8547 aarch64_enc_cset_eq(res));
8548
8549 ins_pipe(pipe_slow);
8550 %}
8551
8552
8553 // ---------------------------------------------------------------------
8554
8555 // BEGIN This section of the file is automatically generated. Do not edit --------------
8556
8557 // Sundry CAS operations. Note that release is always true,
8558 // regardless of the memory ordering of the CAS. This is because we
8559 // need the volatile case to be sequentially consistent but there is
8560 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8561 // can't check the type of memory ordering here, so we always emit a
8562 // STLXR.
8563
8564 // This section is generated from cas.m4
8565
8566
8567 // This pattern is generated automatically from cas.m4.
8568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8569 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8570 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8571 ins_cost(2 * VOLATILE_REF_COST);
8572 effect(TEMP_DEF res, KILL cr);
8573 format %{
8574 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8575 %}
8576 ins_encode %{
8577 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8578 Assembler::byte, /*acquire*/ false, /*release*/ true,
8579 /*weak*/ false, $res$$Register);
8580 __ sxtbw($res$$Register, $res$$Register);
8581 %}
8582 ins_pipe(pipe_slow);
8583 %}
8584
8585 // This pattern is generated automatically from cas.m4.
8586 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8587 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8588 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8589 ins_cost(2 * VOLATILE_REF_COST);
8590 effect(TEMP_DEF res, KILL cr);
8591 format %{
8592 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8593 %}
8594 ins_encode %{
8595 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8596 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8597 /*weak*/ false, $res$$Register);
8598 __ sxthw($res$$Register, $res$$Register);
8599 %}
8600 ins_pipe(pipe_slow);
8601 %}
8602
8603 // This pattern is generated automatically from cas.m4.
8604 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8605 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8606 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8607 ins_cost(2 * VOLATILE_REF_COST);
8608 effect(TEMP_DEF res, KILL cr);
8609 format %{
8610 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8611 %}
8612 ins_encode %{
8613 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8614 Assembler::word, /*acquire*/ false, /*release*/ true,
8615 /*weak*/ false, $res$$Register);
8616 %}
8617 ins_pipe(pipe_slow);
8618 %}
8619
8620 // This pattern is generated automatically from cas.m4.
8621 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8622 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8623 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8624 ins_cost(2 * VOLATILE_REF_COST);
8625 effect(TEMP_DEF res, KILL cr);
8626 format %{
8627 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8628 %}
8629 ins_encode %{
8630 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8631 Assembler::xword, /*acquire*/ false, /*release*/ true,
8632 /*weak*/ false, $res$$Register);
8633 %}
8634 ins_pipe(pipe_slow);
8635 %}
8636
8637 // This pattern is generated automatically from cas.m4.
8638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8639 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8640 predicate(n->as_LoadStore()->barrier_data() == 0);
8641 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8642 ins_cost(2 * VOLATILE_REF_COST);
8643 effect(TEMP_DEF res, KILL cr);
8644 format %{
8645 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8646 %}
8647 ins_encode %{
8648 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8649 Assembler::word, /*acquire*/ false, /*release*/ true,
8650 /*weak*/ false, $res$$Register);
8651 %}
8652 ins_pipe(pipe_slow);
8653 %}
8654
8655 // This pattern is generated automatically from cas.m4.
8656 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8657 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8658 predicate(n->as_LoadStore()->barrier_data() == 0);
8659 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8660 ins_cost(2 * VOLATILE_REF_COST);
8661 effect(TEMP_DEF res, KILL cr);
8662 format %{
8663 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8664 %}
8665 ins_encode %{
8666 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8667 Assembler::xword, /*acquire*/ false, /*release*/ true,
8668 /*weak*/ false, $res$$Register);
8669 %}
8670 ins_pipe(pipe_slow);
8671 %}
8672
8673 // This pattern is generated automatically from cas.m4.
8674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8675 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8676 predicate(needs_acquiring_load_exclusive(n));
8677 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8678 ins_cost(VOLATILE_REF_COST);
8679 effect(TEMP_DEF res, KILL cr);
8680 format %{
8681 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8682 %}
8683 ins_encode %{
8684 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8685 Assembler::byte, /*acquire*/ true, /*release*/ true,
8686 /*weak*/ false, $res$$Register);
8687 __ sxtbw($res$$Register, $res$$Register);
8688 %}
8689 ins_pipe(pipe_slow);
8690 %}
8691
8692 // This pattern is generated automatically from cas.m4.
8693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8694 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8695 predicate(needs_acquiring_load_exclusive(n));
8696 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8697 ins_cost(VOLATILE_REF_COST);
8698 effect(TEMP_DEF res, KILL cr);
8699 format %{
8700 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8701 %}
8702 ins_encode %{
8703 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8704 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8705 /*weak*/ false, $res$$Register);
8706 __ sxthw($res$$Register, $res$$Register);
8707 %}
8708 ins_pipe(pipe_slow);
8709 %}
8710
8711 // This pattern is generated automatically from cas.m4.
8712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8713 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8714 predicate(needs_acquiring_load_exclusive(n));
8715 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8716 ins_cost(VOLATILE_REF_COST);
8717 effect(TEMP_DEF res, KILL cr);
8718 format %{
8719 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8720 %}
8721 ins_encode %{
8722 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8723 Assembler::word, /*acquire*/ true, /*release*/ true,
8724 /*weak*/ false, $res$$Register);
8725 %}
8726 ins_pipe(pipe_slow);
8727 %}
8728
8729 // This pattern is generated automatically from cas.m4.
8730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8731 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8732 predicate(needs_acquiring_load_exclusive(n));
8733 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8734 ins_cost(VOLATILE_REF_COST);
8735 effect(TEMP_DEF res, KILL cr);
8736 format %{
8737 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8738 %}
8739 ins_encode %{
8740 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8741 Assembler::xword, /*acquire*/ true, /*release*/ true,
8742 /*weak*/ false, $res$$Register);
8743 %}
8744 ins_pipe(pipe_slow);
8745 %}
8746
8747 // This pattern is generated automatically from cas.m4.
8748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8749 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8750 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8751 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8752 ins_cost(VOLATILE_REF_COST);
8753 effect(TEMP_DEF res, KILL cr);
8754 format %{
8755 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8756 %}
8757 ins_encode %{
8758 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8759 Assembler::word, /*acquire*/ true, /*release*/ true,
8760 /*weak*/ false, $res$$Register);
8761 %}
8762 ins_pipe(pipe_slow);
8763 %}
8764
8765 // This pattern is generated automatically from cas.m4.
8766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8767 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8768 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8769 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8770 ins_cost(VOLATILE_REF_COST);
8771 effect(TEMP_DEF res, KILL cr);
8772 format %{
8773 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8774 %}
8775 ins_encode %{
8776 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8777 Assembler::xword, /*acquire*/ true, /*release*/ true,
8778 /*weak*/ false, $res$$Register);
8779 %}
8780 ins_pipe(pipe_slow);
8781 %}
8782
8783 // This pattern is generated automatically from cas.m4.
8784 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8785 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8786 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8787 ins_cost(2 * VOLATILE_REF_COST);
8788 effect(KILL cr);
8789 format %{
8790 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8791 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8792 %}
8793 ins_encode %{
8794 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8795 Assembler::byte, /*acquire*/ false, /*release*/ true,
8796 /*weak*/ true, noreg);
8797 __ csetw($res$$Register, Assembler::EQ);
8798 %}
8799 ins_pipe(pipe_slow);
8800 %}
8801
8802 // This pattern is generated automatically from cas.m4.
8803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8804 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8805 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8806 ins_cost(2 * VOLATILE_REF_COST);
8807 effect(KILL cr);
8808 format %{
8809 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8810 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8811 %}
8812 ins_encode %{
8813 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8814 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8815 /*weak*/ true, noreg);
8816 __ csetw($res$$Register, Assembler::EQ);
8817 %}
8818 ins_pipe(pipe_slow);
8819 %}
8820
8821 // This pattern is generated automatically from cas.m4.
8822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8823 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8824 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8825 ins_cost(2 * VOLATILE_REF_COST);
8826 effect(KILL cr);
8827 format %{
8828 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8829 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8830 %}
8831 ins_encode %{
8832 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8833 Assembler::word, /*acquire*/ false, /*release*/ true,
8834 /*weak*/ true, noreg);
8835 __ csetw($res$$Register, Assembler::EQ);
8836 %}
8837 ins_pipe(pipe_slow);
8838 %}
8839
8840 // This pattern is generated automatically from cas.m4.
8841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8842 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8843 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8844 ins_cost(2 * VOLATILE_REF_COST);
8845 effect(KILL cr);
8846 format %{
8847 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8848 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8849 %}
8850 ins_encode %{
8851 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8852 Assembler::xword, /*acquire*/ false, /*release*/ true,
8853 /*weak*/ true, noreg);
8854 __ csetw($res$$Register, Assembler::EQ);
8855 %}
8856 ins_pipe(pipe_slow);
8857 %}
8858
8859 // This pattern is generated automatically from cas.m4.
8860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8861 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8862 predicate(n->as_LoadStore()->barrier_data() == 0);
8863 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8864 ins_cost(2 * VOLATILE_REF_COST);
8865 effect(KILL cr);
8866 format %{
8867 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8868 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8869 %}
8870 ins_encode %{
8871 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8872 Assembler::word, /*acquire*/ false, /*release*/ true,
8873 /*weak*/ true, noreg);
8874 __ csetw($res$$Register, Assembler::EQ);
8875 %}
8876 ins_pipe(pipe_slow);
8877 %}
8878
8879 // This pattern is generated automatically from cas.m4.
8880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8881 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8882 predicate(n->as_LoadStore()->barrier_data() == 0);
8883 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8884 ins_cost(2 * VOLATILE_REF_COST);
8885 effect(KILL cr);
8886 format %{
8887 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8888 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8889 %}
8890 ins_encode %{
8891 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8892 Assembler::xword, /*acquire*/ false, /*release*/ true,
8893 /*weak*/ true, noreg);
8894 __ csetw($res$$Register, Assembler::EQ);
8895 %}
8896 ins_pipe(pipe_slow);
8897 %}
8898
8899 // This pattern is generated automatically from cas.m4.
8900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8901 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8902 predicate(needs_acquiring_load_exclusive(n));
8903 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8904 ins_cost(VOLATILE_REF_COST);
8905 effect(KILL cr);
8906 format %{
8907 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8908 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8909 %}
8910 ins_encode %{
8911 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8912 Assembler::byte, /*acquire*/ true, /*release*/ true,
8913 /*weak*/ true, noreg);
8914 __ csetw($res$$Register, Assembler::EQ);
8915 %}
8916 ins_pipe(pipe_slow);
8917 %}
8918
8919 // This pattern is generated automatically from cas.m4.
8920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8921 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8922 predicate(needs_acquiring_load_exclusive(n));
8923 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8924 ins_cost(VOLATILE_REF_COST);
8925 effect(KILL cr);
8926 format %{
8927 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8928 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8929 %}
8930 ins_encode %{
8931 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8932 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8933 /*weak*/ true, noreg);
8934 __ csetw($res$$Register, Assembler::EQ);
8935 %}
8936 ins_pipe(pipe_slow);
8937 %}
8938
8939 // This pattern is generated automatically from cas.m4.
8940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8941 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8942 predicate(needs_acquiring_load_exclusive(n));
8943 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8944 ins_cost(VOLATILE_REF_COST);
8945 effect(KILL cr);
8946 format %{
8947 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8948 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8949 %}
8950 ins_encode %{
8951 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8952 Assembler::word, /*acquire*/ true, /*release*/ true,
8953 /*weak*/ true, noreg);
8954 __ csetw($res$$Register, Assembler::EQ);
8955 %}
8956 ins_pipe(pipe_slow);
8957 %}
8958
8959 // This pattern is generated automatically from cas.m4.
8960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8961 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8962 predicate(needs_acquiring_load_exclusive(n));
8963 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8964 ins_cost(VOLATILE_REF_COST);
8965 effect(KILL cr);
8966 format %{
8967 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8968 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8969 %}
8970 ins_encode %{
8971 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8972 Assembler::xword, /*acquire*/ true, /*release*/ true,
8973 /*weak*/ true, noreg);
8974 __ csetw($res$$Register, Assembler::EQ);
8975 %}
8976 ins_pipe(pipe_slow);
8977 %}
8978
8979 // This pattern is generated automatically from cas.m4.
8980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8981 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8982 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8983 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8984 ins_cost(VOLATILE_REF_COST);
8985 effect(KILL cr);
8986 format %{
8987 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8988 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8989 %}
8990 ins_encode %{
8991 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8992 Assembler::word, /*acquire*/ true, /*release*/ true,
8993 /*weak*/ true, noreg);
8994 __ csetw($res$$Register, Assembler::EQ);
8995 %}
8996 ins_pipe(pipe_slow);
8997 %}
8998
8999 // This pattern is generated automatically from cas.m4.
9000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9001 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9002 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9003 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9004 ins_cost(VOLATILE_REF_COST);
9005 effect(KILL cr);
9006 format %{
9007 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9008 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9009 %}
9010 ins_encode %{
9011 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9012 Assembler::xword, /*acquire*/ true, /*release*/ true,
9013 /*weak*/ true, noreg);
9014 __ csetw($res$$Register, Assembler::EQ);
9015 %}
9016 ins_pipe(pipe_slow);
9017 %}
9018
9019 // END This section of the file is automatically generated. Do not edit --------------
9020 // ---------------------------------------------------------------------
9021
9022 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9023 match(Set prev (GetAndSetI mem newv));
9024 ins_cost(2 * VOLATILE_REF_COST);
9025 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9026 ins_encode %{
9027 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9028 %}
9029 ins_pipe(pipe_serial);
9030 %}
9031
9032 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9033 match(Set prev (GetAndSetL mem newv));
9034 ins_cost(2 * VOLATILE_REF_COST);
9035 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9036 ins_encode %{
9037 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9038 %}
9039 ins_pipe(pipe_serial);
9040 %}
9041
9042 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9043 predicate(n->as_LoadStore()->barrier_data() == 0);
9044 match(Set prev (GetAndSetN mem newv));
9045 ins_cost(2 * VOLATILE_REF_COST);
9046 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9047 ins_encode %{
9048 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9049 %}
9050 ins_pipe(pipe_serial);
9051 %}
9052
9053 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9054 predicate(n->as_LoadStore()->barrier_data() == 0);
9055 match(Set prev (GetAndSetP mem newv));
9056 ins_cost(2 * VOLATILE_REF_COST);
9057 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9058 ins_encode %{
9059 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9060 %}
9061 ins_pipe(pipe_serial);
9062 %}
9063
9064 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9065 predicate(needs_acquiring_load_exclusive(n));
9066 match(Set prev (GetAndSetI mem newv));
9067 ins_cost(VOLATILE_REF_COST);
9068 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9069 ins_encode %{
9070 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9071 %}
9072 ins_pipe(pipe_serial);
9073 %}
9074
9075 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9076 predicate(needs_acquiring_load_exclusive(n));
9077 match(Set prev (GetAndSetL mem newv));
9078 ins_cost(VOLATILE_REF_COST);
9079 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9080 ins_encode %{
9081 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9082 %}
9083 ins_pipe(pipe_serial);
9084 %}
9085
9086 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9087 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9088 match(Set prev (GetAndSetN mem newv));
9089 ins_cost(VOLATILE_REF_COST);
9090 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9091 ins_encode %{
9092 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9093 %}
9094 ins_pipe(pipe_serial);
9095 %}
9096
9097 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9098 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9099 match(Set prev (GetAndSetP mem newv));
9100 ins_cost(VOLATILE_REF_COST);
9101 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9102 ins_encode %{
9103 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9104 %}
9105 ins_pipe(pipe_serial);
9106 %}
9107
9108
9109 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9110 match(Set newval (GetAndAddL mem incr));
9111 ins_cost(2 * VOLATILE_REF_COST + 1);
9112 format %{ "get_and_addL $newval, [$mem], $incr" %}
9113 ins_encode %{
9114 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9115 %}
9116 ins_pipe(pipe_serial);
9117 %}
9118
9119 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9120 predicate(n->as_LoadStore()->result_not_used());
9121 match(Set dummy (GetAndAddL mem incr));
9122 ins_cost(2 * VOLATILE_REF_COST);
9123 format %{ "get_and_addL [$mem], $incr" %}
9124 ins_encode %{
9125 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9126 %}
9127 ins_pipe(pipe_serial);
9128 %}
9129
9130 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9131 match(Set newval (GetAndAddL mem incr));
9132 ins_cost(2 * VOLATILE_REF_COST + 1);
9133 format %{ "get_and_addL $newval, [$mem], $incr" %}
9134 ins_encode %{
9135 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9136 %}
9137 ins_pipe(pipe_serial);
9138 %}
9139
9140 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9141 predicate(n->as_LoadStore()->result_not_used());
9142 match(Set dummy (GetAndAddL mem incr));
9143 ins_cost(2 * VOLATILE_REF_COST);
9144 format %{ "get_and_addL [$mem], $incr" %}
9145 ins_encode %{
9146 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9147 %}
9148 ins_pipe(pipe_serial);
9149 %}
9150
9151 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9152 match(Set newval (GetAndAddI mem incr));
9153 ins_cost(2 * VOLATILE_REF_COST + 1);
9154 format %{ "get_and_addI $newval, [$mem], $incr" %}
9155 ins_encode %{
9156 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9157 %}
9158 ins_pipe(pipe_serial);
9159 %}
9160
9161 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9162 predicate(n->as_LoadStore()->result_not_used());
9163 match(Set dummy (GetAndAddI mem incr));
9164 ins_cost(2 * VOLATILE_REF_COST);
9165 format %{ "get_and_addI [$mem], $incr" %}
9166 ins_encode %{
9167 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9168 %}
9169 ins_pipe(pipe_serial);
9170 %}
9171
9172 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9173 match(Set newval (GetAndAddI mem incr));
9174 ins_cost(2 * VOLATILE_REF_COST + 1);
9175 format %{ "get_and_addI $newval, [$mem], $incr" %}
9176 ins_encode %{
9177 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9178 %}
9179 ins_pipe(pipe_serial);
9180 %}
9181
9182 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9183 predicate(n->as_LoadStore()->result_not_used());
9184 match(Set dummy (GetAndAddI mem incr));
9185 ins_cost(2 * VOLATILE_REF_COST);
9186 format %{ "get_and_addI [$mem], $incr" %}
9187 ins_encode %{
9188 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9189 %}
9190 ins_pipe(pipe_serial);
9191 %}
9192
9193 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9194 predicate(needs_acquiring_load_exclusive(n));
9195 match(Set newval (GetAndAddL mem incr));
9196 ins_cost(VOLATILE_REF_COST + 1);
9197 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9198 ins_encode %{
9199 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9200 %}
9201 ins_pipe(pipe_serial);
9202 %}
9203
9204 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9205 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9206 match(Set dummy (GetAndAddL mem incr));
9207 ins_cost(VOLATILE_REF_COST);
9208 format %{ "get_and_addL_acq [$mem], $incr" %}
9209 ins_encode %{
9210 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9211 %}
9212 ins_pipe(pipe_serial);
9213 %}
9214
9215 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9216 predicate(needs_acquiring_load_exclusive(n));
9217 match(Set newval (GetAndAddL mem incr));
9218 ins_cost(VOLATILE_REF_COST + 1);
9219 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9220 ins_encode %{
9221 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9222 %}
9223 ins_pipe(pipe_serial);
9224 %}
9225
9226 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9227 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9228 match(Set dummy (GetAndAddL mem incr));
9229 ins_cost(VOLATILE_REF_COST);
9230 format %{ "get_and_addL_acq [$mem], $incr" %}
9231 ins_encode %{
9232 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9233 %}
9234 ins_pipe(pipe_serial);
9235 %}
9236
9237 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9238 predicate(needs_acquiring_load_exclusive(n));
9239 match(Set newval (GetAndAddI mem incr));
9240 ins_cost(VOLATILE_REF_COST + 1);
9241 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9242 ins_encode %{
9243 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9244 %}
9245 ins_pipe(pipe_serial);
9246 %}
9247
9248 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9249 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9250 match(Set dummy (GetAndAddI mem incr));
9251 ins_cost(VOLATILE_REF_COST);
9252 format %{ "get_and_addI_acq [$mem], $incr" %}
9253 ins_encode %{
9254 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9255 %}
9256 ins_pipe(pipe_serial);
9257 %}
9258
9259 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9260 predicate(needs_acquiring_load_exclusive(n));
9261 match(Set newval (GetAndAddI mem incr));
9262 ins_cost(VOLATILE_REF_COST + 1);
9263 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9264 ins_encode %{
9265 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9266 %}
9267 ins_pipe(pipe_serial);
9268 %}
9269
9270 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9271 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9272 match(Set dummy (GetAndAddI mem incr));
9273 ins_cost(VOLATILE_REF_COST);
9274 format %{ "get_and_addI_acq [$mem], $incr" %}
9275 ins_encode %{
9276 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9277 %}
9278 ins_pipe(pipe_serial);
9279 %}
9280
9281 // Manifest a CmpU result in an integer register.
9282 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9283 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9284 %{
9285 match(Set dst (CmpU3 src1 src2));
9286 effect(KILL flags);
9287
9288 ins_cost(INSN_COST * 3);
9289 format %{
9290 "cmpw $src1, $src2\n\t"
9291 "csetw $dst, ne\n\t"
9292 "cnegw $dst, lo\t# CmpU3(reg)"
9293 %}
9294 ins_encode %{
9295 __ cmpw($src1$$Register, $src2$$Register);
9296 __ csetw($dst$$Register, Assembler::NE);
9297 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9298 %}
9299
9300 ins_pipe(pipe_class_default);
9301 %}
9302
9303 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9304 %{
9305 match(Set dst (CmpU3 src1 src2));
9306 effect(KILL flags);
9307
9308 ins_cost(INSN_COST * 3);
9309 format %{
9310 "subsw zr, $src1, $src2\n\t"
9311 "csetw $dst, ne\n\t"
9312 "cnegw $dst, lo\t# CmpU3(imm)"
9313 %}
9314 ins_encode %{
9315 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9316 __ csetw($dst$$Register, Assembler::NE);
9317 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9318 %}
9319
9320 ins_pipe(pipe_class_default);
9321 %}
9322
9323 // Manifest a CmpUL result in an integer register.
9324 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9325 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9326 %{
9327 match(Set dst (CmpUL3 src1 src2));
9328 effect(KILL flags);
9329
9330 ins_cost(INSN_COST * 3);
9331 format %{
9332 "cmp $src1, $src2\n\t"
9333 "csetw $dst, ne\n\t"
9334 "cnegw $dst, lo\t# CmpUL3(reg)"
9335 %}
9336 ins_encode %{
9337 __ cmp($src1$$Register, $src2$$Register);
9338 __ csetw($dst$$Register, Assembler::NE);
9339 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9340 %}
9341
9342 ins_pipe(pipe_class_default);
9343 %}
9344
9345 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9346 %{
9347 match(Set dst (CmpUL3 src1 src2));
9348 effect(KILL flags);
9349
9350 ins_cost(INSN_COST * 3);
9351 format %{
9352 "subs zr, $src1, $src2\n\t"
9353 "csetw $dst, ne\n\t"
9354 "cnegw $dst, lo\t# CmpUL3(imm)"
9355 %}
9356 ins_encode %{
9357 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9358 __ csetw($dst$$Register, Assembler::NE);
9359 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9360 %}
9361
9362 ins_pipe(pipe_class_default);
9363 %}
9364
9365 // Manifest a CmpL result in an integer register.
9366 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9367 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9368 %{
9369 match(Set dst (CmpL3 src1 src2));
9370 effect(KILL flags);
9371
9372 ins_cost(INSN_COST * 3);
9373 format %{
9374 "cmp $src1, $src2\n\t"
9375 "csetw $dst, ne\n\t"
9376 "cnegw $dst, lt\t# CmpL3(reg)"
9377 %}
9378 ins_encode %{
9379 __ cmp($src1$$Register, $src2$$Register);
9380 __ csetw($dst$$Register, Assembler::NE);
9381 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9382 %}
9383
9384 ins_pipe(pipe_class_default);
9385 %}
9386
9387 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9388 %{
9389 match(Set dst (CmpL3 src1 src2));
9390 effect(KILL flags);
9391
9392 ins_cost(INSN_COST * 3);
9393 format %{
9394 "subs zr, $src1, $src2\n\t"
9395 "csetw $dst, ne\n\t"
9396 "cnegw $dst, lt\t# CmpL3(imm)"
9397 %}
9398 ins_encode %{
9399 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9400 __ csetw($dst$$Register, Assembler::NE);
9401 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9402 %}
9403
9404 ins_pipe(pipe_class_default);
9405 %}
9406
9407 // ============================================================================
9408 // Conditional Move Instructions
9409
9410 // n.b. we have identical rules for both a signed compare op (cmpOp)
9411 // and an unsigned compare op (cmpOpU). it would be nice if we could
9412 // define an op class which merged both inputs and use it to type the
9413 // argument to a single rule. unfortunatelyt his fails because the
9414 // opclass does not live up to the COND_INTER interface of its
9415 // component operands. When the generic code tries to negate the
9416 // operand it ends up running the generci Machoper::negate method
9417 // which throws a ShouldNotHappen. So, we have to provide two flavours
9418 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9419
9420 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9421 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9422
9423 ins_cost(INSN_COST * 2);
9424 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9425
9426 ins_encode %{
9427 __ cselw(as_Register($dst$$reg),
9428 as_Register($src2$$reg),
9429 as_Register($src1$$reg),
9430 (Assembler::Condition)$cmp$$cmpcode);
9431 %}
9432
9433 ins_pipe(icond_reg_reg);
9434 %}
9435
9436 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9437 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9438
9439 ins_cost(INSN_COST * 2);
9440 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9441
9442 ins_encode %{
9443 __ cselw(as_Register($dst$$reg),
9444 as_Register($src2$$reg),
9445 as_Register($src1$$reg),
9446 (Assembler::Condition)$cmp$$cmpcode);
9447 %}
9448
9449 ins_pipe(icond_reg_reg);
9450 %}
9451
9452 // special cases where one arg is zero
9453
9454 // n.b. this is selected in preference to the rule above because it
9455 // avoids loading constant 0 into a source register
9456
9457 // TODO
9458 // we ought only to be able to cull one of these variants as the ideal
9459 // transforms ought always to order the zero consistently (to left/right?)
9460
9461 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9462 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9463
9464 ins_cost(INSN_COST * 2);
9465 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9466
9467 ins_encode %{
9468 __ cselw(as_Register($dst$$reg),
9469 as_Register($src$$reg),
9470 zr,
9471 (Assembler::Condition)$cmp$$cmpcode);
9472 %}
9473
9474 ins_pipe(icond_reg);
9475 %}
9476
9477 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9478 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9479
9480 ins_cost(INSN_COST * 2);
9481 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9482
9483 ins_encode %{
9484 __ cselw(as_Register($dst$$reg),
9485 as_Register($src$$reg),
9486 zr,
9487 (Assembler::Condition)$cmp$$cmpcode);
9488 %}
9489
9490 ins_pipe(icond_reg);
9491 %}
9492
9493 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9494 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9495
9496 ins_cost(INSN_COST * 2);
9497 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9498
9499 ins_encode %{
9500 __ cselw(as_Register($dst$$reg),
9501 zr,
9502 as_Register($src$$reg),
9503 (Assembler::Condition)$cmp$$cmpcode);
9504 %}
9505
9506 ins_pipe(icond_reg);
9507 %}
9508
9509 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9510 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9511
9512 ins_cost(INSN_COST * 2);
9513 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9514
9515 ins_encode %{
9516 __ cselw(as_Register($dst$$reg),
9517 zr,
9518 as_Register($src$$reg),
9519 (Assembler::Condition)$cmp$$cmpcode);
9520 %}
9521
9522 ins_pipe(icond_reg);
9523 %}
9524
9525 // special case for creating a boolean 0 or 1
9526
9527 // n.b. this is selected in preference to the rule above because it
9528 // avoids loading constants 0 and 1 into a source register
9529
9530 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9531 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9532
9533 ins_cost(INSN_COST * 2);
9534 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9535
9536 ins_encode %{
9537 // equivalently
9538 // cset(as_Register($dst$$reg),
9539 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9540 __ csincw(as_Register($dst$$reg),
9541 zr,
9542 zr,
9543 (Assembler::Condition)$cmp$$cmpcode);
9544 %}
9545
9546 ins_pipe(icond_none);
9547 %}
9548
9549 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9550 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9551
9552 ins_cost(INSN_COST * 2);
9553 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9554
9555 ins_encode %{
9556 // equivalently
9557 // cset(as_Register($dst$$reg),
9558 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9559 __ csincw(as_Register($dst$$reg),
9560 zr,
9561 zr,
9562 (Assembler::Condition)$cmp$$cmpcode);
9563 %}
9564
9565 ins_pipe(icond_none);
9566 %}
9567
9568 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9569 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9570
9571 ins_cost(INSN_COST * 2);
9572 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9573
9574 ins_encode %{
9575 __ csel(as_Register($dst$$reg),
9576 as_Register($src2$$reg),
9577 as_Register($src1$$reg),
9578 (Assembler::Condition)$cmp$$cmpcode);
9579 %}
9580
9581 ins_pipe(icond_reg_reg);
9582 %}
9583
9584 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9585 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9586
9587 ins_cost(INSN_COST * 2);
9588 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9589
9590 ins_encode %{
9591 __ csel(as_Register($dst$$reg),
9592 as_Register($src2$$reg),
9593 as_Register($src1$$reg),
9594 (Assembler::Condition)$cmp$$cmpcode);
9595 %}
9596
9597 ins_pipe(icond_reg_reg);
9598 %}
9599
9600 // special cases where one arg is zero
9601
9602 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9603 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9604
9605 ins_cost(INSN_COST * 2);
9606 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9607
9608 ins_encode %{
9609 __ csel(as_Register($dst$$reg),
9610 zr,
9611 as_Register($src$$reg),
9612 (Assembler::Condition)$cmp$$cmpcode);
9613 %}
9614
9615 ins_pipe(icond_reg);
9616 %}
9617
9618 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9619 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9620
9621 ins_cost(INSN_COST * 2);
9622 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9623
9624 ins_encode %{
9625 __ csel(as_Register($dst$$reg),
9626 zr,
9627 as_Register($src$$reg),
9628 (Assembler::Condition)$cmp$$cmpcode);
9629 %}
9630
9631 ins_pipe(icond_reg);
9632 %}
9633
9634 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9635 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9636
9637 ins_cost(INSN_COST * 2);
9638 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9639
9640 ins_encode %{
9641 __ csel(as_Register($dst$$reg),
9642 as_Register($src$$reg),
9643 zr,
9644 (Assembler::Condition)$cmp$$cmpcode);
9645 %}
9646
9647 ins_pipe(icond_reg);
9648 %}
9649
9650 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9651 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9652
9653 ins_cost(INSN_COST * 2);
9654 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9655
9656 ins_encode %{
9657 __ csel(as_Register($dst$$reg),
9658 as_Register($src$$reg),
9659 zr,
9660 (Assembler::Condition)$cmp$$cmpcode);
9661 %}
9662
9663 ins_pipe(icond_reg);
9664 %}
9665
9666 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9667 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9668
9669 ins_cost(INSN_COST * 2);
9670 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9671
9672 ins_encode %{
9673 __ csel(as_Register($dst$$reg),
9674 as_Register($src2$$reg),
9675 as_Register($src1$$reg),
9676 (Assembler::Condition)$cmp$$cmpcode);
9677 %}
9678
9679 ins_pipe(icond_reg_reg);
9680 %}
9681
9682 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9683 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9684
9685 ins_cost(INSN_COST * 2);
9686 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9687
9688 ins_encode %{
9689 __ csel(as_Register($dst$$reg),
9690 as_Register($src2$$reg),
9691 as_Register($src1$$reg),
9692 (Assembler::Condition)$cmp$$cmpcode);
9693 %}
9694
9695 ins_pipe(icond_reg_reg);
9696 %}
9697
9698 // special cases where one arg is zero
9699
9700 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9701 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9702
9703 ins_cost(INSN_COST * 2);
9704 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9705
9706 ins_encode %{
9707 __ csel(as_Register($dst$$reg),
9708 zr,
9709 as_Register($src$$reg),
9710 (Assembler::Condition)$cmp$$cmpcode);
9711 %}
9712
9713 ins_pipe(icond_reg);
9714 %}
9715
9716 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9717 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9718
9719 ins_cost(INSN_COST * 2);
9720 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9721
9722 ins_encode %{
9723 __ csel(as_Register($dst$$reg),
9724 zr,
9725 as_Register($src$$reg),
9726 (Assembler::Condition)$cmp$$cmpcode);
9727 %}
9728
9729 ins_pipe(icond_reg);
9730 %}
9731
9732 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9733 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9734
9735 ins_cost(INSN_COST * 2);
9736 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9737
9738 ins_encode %{
9739 __ csel(as_Register($dst$$reg),
9740 as_Register($src$$reg),
9741 zr,
9742 (Assembler::Condition)$cmp$$cmpcode);
9743 %}
9744
9745 ins_pipe(icond_reg);
9746 %}
9747
9748 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9749 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9750
9751 ins_cost(INSN_COST * 2);
9752 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9753
9754 ins_encode %{
9755 __ csel(as_Register($dst$$reg),
9756 as_Register($src$$reg),
9757 zr,
9758 (Assembler::Condition)$cmp$$cmpcode);
9759 %}
9760
9761 ins_pipe(icond_reg);
9762 %}
9763
9764 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9765 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9766
9767 ins_cost(INSN_COST * 2);
9768 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9769
9770 ins_encode %{
9771 __ cselw(as_Register($dst$$reg),
9772 as_Register($src2$$reg),
9773 as_Register($src1$$reg),
9774 (Assembler::Condition)$cmp$$cmpcode);
9775 %}
9776
9777 ins_pipe(icond_reg_reg);
9778 %}
9779
9780 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9781 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9782
9783 ins_cost(INSN_COST * 2);
9784 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9785
9786 ins_encode %{
9787 __ cselw(as_Register($dst$$reg),
9788 as_Register($src2$$reg),
9789 as_Register($src1$$reg),
9790 (Assembler::Condition)$cmp$$cmpcode);
9791 %}
9792
9793 ins_pipe(icond_reg_reg);
9794 %}
9795
9796 // special cases where one arg is zero
9797
9798 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9799 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9800
9801 ins_cost(INSN_COST * 2);
9802 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9803
9804 ins_encode %{
9805 __ cselw(as_Register($dst$$reg),
9806 zr,
9807 as_Register($src$$reg),
9808 (Assembler::Condition)$cmp$$cmpcode);
9809 %}
9810
9811 ins_pipe(icond_reg);
9812 %}
9813
9814 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9815 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9816
9817 ins_cost(INSN_COST * 2);
9818 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9819
9820 ins_encode %{
9821 __ cselw(as_Register($dst$$reg),
9822 zr,
9823 as_Register($src$$reg),
9824 (Assembler::Condition)$cmp$$cmpcode);
9825 %}
9826
9827 ins_pipe(icond_reg);
9828 %}
9829
9830 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9831 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9832
9833 ins_cost(INSN_COST * 2);
9834 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9835
9836 ins_encode %{
9837 __ cselw(as_Register($dst$$reg),
9838 as_Register($src$$reg),
9839 zr,
9840 (Assembler::Condition)$cmp$$cmpcode);
9841 %}
9842
9843 ins_pipe(icond_reg);
9844 %}
9845
9846 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9847 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9848
9849 ins_cost(INSN_COST * 2);
9850 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9851
9852 ins_encode %{
9853 __ cselw(as_Register($dst$$reg),
9854 as_Register($src$$reg),
9855 zr,
9856 (Assembler::Condition)$cmp$$cmpcode);
9857 %}
9858
9859 ins_pipe(icond_reg);
9860 %}
9861
9862 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9863 %{
9864 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9865
9866 ins_cost(INSN_COST * 3);
9867
9868 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9869 ins_encode %{
9870 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9871 __ fcsels(as_FloatRegister($dst$$reg),
9872 as_FloatRegister($src2$$reg),
9873 as_FloatRegister($src1$$reg),
9874 cond);
9875 %}
9876
9877 ins_pipe(fp_cond_reg_reg_s);
9878 %}
9879
9880 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9881 %{
9882 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9883
9884 ins_cost(INSN_COST * 3);
9885
9886 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9887 ins_encode %{
9888 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9889 __ fcsels(as_FloatRegister($dst$$reg),
9890 as_FloatRegister($src2$$reg),
9891 as_FloatRegister($src1$$reg),
9892 cond);
9893 %}
9894
9895 ins_pipe(fp_cond_reg_reg_s);
9896 %}
9897
9898 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9899 %{
9900 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9901
9902 ins_cost(INSN_COST * 3);
9903
9904 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9905 ins_encode %{
9906 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9907 __ fcseld(as_FloatRegister($dst$$reg),
9908 as_FloatRegister($src2$$reg),
9909 as_FloatRegister($src1$$reg),
9910 cond);
9911 %}
9912
9913 ins_pipe(fp_cond_reg_reg_d);
9914 %}
9915
9916 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9917 %{
9918 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9919
9920 ins_cost(INSN_COST * 3);
9921
9922 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9923 ins_encode %{
9924 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9925 __ fcseld(as_FloatRegister($dst$$reg),
9926 as_FloatRegister($src2$$reg),
9927 as_FloatRegister($src1$$reg),
9928 cond);
9929 %}
9930
9931 ins_pipe(fp_cond_reg_reg_d);
9932 %}
9933
9934 // ============================================================================
9935 // Arithmetic Instructions
9936 //
9937
9938 // Integer Addition
9939
9940 // TODO
9941 // these currently employ operations which do not set CR and hence are
9942 // not flagged as killing CR but we would like to isolate the cases
9943 // where we want to set flags from those where we don't. need to work
9944 // out how to do that.
9945
9946 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9947 match(Set dst (AddI src1 src2));
9948
9949 ins_cost(INSN_COST);
9950 format %{ "addw $dst, $src1, $src2" %}
9951
9952 ins_encode %{
9953 __ addw(as_Register($dst$$reg),
9954 as_Register($src1$$reg),
9955 as_Register($src2$$reg));
9956 %}
9957
9958 ins_pipe(ialu_reg_reg);
9959 %}
9960
9961 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9962 match(Set dst (AddI src1 src2));
9963
9964 ins_cost(INSN_COST);
9965 format %{ "addw $dst, $src1, $src2" %}
9966
9967 // use opcode to indicate that this is an add not a sub
9968 opcode(0x0);
9969
9970 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9971
9972 ins_pipe(ialu_reg_imm);
9973 %}
9974
9975 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9976 match(Set dst (AddI (ConvL2I src1) src2));
9977
9978 ins_cost(INSN_COST);
9979 format %{ "addw $dst, $src1, $src2" %}
9980
9981 // use opcode to indicate that this is an add not a sub
9982 opcode(0x0);
9983
9984 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9985
9986 ins_pipe(ialu_reg_imm);
9987 %}
9988
9989 // Pointer Addition
9990 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9991 match(Set dst (AddP src1 src2));
9992
9993 ins_cost(INSN_COST);
9994 format %{ "add $dst, $src1, $src2\t# ptr" %}
9995
9996 ins_encode %{
9997 __ add(as_Register($dst$$reg),
9998 as_Register($src1$$reg),
9999 as_Register($src2$$reg));
10000 %}
10001
10002 ins_pipe(ialu_reg_reg);
10003 %}
10004
10005 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
10006 match(Set dst (AddP src1 (ConvI2L src2)));
10007
10008 ins_cost(1.9 * INSN_COST);
10009 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10010
10011 ins_encode %{
10012 __ add(as_Register($dst$$reg),
10013 as_Register($src1$$reg),
10014 as_Register($src2$$reg), ext::sxtw);
10015 %}
10016
10017 ins_pipe(ialu_reg_reg);
10018 %}
10019
10020 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10021 match(Set dst (AddP src1 (LShiftL src2 scale)));
10022
10023 ins_cost(1.9 * INSN_COST);
10024 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10025
10026 ins_encode %{
10027 __ lea(as_Register($dst$$reg),
10028 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10029 Address::lsl($scale$$constant)));
10030 %}
10031
10032 ins_pipe(ialu_reg_reg_shift);
10033 %}
10034
10035 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10036 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10037
10038 ins_cost(1.9 * INSN_COST);
10039 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10040
10041 ins_encode %{
10042 __ lea(as_Register($dst$$reg),
10043 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10044 Address::sxtw($scale$$constant)));
10045 %}
10046
10047 ins_pipe(ialu_reg_reg_shift);
10048 %}
10049
10050 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10051 match(Set dst (LShiftL (ConvI2L src) scale));
10052
10053 ins_cost(INSN_COST);
10054 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10055
10056 ins_encode %{
10057 __ sbfiz(as_Register($dst$$reg),
10058 as_Register($src$$reg),
10059 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10060 %}
10061
10062 ins_pipe(ialu_reg_shift);
10063 %}
10064
10065 // Pointer Immediate Addition
10066 // n.b. this needs to be more expensive than using an indirect memory
10067 // operand
10068 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10069 match(Set dst (AddP src1 src2));
10070
10071 ins_cost(INSN_COST);
10072 format %{ "add $dst, $src1, $src2\t# ptr" %}
10073
10074 // use opcode to indicate that this is an add not a sub
10075 opcode(0x0);
10076
10077 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10078
10079 ins_pipe(ialu_reg_imm);
10080 %}
10081
10082 // Long Addition
10083 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10084
10085 match(Set dst (AddL src1 src2));
10086
10087 ins_cost(INSN_COST);
10088 format %{ "add $dst, $src1, $src2" %}
10089
10090 ins_encode %{
10091 __ add(as_Register($dst$$reg),
10092 as_Register($src1$$reg),
10093 as_Register($src2$$reg));
10094 %}
10095
10096 ins_pipe(ialu_reg_reg);
10097 %}
10098
10099 // No constant pool entries requiredLong Immediate Addition.
10100 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10101 match(Set dst (AddL src1 src2));
10102
10103 ins_cost(INSN_COST);
10104 format %{ "add $dst, $src1, $src2" %}
10105
10106 // use opcode to indicate that this is an add not a sub
10107 opcode(0x0);
10108
10109 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10110
10111 ins_pipe(ialu_reg_imm);
10112 %}
10113
10114 // Integer Subtraction
10115 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10116 match(Set dst (SubI src1 src2));
10117
10118 ins_cost(INSN_COST);
10119 format %{ "subw $dst, $src1, $src2" %}
10120
10121 ins_encode %{
10122 __ subw(as_Register($dst$$reg),
10123 as_Register($src1$$reg),
10124 as_Register($src2$$reg));
10125 %}
10126
10127 ins_pipe(ialu_reg_reg);
10128 %}
10129
10130 // Immediate Subtraction
10131 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10132 match(Set dst (SubI src1 src2));
10133
10134 ins_cost(INSN_COST);
10135 format %{ "subw $dst, $src1, $src2" %}
10136
10137 // use opcode to indicate that this is a sub not an add
10138 opcode(0x1);
10139
10140 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10141
10142 ins_pipe(ialu_reg_imm);
10143 %}
10144
10145 // Long Subtraction
10146 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10147
10148 match(Set dst (SubL src1 src2));
10149
10150 ins_cost(INSN_COST);
10151 format %{ "sub $dst, $src1, $src2" %}
10152
10153 ins_encode %{
10154 __ sub(as_Register($dst$$reg),
10155 as_Register($src1$$reg),
10156 as_Register($src2$$reg));
10157 %}
10158
10159 ins_pipe(ialu_reg_reg);
10160 %}
10161
10162 // No constant pool entries requiredLong Immediate Subtraction.
10163 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10164 match(Set dst (SubL src1 src2));
10165
10166 ins_cost(INSN_COST);
10167 format %{ "sub$dst, $src1, $src2" %}
10168
10169 // use opcode to indicate that this is a sub not an add
10170 opcode(0x1);
10171
10172 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10173
10174 ins_pipe(ialu_reg_imm);
10175 %}
10176
10177 // Integer Negation (special case for sub)
10178
10179 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10180 match(Set dst (SubI zero src));
10181
10182 ins_cost(INSN_COST);
10183 format %{ "negw $dst, $src\t# int" %}
10184
10185 ins_encode %{
10186 __ negw(as_Register($dst$$reg),
10187 as_Register($src$$reg));
10188 %}
10189
10190 ins_pipe(ialu_reg);
10191 %}
10192
10193 // Long Negation
10194
10195 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10196 match(Set dst (SubL zero src));
10197
10198 ins_cost(INSN_COST);
10199 format %{ "neg $dst, $src\t# long" %}
10200
10201 ins_encode %{
10202 __ neg(as_Register($dst$$reg),
10203 as_Register($src$$reg));
10204 %}
10205
10206 ins_pipe(ialu_reg);
10207 %}
10208
10209 // Integer Multiply
10210
10211 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10212 match(Set dst (MulI src1 src2));
10213
10214 ins_cost(INSN_COST * 3);
10215 format %{ "mulw $dst, $src1, $src2" %}
10216
10217 ins_encode %{
10218 __ mulw(as_Register($dst$$reg),
10219 as_Register($src1$$reg),
10220 as_Register($src2$$reg));
10221 %}
10222
10223 ins_pipe(imul_reg_reg);
10224 %}
10225
10226 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10227 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10228
10229 ins_cost(INSN_COST * 3);
10230 format %{ "smull $dst, $src1, $src2" %}
10231
10232 ins_encode %{
10233 __ smull(as_Register($dst$$reg),
10234 as_Register($src1$$reg),
10235 as_Register($src2$$reg));
10236 %}
10237
10238 ins_pipe(imul_reg_reg);
10239 %}
10240
10241 // Long Multiply
10242
10243 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10244 match(Set dst (MulL src1 src2));
10245
10246 ins_cost(INSN_COST * 5);
10247 format %{ "mul $dst, $src1, $src2" %}
10248
10249 ins_encode %{
10250 __ mul(as_Register($dst$$reg),
10251 as_Register($src1$$reg),
10252 as_Register($src2$$reg));
10253 %}
10254
10255 ins_pipe(lmul_reg_reg);
10256 %}
10257
10258 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10259 %{
10260 match(Set dst (MulHiL src1 src2));
10261
10262 ins_cost(INSN_COST * 7);
10263 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10264
10265 ins_encode %{
10266 __ smulh(as_Register($dst$$reg),
10267 as_Register($src1$$reg),
10268 as_Register($src2$$reg));
10269 %}
10270
10271 ins_pipe(lmul_reg_reg);
10272 %}
10273
10274 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10275 %{
10276 match(Set dst (UMulHiL src1 src2));
10277
10278 ins_cost(INSN_COST * 7);
10279 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10280
10281 ins_encode %{
10282 __ umulh(as_Register($dst$$reg),
10283 as_Register($src1$$reg),
10284 as_Register($src2$$reg));
10285 %}
10286
10287 ins_pipe(lmul_reg_reg);
10288 %}
10289
10290 // Combined Integer Multiply & Add/Sub
10291
10292 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10293 match(Set dst (AddI src3 (MulI src1 src2)));
10294
10295 ins_cost(INSN_COST * 3);
10296 format %{ "madd $dst, $src1, $src2, $src3" %}
10297
10298 ins_encode %{
10299 __ maddw(as_Register($dst$$reg),
10300 as_Register($src1$$reg),
10301 as_Register($src2$$reg),
10302 as_Register($src3$$reg));
10303 %}
10304
10305 ins_pipe(imac_reg_reg);
10306 %}
10307
10308 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10309 match(Set dst (SubI src3 (MulI src1 src2)));
10310
10311 ins_cost(INSN_COST * 3);
10312 format %{ "msub $dst, $src1, $src2, $src3" %}
10313
10314 ins_encode %{
10315 __ msubw(as_Register($dst$$reg),
10316 as_Register($src1$$reg),
10317 as_Register($src2$$reg),
10318 as_Register($src3$$reg));
10319 %}
10320
10321 ins_pipe(imac_reg_reg);
10322 %}
10323
10324 // Combined Integer Multiply & Neg
10325
10326 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10327 match(Set dst (MulI (SubI zero src1) src2));
10328
10329 ins_cost(INSN_COST * 3);
10330 format %{ "mneg $dst, $src1, $src2" %}
10331
10332 ins_encode %{
10333 __ mnegw(as_Register($dst$$reg),
10334 as_Register($src1$$reg),
10335 as_Register($src2$$reg));
10336 %}
10337
10338 ins_pipe(imac_reg_reg);
10339 %}
10340
10341 // Combined Long Multiply & Add/Sub
10342
10343 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10344 match(Set dst (AddL src3 (MulL src1 src2)));
10345
10346 ins_cost(INSN_COST * 5);
10347 format %{ "madd $dst, $src1, $src2, $src3" %}
10348
10349 ins_encode %{
10350 __ madd(as_Register($dst$$reg),
10351 as_Register($src1$$reg),
10352 as_Register($src2$$reg),
10353 as_Register($src3$$reg));
10354 %}
10355
10356 ins_pipe(lmac_reg_reg);
10357 %}
10358
10359 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10360 match(Set dst (SubL src3 (MulL src1 src2)));
10361
10362 ins_cost(INSN_COST * 5);
10363 format %{ "msub $dst, $src1, $src2, $src3" %}
10364
10365 ins_encode %{
10366 __ msub(as_Register($dst$$reg),
10367 as_Register($src1$$reg),
10368 as_Register($src2$$reg),
10369 as_Register($src3$$reg));
10370 %}
10371
10372 ins_pipe(lmac_reg_reg);
10373 %}
10374
10375 // Combined Long Multiply & Neg
10376
10377 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10378 match(Set dst (MulL (SubL zero src1) src2));
10379
10380 ins_cost(INSN_COST * 5);
10381 format %{ "mneg $dst, $src1, $src2" %}
10382
10383 ins_encode %{
10384 __ mneg(as_Register($dst$$reg),
10385 as_Register($src1$$reg),
10386 as_Register($src2$$reg));
10387 %}
10388
10389 ins_pipe(lmac_reg_reg);
10390 %}
10391
10392 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10393
10394 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10395 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10396
10397 ins_cost(INSN_COST * 3);
10398 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10399
10400 ins_encode %{
10401 __ smaddl(as_Register($dst$$reg),
10402 as_Register($src1$$reg),
10403 as_Register($src2$$reg),
10404 as_Register($src3$$reg));
10405 %}
10406
10407 ins_pipe(imac_reg_reg);
10408 %}
10409
10410 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10411 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10412
10413 ins_cost(INSN_COST * 3);
10414 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10415
10416 ins_encode %{
10417 __ smsubl(as_Register($dst$$reg),
10418 as_Register($src1$$reg),
10419 as_Register($src2$$reg),
10420 as_Register($src3$$reg));
10421 %}
10422
10423 ins_pipe(imac_reg_reg);
10424 %}
10425
10426 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10427 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10428
10429 ins_cost(INSN_COST * 3);
10430 format %{ "smnegl $dst, $src1, $src2" %}
10431
10432 ins_encode %{
10433 __ smnegl(as_Register($dst$$reg),
10434 as_Register($src1$$reg),
10435 as_Register($src2$$reg));
10436 %}
10437
10438 ins_pipe(imac_reg_reg);
10439 %}
10440
10441 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10442
10443 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10444 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10445
10446 ins_cost(INSN_COST * 5);
10447 format %{ "mulw rscratch1, $src1, $src2\n\t"
10448 "maddw $dst, $src3, $src4, rscratch1" %}
10449
10450 ins_encode %{
10451 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10452 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10453
10454 ins_pipe(imac_reg_reg);
10455 %}
10456
10457 // Integer Divide
10458
10459 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10460 match(Set dst (DivI src1 src2));
10461
10462 ins_cost(INSN_COST * 19);
10463 format %{ "sdivw $dst, $src1, $src2" %}
10464
10465 ins_encode(aarch64_enc_divw(dst, src1, src2));
10466 ins_pipe(idiv_reg_reg);
10467 %}
10468
10469 // Long Divide
10470
10471 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10472 match(Set dst (DivL src1 src2));
10473
10474 ins_cost(INSN_COST * 35);
10475 format %{ "sdiv $dst, $src1, $src2" %}
10476
10477 ins_encode(aarch64_enc_div(dst, src1, src2));
10478 ins_pipe(ldiv_reg_reg);
10479 %}
10480
10481 // Integer Remainder
10482
10483 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10484 match(Set dst (ModI src1 src2));
10485
10486 ins_cost(INSN_COST * 22);
10487 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10488 "msubw $dst, rscratch1, $src2, $src1" %}
10489
10490 ins_encode(aarch64_enc_modw(dst, src1, src2));
10491 ins_pipe(idiv_reg_reg);
10492 %}
10493
10494 // Long Remainder
10495
10496 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10497 match(Set dst (ModL src1 src2));
10498
10499 ins_cost(INSN_COST * 38);
10500 format %{ "sdiv rscratch1, $src1, $src2\n"
10501 "msub $dst, rscratch1, $src2, $src1" %}
10502
10503 ins_encode(aarch64_enc_mod(dst, src1, src2));
10504 ins_pipe(ldiv_reg_reg);
10505 %}
10506
10507 // Unsigned Integer Divide
10508
10509 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10510 match(Set dst (UDivI src1 src2));
10511
10512 ins_cost(INSN_COST * 19);
10513 format %{ "udivw $dst, $src1, $src2" %}
10514
10515 ins_encode %{
10516 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10517 %}
10518
10519 ins_pipe(idiv_reg_reg);
10520 %}
10521
10522 // Unsigned Long Divide
10523
10524 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10525 match(Set dst (UDivL src1 src2));
10526
10527 ins_cost(INSN_COST * 35);
10528 format %{ "udiv $dst, $src1, $src2" %}
10529
10530 ins_encode %{
10531 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10532 %}
10533
10534 ins_pipe(ldiv_reg_reg);
10535 %}
10536
10537 // Unsigned Integer Remainder
10538
10539 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10540 match(Set dst (UModI src1 src2));
10541
10542 ins_cost(INSN_COST * 22);
10543 format %{ "udivw rscratch1, $src1, $src2\n\t"
10544 "msubw $dst, rscratch1, $src2, $src1" %}
10545
10546 ins_encode %{
10547 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10548 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10549 %}
10550
10551 ins_pipe(idiv_reg_reg);
10552 %}
10553
10554 // Unsigned Long Remainder
10555
10556 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10557 match(Set dst (UModL src1 src2));
10558
10559 ins_cost(INSN_COST * 38);
10560 format %{ "udiv rscratch1, $src1, $src2\n"
10561 "msub $dst, rscratch1, $src2, $src1" %}
10562
10563 ins_encode %{
10564 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10565 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10566 %}
10567
10568 ins_pipe(ldiv_reg_reg);
10569 %}
10570
10571 // Integer Shifts
10572
10573 // Shift Left Register
10574 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10575 match(Set dst (LShiftI src1 src2));
10576
10577 ins_cost(INSN_COST * 2);
10578 format %{ "lslvw $dst, $src1, $src2" %}
10579
10580 ins_encode %{
10581 __ lslvw(as_Register($dst$$reg),
10582 as_Register($src1$$reg),
10583 as_Register($src2$$reg));
10584 %}
10585
10586 ins_pipe(ialu_reg_reg_vshift);
10587 %}
10588
10589 // Shift Left Immediate
10590 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10591 match(Set dst (LShiftI src1 src2));
10592
10593 ins_cost(INSN_COST);
10594 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10595
10596 ins_encode %{
10597 __ lslw(as_Register($dst$$reg),
10598 as_Register($src1$$reg),
10599 $src2$$constant & 0x1f);
10600 %}
10601
10602 ins_pipe(ialu_reg_shift);
10603 %}
10604
10605 // Shift Right Logical Register
10606 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10607 match(Set dst (URShiftI src1 src2));
10608
10609 ins_cost(INSN_COST * 2);
10610 format %{ "lsrvw $dst, $src1, $src2" %}
10611
10612 ins_encode %{
10613 __ lsrvw(as_Register($dst$$reg),
10614 as_Register($src1$$reg),
10615 as_Register($src2$$reg));
10616 %}
10617
10618 ins_pipe(ialu_reg_reg_vshift);
10619 %}
10620
10621 // Shift Right Logical Immediate
10622 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10623 match(Set dst (URShiftI src1 src2));
10624
10625 ins_cost(INSN_COST);
10626 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10627
10628 ins_encode %{
10629 __ lsrw(as_Register($dst$$reg),
10630 as_Register($src1$$reg),
10631 $src2$$constant & 0x1f);
10632 %}
10633
10634 ins_pipe(ialu_reg_shift);
10635 %}
10636
10637 // Shift Right Arithmetic Register
10638 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10639 match(Set dst (RShiftI src1 src2));
10640
10641 ins_cost(INSN_COST * 2);
10642 format %{ "asrvw $dst, $src1, $src2" %}
10643
10644 ins_encode %{
10645 __ asrvw(as_Register($dst$$reg),
10646 as_Register($src1$$reg),
10647 as_Register($src2$$reg));
10648 %}
10649
10650 ins_pipe(ialu_reg_reg_vshift);
10651 %}
10652
10653 // Shift Right Arithmetic Immediate
10654 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10655 match(Set dst (RShiftI src1 src2));
10656
10657 ins_cost(INSN_COST);
10658 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10659
10660 ins_encode %{
10661 __ asrw(as_Register($dst$$reg),
10662 as_Register($src1$$reg),
10663 $src2$$constant & 0x1f);
10664 %}
10665
10666 ins_pipe(ialu_reg_shift);
10667 %}
10668
10669 // Combined Int Mask and Right Shift (using UBFM)
10670 // TODO
10671
10672 // Long Shifts
10673
10674 // Shift Left Register
10675 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10676 match(Set dst (LShiftL src1 src2));
10677
10678 ins_cost(INSN_COST * 2);
10679 format %{ "lslv $dst, $src1, $src2" %}
10680
10681 ins_encode %{
10682 __ lslv(as_Register($dst$$reg),
10683 as_Register($src1$$reg),
10684 as_Register($src2$$reg));
10685 %}
10686
10687 ins_pipe(ialu_reg_reg_vshift);
10688 %}
10689
10690 // Shift Left Immediate
10691 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10692 match(Set dst (LShiftL src1 src2));
10693
10694 ins_cost(INSN_COST);
10695 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10696
10697 ins_encode %{
10698 __ lsl(as_Register($dst$$reg),
10699 as_Register($src1$$reg),
10700 $src2$$constant & 0x3f);
10701 %}
10702
10703 ins_pipe(ialu_reg_shift);
10704 %}
10705
10706 // Shift Right Logical Register
10707 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10708 match(Set dst (URShiftL src1 src2));
10709
10710 ins_cost(INSN_COST * 2);
10711 format %{ "lsrv $dst, $src1, $src2" %}
10712
10713 ins_encode %{
10714 __ lsrv(as_Register($dst$$reg),
10715 as_Register($src1$$reg),
10716 as_Register($src2$$reg));
10717 %}
10718
10719 ins_pipe(ialu_reg_reg_vshift);
10720 %}
10721
10722 // Shift Right Logical Immediate
10723 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10724 match(Set dst (URShiftL src1 src2));
10725
10726 ins_cost(INSN_COST);
10727 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10728
10729 ins_encode %{
10730 __ lsr(as_Register($dst$$reg),
10731 as_Register($src1$$reg),
10732 $src2$$constant & 0x3f);
10733 %}
10734
10735 ins_pipe(ialu_reg_shift);
10736 %}
10737
10738 // A special-case pattern for card table stores.
10739 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10740 match(Set dst (URShiftL (CastP2X src1) src2));
10741
10742 ins_cost(INSN_COST);
10743 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10744
10745 ins_encode %{
10746 __ lsr(as_Register($dst$$reg),
10747 as_Register($src1$$reg),
10748 $src2$$constant & 0x3f);
10749 %}
10750
10751 ins_pipe(ialu_reg_shift);
10752 %}
10753
10754 // Shift Right Arithmetic Register
10755 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10756 match(Set dst (RShiftL src1 src2));
10757
10758 ins_cost(INSN_COST * 2);
10759 format %{ "asrv $dst, $src1, $src2" %}
10760
10761 ins_encode %{
10762 __ asrv(as_Register($dst$$reg),
10763 as_Register($src1$$reg),
10764 as_Register($src2$$reg));
10765 %}
10766
10767 ins_pipe(ialu_reg_reg_vshift);
10768 %}
10769
10770 // Shift Right Arithmetic Immediate
10771 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10772 match(Set dst (RShiftL src1 src2));
10773
10774 ins_cost(INSN_COST);
10775 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10776
10777 ins_encode %{
10778 __ asr(as_Register($dst$$reg),
10779 as_Register($src1$$reg),
10780 $src2$$constant & 0x3f);
10781 %}
10782
10783 ins_pipe(ialu_reg_shift);
10784 %}
10785
10786 // BEGIN This section of the file is automatically generated. Do not edit --------------
10787 // This section is generated from aarch64_ad.m4
10788
10789 // This pattern is automatically generated from aarch64_ad.m4
10790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10791 instruct regL_not_reg(iRegLNoSp dst,
10792 iRegL src1, immL_M1 m1,
10793 rFlagsReg cr) %{
10794 match(Set dst (XorL src1 m1));
10795 ins_cost(INSN_COST);
10796 format %{ "eon $dst, $src1, zr" %}
10797
10798 ins_encode %{
10799 __ eon(as_Register($dst$$reg),
10800 as_Register($src1$$reg),
10801 zr,
10802 Assembler::LSL, 0);
10803 %}
10804
10805 ins_pipe(ialu_reg);
10806 %}
10807
10808 // This pattern is automatically generated from aarch64_ad.m4
10809 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10810 instruct regI_not_reg(iRegINoSp dst,
10811 iRegIorL2I src1, immI_M1 m1,
10812 rFlagsReg cr) %{
10813 match(Set dst (XorI src1 m1));
10814 ins_cost(INSN_COST);
10815 format %{ "eonw $dst, $src1, zr" %}
10816
10817 ins_encode %{
10818 __ eonw(as_Register($dst$$reg),
10819 as_Register($src1$$reg),
10820 zr,
10821 Assembler::LSL, 0);
10822 %}
10823
10824 ins_pipe(ialu_reg);
10825 %}
10826
10827 // This pattern is automatically generated from aarch64_ad.m4
10828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10829 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10830 immI0 zero, iRegIorL2I src1, immI src2) %{
10831 match(Set dst (SubI zero (URShiftI src1 src2)));
10832
10833 ins_cost(1.9 * INSN_COST);
10834 format %{ "negw $dst, $src1, LSR $src2" %}
10835
10836 ins_encode %{
10837 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10838 Assembler::LSR, $src2$$constant & 0x1f);
10839 %}
10840
10841 ins_pipe(ialu_reg_shift);
10842 %}
10843
10844 // This pattern is automatically generated from aarch64_ad.m4
10845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10846 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10847 immI0 zero, iRegIorL2I src1, immI src2) %{
10848 match(Set dst (SubI zero (RShiftI src1 src2)));
10849
10850 ins_cost(1.9 * INSN_COST);
10851 format %{ "negw $dst, $src1, ASR $src2" %}
10852
10853 ins_encode %{
10854 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10855 Assembler::ASR, $src2$$constant & 0x1f);
10856 %}
10857
10858 ins_pipe(ialu_reg_shift);
10859 %}
10860
10861 // This pattern is automatically generated from aarch64_ad.m4
10862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10863 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10864 immI0 zero, iRegIorL2I src1, immI src2) %{
10865 match(Set dst (SubI zero (LShiftI src1 src2)));
10866
10867 ins_cost(1.9 * INSN_COST);
10868 format %{ "negw $dst, $src1, LSL $src2" %}
10869
10870 ins_encode %{
10871 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10872 Assembler::LSL, $src2$$constant & 0x1f);
10873 %}
10874
10875 ins_pipe(ialu_reg_shift);
10876 %}
10877
10878 // This pattern is automatically generated from aarch64_ad.m4
10879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10880 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10881 immL0 zero, iRegL src1, immI src2) %{
10882 match(Set dst (SubL zero (URShiftL src1 src2)));
10883
10884 ins_cost(1.9 * INSN_COST);
10885 format %{ "neg $dst, $src1, LSR $src2" %}
10886
10887 ins_encode %{
10888 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10889 Assembler::LSR, $src2$$constant & 0x3f);
10890 %}
10891
10892 ins_pipe(ialu_reg_shift);
10893 %}
10894
10895 // This pattern is automatically generated from aarch64_ad.m4
10896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10897 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10898 immL0 zero, iRegL src1, immI src2) %{
10899 match(Set dst (SubL zero (RShiftL src1 src2)));
10900
10901 ins_cost(1.9 * INSN_COST);
10902 format %{ "neg $dst, $src1, ASR $src2" %}
10903
10904 ins_encode %{
10905 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10906 Assembler::ASR, $src2$$constant & 0x3f);
10907 %}
10908
10909 ins_pipe(ialu_reg_shift);
10910 %}
10911
10912 // This pattern is automatically generated from aarch64_ad.m4
10913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10914 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10915 immL0 zero, iRegL src1, immI src2) %{
10916 match(Set dst (SubL zero (LShiftL src1 src2)));
10917
10918 ins_cost(1.9 * INSN_COST);
10919 format %{ "neg $dst, $src1, LSL $src2" %}
10920
10921 ins_encode %{
10922 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10923 Assembler::LSL, $src2$$constant & 0x3f);
10924 %}
10925
10926 ins_pipe(ialu_reg_shift);
10927 %}
10928
10929 // This pattern is automatically generated from aarch64_ad.m4
10930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10931 instruct AndI_reg_not_reg(iRegINoSp dst,
10932 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10933 match(Set dst (AndI src1 (XorI src2 m1)));
10934 ins_cost(INSN_COST);
10935 format %{ "bicw $dst, $src1, $src2" %}
10936
10937 ins_encode %{
10938 __ bicw(as_Register($dst$$reg),
10939 as_Register($src1$$reg),
10940 as_Register($src2$$reg),
10941 Assembler::LSL, 0);
10942 %}
10943
10944 ins_pipe(ialu_reg_reg);
10945 %}
10946
10947 // This pattern is automatically generated from aarch64_ad.m4
10948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10949 instruct AndL_reg_not_reg(iRegLNoSp dst,
10950 iRegL src1, iRegL src2, immL_M1 m1) %{
10951 match(Set dst (AndL src1 (XorL src2 m1)));
10952 ins_cost(INSN_COST);
10953 format %{ "bic $dst, $src1, $src2" %}
10954
10955 ins_encode %{
10956 __ bic(as_Register($dst$$reg),
10957 as_Register($src1$$reg),
10958 as_Register($src2$$reg),
10959 Assembler::LSL, 0);
10960 %}
10961
10962 ins_pipe(ialu_reg_reg);
10963 %}
10964
10965 // This pattern is automatically generated from aarch64_ad.m4
10966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10967 instruct OrI_reg_not_reg(iRegINoSp dst,
10968 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10969 match(Set dst (OrI src1 (XorI src2 m1)));
10970 ins_cost(INSN_COST);
10971 format %{ "ornw $dst, $src1, $src2" %}
10972
10973 ins_encode %{
10974 __ ornw(as_Register($dst$$reg),
10975 as_Register($src1$$reg),
10976 as_Register($src2$$reg),
10977 Assembler::LSL, 0);
10978 %}
10979
10980 ins_pipe(ialu_reg_reg);
10981 %}
10982
10983 // This pattern is automatically generated from aarch64_ad.m4
10984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10985 instruct OrL_reg_not_reg(iRegLNoSp dst,
10986 iRegL src1, iRegL src2, immL_M1 m1) %{
10987 match(Set dst (OrL src1 (XorL src2 m1)));
10988 ins_cost(INSN_COST);
10989 format %{ "orn $dst, $src1, $src2" %}
10990
10991 ins_encode %{
10992 __ orn(as_Register($dst$$reg),
10993 as_Register($src1$$reg),
10994 as_Register($src2$$reg),
10995 Assembler::LSL, 0);
10996 %}
10997
10998 ins_pipe(ialu_reg_reg);
10999 %}
11000
11001 // This pattern is automatically generated from aarch64_ad.m4
11002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11003 instruct XorI_reg_not_reg(iRegINoSp dst,
11004 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11005 match(Set dst (XorI m1 (XorI src2 src1)));
11006 ins_cost(INSN_COST);
11007 format %{ "eonw $dst, $src1, $src2" %}
11008
11009 ins_encode %{
11010 __ eonw(as_Register($dst$$reg),
11011 as_Register($src1$$reg),
11012 as_Register($src2$$reg),
11013 Assembler::LSL, 0);
11014 %}
11015
11016 ins_pipe(ialu_reg_reg);
11017 %}
11018
11019 // This pattern is automatically generated from aarch64_ad.m4
11020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11021 instruct XorL_reg_not_reg(iRegLNoSp dst,
11022 iRegL src1, iRegL src2, immL_M1 m1) %{
11023 match(Set dst (XorL m1 (XorL src2 src1)));
11024 ins_cost(INSN_COST);
11025 format %{ "eon $dst, $src1, $src2" %}
11026
11027 ins_encode %{
11028 __ eon(as_Register($dst$$reg),
11029 as_Register($src1$$reg),
11030 as_Register($src2$$reg),
11031 Assembler::LSL, 0);
11032 %}
11033
11034 ins_pipe(ialu_reg_reg);
11035 %}
11036
11037 // This pattern is automatically generated from aarch64_ad.m4
11038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11039 // val & (-1 ^ (val >>> shift)) ==> bicw
11040 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11041 iRegIorL2I src1, iRegIorL2I src2,
11042 immI src3, immI_M1 src4) %{
11043 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11044 ins_cost(1.9 * INSN_COST);
11045 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11046
11047 ins_encode %{
11048 __ bicw(as_Register($dst$$reg),
11049 as_Register($src1$$reg),
11050 as_Register($src2$$reg),
11051 Assembler::LSR,
11052 $src3$$constant & 0x1f);
11053 %}
11054
11055 ins_pipe(ialu_reg_reg_shift);
11056 %}
11057
11058 // This pattern is automatically generated from aarch64_ad.m4
11059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11060 // val & (-1 ^ (val >>> shift)) ==> bic
11061 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11062 iRegL src1, iRegL src2,
11063 immI src3, immL_M1 src4) %{
11064 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11065 ins_cost(1.9 * INSN_COST);
11066 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11067
11068 ins_encode %{
11069 __ bic(as_Register($dst$$reg),
11070 as_Register($src1$$reg),
11071 as_Register($src2$$reg),
11072 Assembler::LSR,
11073 $src3$$constant & 0x3f);
11074 %}
11075
11076 ins_pipe(ialu_reg_reg_shift);
11077 %}
11078
11079 // This pattern is automatically generated from aarch64_ad.m4
11080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11081 // val & (-1 ^ (val >> shift)) ==> bicw
11082 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11083 iRegIorL2I src1, iRegIorL2I src2,
11084 immI src3, immI_M1 src4) %{
11085 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11086 ins_cost(1.9 * INSN_COST);
11087 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11088
11089 ins_encode %{
11090 __ bicw(as_Register($dst$$reg),
11091 as_Register($src1$$reg),
11092 as_Register($src2$$reg),
11093 Assembler::ASR,
11094 $src3$$constant & 0x1f);
11095 %}
11096
11097 ins_pipe(ialu_reg_reg_shift);
11098 %}
11099
11100 // This pattern is automatically generated from aarch64_ad.m4
11101 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11102 // val & (-1 ^ (val >> shift)) ==> bic
11103 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11104 iRegL src1, iRegL src2,
11105 immI src3, immL_M1 src4) %{
11106 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11107 ins_cost(1.9 * INSN_COST);
11108 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11109
11110 ins_encode %{
11111 __ bic(as_Register($dst$$reg),
11112 as_Register($src1$$reg),
11113 as_Register($src2$$reg),
11114 Assembler::ASR,
11115 $src3$$constant & 0x3f);
11116 %}
11117
11118 ins_pipe(ialu_reg_reg_shift);
11119 %}
11120
11121 // This pattern is automatically generated from aarch64_ad.m4
11122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11123 // val & (-1 ^ (val ror shift)) ==> bicw
11124 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11125 iRegIorL2I src1, iRegIorL2I src2,
11126 immI src3, immI_M1 src4) %{
11127 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11128 ins_cost(1.9 * INSN_COST);
11129 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11130
11131 ins_encode %{
11132 __ bicw(as_Register($dst$$reg),
11133 as_Register($src1$$reg),
11134 as_Register($src2$$reg),
11135 Assembler::ROR,
11136 $src3$$constant & 0x1f);
11137 %}
11138
11139 ins_pipe(ialu_reg_reg_shift);
11140 %}
11141
11142 // This pattern is automatically generated from aarch64_ad.m4
11143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11144 // val & (-1 ^ (val ror shift)) ==> bic
11145 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11146 iRegL src1, iRegL src2,
11147 immI src3, immL_M1 src4) %{
11148 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11149 ins_cost(1.9 * INSN_COST);
11150 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11151
11152 ins_encode %{
11153 __ bic(as_Register($dst$$reg),
11154 as_Register($src1$$reg),
11155 as_Register($src2$$reg),
11156 Assembler::ROR,
11157 $src3$$constant & 0x3f);
11158 %}
11159
11160 ins_pipe(ialu_reg_reg_shift);
11161 %}
11162
11163 // This pattern is automatically generated from aarch64_ad.m4
11164 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11165 // val & (-1 ^ (val << shift)) ==> bicw
11166 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11167 iRegIorL2I src1, iRegIorL2I src2,
11168 immI src3, immI_M1 src4) %{
11169 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11170 ins_cost(1.9 * INSN_COST);
11171 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11172
11173 ins_encode %{
11174 __ bicw(as_Register($dst$$reg),
11175 as_Register($src1$$reg),
11176 as_Register($src2$$reg),
11177 Assembler::LSL,
11178 $src3$$constant & 0x1f);
11179 %}
11180
11181 ins_pipe(ialu_reg_reg_shift);
11182 %}
11183
11184 // This pattern is automatically generated from aarch64_ad.m4
11185 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11186 // val & (-1 ^ (val << shift)) ==> bic
11187 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11188 iRegL src1, iRegL src2,
11189 immI src3, immL_M1 src4) %{
11190 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11191 ins_cost(1.9 * INSN_COST);
11192 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11193
11194 ins_encode %{
11195 __ bic(as_Register($dst$$reg),
11196 as_Register($src1$$reg),
11197 as_Register($src2$$reg),
11198 Assembler::LSL,
11199 $src3$$constant & 0x3f);
11200 %}
11201
11202 ins_pipe(ialu_reg_reg_shift);
11203 %}
11204
11205 // This pattern is automatically generated from aarch64_ad.m4
11206 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11207 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11208 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11209 iRegIorL2I src1, iRegIorL2I src2,
11210 immI src3, immI_M1 src4) %{
11211 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11212 ins_cost(1.9 * INSN_COST);
11213 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11214
11215 ins_encode %{
11216 __ eonw(as_Register($dst$$reg),
11217 as_Register($src1$$reg),
11218 as_Register($src2$$reg),
11219 Assembler::LSR,
11220 $src3$$constant & 0x1f);
11221 %}
11222
11223 ins_pipe(ialu_reg_reg_shift);
11224 %}
11225
11226 // This pattern is automatically generated from aarch64_ad.m4
11227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11228 // val ^ (-1 ^ (val >>> shift)) ==> eon
11229 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11230 iRegL src1, iRegL src2,
11231 immI src3, immL_M1 src4) %{
11232 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11233 ins_cost(1.9 * INSN_COST);
11234 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11235
11236 ins_encode %{
11237 __ eon(as_Register($dst$$reg),
11238 as_Register($src1$$reg),
11239 as_Register($src2$$reg),
11240 Assembler::LSR,
11241 $src3$$constant & 0x3f);
11242 %}
11243
11244 ins_pipe(ialu_reg_reg_shift);
11245 %}
11246
11247 // This pattern is automatically generated from aarch64_ad.m4
11248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11249 // val ^ (-1 ^ (val >> shift)) ==> eonw
11250 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11251 iRegIorL2I src1, iRegIorL2I src2,
11252 immI src3, immI_M1 src4) %{
11253 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11254 ins_cost(1.9 * INSN_COST);
11255 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11256
11257 ins_encode %{
11258 __ eonw(as_Register($dst$$reg),
11259 as_Register($src1$$reg),
11260 as_Register($src2$$reg),
11261 Assembler::ASR,
11262 $src3$$constant & 0x1f);
11263 %}
11264
11265 ins_pipe(ialu_reg_reg_shift);
11266 %}
11267
11268 // This pattern is automatically generated from aarch64_ad.m4
11269 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11270 // val ^ (-1 ^ (val >> shift)) ==> eon
11271 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11272 iRegL src1, iRegL src2,
11273 immI src3, immL_M1 src4) %{
11274 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11275 ins_cost(1.9 * INSN_COST);
11276 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11277
11278 ins_encode %{
11279 __ eon(as_Register($dst$$reg),
11280 as_Register($src1$$reg),
11281 as_Register($src2$$reg),
11282 Assembler::ASR,
11283 $src3$$constant & 0x3f);
11284 %}
11285
11286 ins_pipe(ialu_reg_reg_shift);
11287 %}
11288
11289 // This pattern is automatically generated from aarch64_ad.m4
11290 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11291 // val ^ (-1 ^ (val ror shift)) ==> eonw
11292 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11293 iRegIorL2I src1, iRegIorL2I src2,
11294 immI src3, immI_M1 src4) %{
11295 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11296 ins_cost(1.9 * INSN_COST);
11297 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11298
11299 ins_encode %{
11300 __ eonw(as_Register($dst$$reg),
11301 as_Register($src1$$reg),
11302 as_Register($src2$$reg),
11303 Assembler::ROR,
11304 $src3$$constant & 0x1f);
11305 %}
11306
11307 ins_pipe(ialu_reg_reg_shift);
11308 %}
11309
11310 // This pattern is automatically generated from aarch64_ad.m4
11311 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11312 // val ^ (-1 ^ (val ror shift)) ==> eon
11313 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11314 iRegL src1, iRegL src2,
11315 immI src3, immL_M1 src4) %{
11316 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11317 ins_cost(1.9 * INSN_COST);
11318 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11319
11320 ins_encode %{
11321 __ eon(as_Register($dst$$reg),
11322 as_Register($src1$$reg),
11323 as_Register($src2$$reg),
11324 Assembler::ROR,
11325 $src3$$constant & 0x3f);
11326 %}
11327
11328 ins_pipe(ialu_reg_reg_shift);
11329 %}
11330
11331 // This pattern is automatically generated from aarch64_ad.m4
11332 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11333 // val ^ (-1 ^ (val << shift)) ==> eonw
11334 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11335 iRegIorL2I src1, iRegIorL2I src2,
11336 immI src3, immI_M1 src4) %{
11337 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11338 ins_cost(1.9 * INSN_COST);
11339 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11340
11341 ins_encode %{
11342 __ eonw(as_Register($dst$$reg),
11343 as_Register($src1$$reg),
11344 as_Register($src2$$reg),
11345 Assembler::LSL,
11346 $src3$$constant & 0x1f);
11347 %}
11348
11349 ins_pipe(ialu_reg_reg_shift);
11350 %}
11351
11352 // This pattern is automatically generated from aarch64_ad.m4
11353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11354 // val ^ (-1 ^ (val << shift)) ==> eon
11355 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11356 iRegL src1, iRegL src2,
11357 immI src3, immL_M1 src4) %{
11358 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11359 ins_cost(1.9 * INSN_COST);
11360 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11361
11362 ins_encode %{
11363 __ eon(as_Register($dst$$reg),
11364 as_Register($src1$$reg),
11365 as_Register($src2$$reg),
11366 Assembler::LSL,
11367 $src3$$constant & 0x3f);
11368 %}
11369
11370 ins_pipe(ialu_reg_reg_shift);
11371 %}
11372
11373 // This pattern is automatically generated from aarch64_ad.m4
11374 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11375 // val | (-1 ^ (val >>> shift)) ==> ornw
11376 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11377 iRegIorL2I src1, iRegIorL2I src2,
11378 immI src3, immI_M1 src4) %{
11379 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11380 ins_cost(1.9 * INSN_COST);
11381 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11382
11383 ins_encode %{
11384 __ ornw(as_Register($dst$$reg),
11385 as_Register($src1$$reg),
11386 as_Register($src2$$reg),
11387 Assembler::LSR,
11388 $src3$$constant & 0x1f);
11389 %}
11390
11391 ins_pipe(ialu_reg_reg_shift);
11392 %}
11393
11394 // This pattern is automatically generated from aarch64_ad.m4
11395 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11396 // val | (-1 ^ (val >>> shift)) ==> orn
11397 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11398 iRegL src1, iRegL src2,
11399 immI src3, immL_M1 src4) %{
11400 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11401 ins_cost(1.9 * INSN_COST);
11402 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11403
11404 ins_encode %{
11405 __ orn(as_Register($dst$$reg),
11406 as_Register($src1$$reg),
11407 as_Register($src2$$reg),
11408 Assembler::LSR,
11409 $src3$$constant & 0x3f);
11410 %}
11411
11412 ins_pipe(ialu_reg_reg_shift);
11413 %}
11414
11415 // This pattern is automatically generated from aarch64_ad.m4
11416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11417 // val | (-1 ^ (val >> shift)) ==> ornw
11418 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11419 iRegIorL2I src1, iRegIorL2I src2,
11420 immI src3, immI_M1 src4) %{
11421 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11422 ins_cost(1.9 * INSN_COST);
11423 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11424
11425 ins_encode %{
11426 __ ornw(as_Register($dst$$reg),
11427 as_Register($src1$$reg),
11428 as_Register($src2$$reg),
11429 Assembler::ASR,
11430 $src3$$constant & 0x1f);
11431 %}
11432
11433 ins_pipe(ialu_reg_reg_shift);
11434 %}
11435
11436 // This pattern is automatically generated from aarch64_ad.m4
11437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11438 // val | (-1 ^ (val >> shift)) ==> orn
11439 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11440 iRegL src1, iRegL src2,
11441 immI src3, immL_M1 src4) %{
11442 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11443 ins_cost(1.9 * INSN_COST);
11444 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11445
11446 ins_encode %{
11447 __ orn(as_Register($dst$$reg),
11448 as_Register($src1$$reg),
11449 as_Register($src2$$reg),
11450 Assembler::ASR,
11451 $src3$$constant & 0x3f);
11452 %}
11453
11454 ins_pipe(ialu_reg_reg_shift);
11455 %}
11456
11457 // This pattern is automatically generated from aarch64_ad.m4
11458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11459 // val | (-1 ^ (val ror shift)) ==> ornw
11460 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11461 iRegIorL2I src1, iRegIorL2I src2,
11462 immI src3, immI_M1 src4) %{
11463 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11464 ins_cost(1.9 * INSN_COST);
11465 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11466
11467 ins_encode %{
11468 __ ornw(as_Register($dst$$reg),
11469 as_Register($src1$$reg),
11470 as_Register($src2$$reg),
11471 Assembler::ROR,
11472 $src3$$constant & 0x1f);
11473 %}
11474
11475 ins_pipe(ialu_reg_reg_shift);
11476 %}
11477
11478 // This pattern is automatically generated from aarch64_ad.m4
11479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11480 // val | (-1 ^ (val ror shift)) ==> orn
11481 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11482 iRegL src1, iRegL src2,
11483 immI src3, immL_M1 src4) %{
11484 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11485 ins_cost(1.9 * INSN_COST);
11486 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11487
11488 ins_encode %{
11489 __ orn(as_Register($dst$$reg),
11490 as_Register($src1$$reg),
11491 as_Register($src2$$reg),
11492 Assembler::ROR,
11493 $src3$$constant & 0x3f);
11494 %}
11495
11496 ins_pipe(ialu_reg_reg_shift);
11497 %}
11498
11499 // This pattern is automatically generated from aarch64_ad.m4
11500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11501 // val | (-1 ^ (val << shift)) ==> ornw
11502 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11503 iRegIorL2I src1, iRegIorL2I src2,
11504 immI src3, immI_M1 src4) %{
11505 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11506 ins_cost(1.9 * INSN_COST);
11507 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11508
11509 ins_encode %{
11510 __ ornw(as_Register($dst$$reg),
11511 as_Register($src1$$reg),
11512 as_Register($src2$$reg),
11513 Assembler::LSL,
11514 $src3$$constant & 0x1f);
11515 %}
11516
11517 ins_pipe(ialu_reg_reg_shift);
11518 %}
11519
11520 // This pattern is automatically generated from aarch64_ad.m4
11521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11522 // val | (-1 ^ (val << shift)) ==> orn
11523 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11524 iRegL src1, iRegL src2,
11525 immI src3, immL_M1 src4) %{
11526 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11527 ins_cost(1.9 * INSN_COST);
11528 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11529
11530 ins_encode %{
11531 __ orn(as_Register($dst$$reg),
11532 as_Register($src1$$reg),
11533 as_Register($src2$$reg),
11534 Assembler::LSL,
11535 $src3$$constant & 0x3f);
11536 %}
11537
11538 ins_pipe(ialu_reg_reg_shift);
11539 %}
11540
11541 // This pattern is automatically generated from aarch64_ad.m4
11542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11543 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11544 iRegIorL2I src1, iRegIorL2I src2,
11545 immI src3) %{
11546 match(Set dst (AndI src1 (URShiftI src2 src3)));
11547
11548 ins_cost(1.9 * INSN_COST);
11549 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11550
11551 ins_encode %{
11552 __ andw(as_Register($dst$$reg),
11553 as_Register($src1$$reg),
11554 as_Register($src2$$reg),
11555 Assembler::LSR,
11556 $src3$$constant & 0x1f);
11557 %}
11558
11559 ins_pipe(ialu_reg_reg_shift);
11560 %}
11561
11562 // This pattern is automatically generated from aarch64_ad.m4
11563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11564 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11565 iRegL src1, iRegL src2,
11566 immI src3) %{
11567 match(Set dst (AndL src1 (URShiftL src2 src3)));
11568
11569 ins_cost(1.9 * INSN_COST);
11570 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11571
11572 ins_encode %{
11573 __ andr(as_Register($dst$$reg),
11574 as_Register($src1$$reg),
11575 as_Register($src2$$reg),
11576 Assembler::LSR,
11577 $src3$$constant & 0x3f);
11578 %}
11579
11580 ins_pipe(ialu_reg_reg_shift);
11581 %}
11582
11583 // This pattern is automatically generated from aarch64_ad.m4
11584 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11585 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11586 iRegIorL2I src1, iRegIorL2I src2,
11587 immI src3) %{
11588 match(Set dst (AndI src1 (RShiftI src2 src3)));
11589
11590 ins_cost(1.9 * INSN_COST);
11591 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11592
11593 ins_encode %{
11594 __ andw(as_Register($dst$$reg),
11595 as_Register($src1$$reg),
11596 as_Register($src2$$reg),
11597 Assembler::ASR,
11598 $src3$$constant & 0x1f);
11599 %}
11600
11601 ins_pipe(ialu_reg_reg_shift);
11602 %}
11603
11604 // This pattern is automatically generated from aarch64_ad.m4
11605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11606 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11607 iRegL src1, iRegL src2,
11608 immI src3) %{
11609 match(Set dst (AndL src1 (RShiftL src2 src3)));
11610
11611 ins_cost(1.9 * INSN_COST);
11612 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11613
11614 ins_encode %{
11615 __ andr(as_Register($dst$$reg),
11616 as_Register($src1$$reg),
11617 as_Register($src2$$reg),
11618 Assembler::ASR,
11619 $src3$$constant & 0x3f);
11620 %}
11621
11622 ins_pipe(ialu_reg_reg_shift);
11623 %}
11624
11625 // This pattern is automatically generated from aarch64_ad.m4
11626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11627 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11628 iRegIorL2I src1, iRegIorL2I src2,
11629 immI src3) %{
11630 match(Set dst (AndI src1 (LShiftI src2 src3)));
11631
11632 ins_cost(1.9 * INSN_COST);
11633 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11634
11635 ins_encode %{
11636 __ andw(as_Register($dst$$reg),
11637 as_Register($src1$$reg),
11638 as_Register($src2$$reg),
11639 Assembler::LSL,
11640 $src3$$constant & 0x1f);
11641 %}
11642
11643 ins_pipe(ialu_reg_reg_shift);
11644 %}
11645
11646 // This pattern is automatically generated from aarch64_ad.m4
11647 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11648 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11649 iRegL src1, iRegL src2,
11650 immI src3) %{
11651 match(Set dst (AndL src1 (LShiftL src2 src3)));
11652
11653 ins_cost(1.9 * INSN_COST);
11654 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11655
11656 ins_encode %{
11657 __ andr(as_Register($dst$$reg),
11658 as_Register($src1$$reg),
11659 as_Register($src2$$reg),
11660 Assembler::LSL,
11661 $src3$$constant & 0x3f);
11662 %}
11663
11664 ins_pipe(ialu_reg_reg_shift);
11665 %}
11666
11667 // This pattern is automatically generated from aarch64_ad.m4
11668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11669 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11670 iRegIorL2I src1, iRegIorL2I src2,
11671 immI src3) %{
11672 match(Set dst (AndI src1 (RotateRight src2 src3)));
11673
11674 ins_cost(1.9 * INSN_COST);
11675 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11676
11677 ins_encode %{
11678 __ andw(as_Register($dst$$reg),
11679 as_Register($src1$$reg),
11680 as_Register($src2$$reg),
11681 Assembler::ROR,
11682 $src3$$constant & 0x1f);
11683 %}
11684
11685 ins_pipe(ialu_reg_reg_shift);
11686 %}
11687
11688 // This pattern is automatically generated from aarch64_ad.m4
11689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11690 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11691 iRegL src1, iRegL src2,
11692 immI src3) %{
11693 match(Set dst (AndL src1 (RotateRight src2 src3)));
11694
11695 ins_cost(1.9 * INSN_COST);
11696 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11697
11698 ins_encode %{
11699 __ andr(as_Register($dst$$reg),
11700 as_Register($src1$$reg),
11701 as_Register($src2$$reg),
11702 Assembler::ROR,
11703 $src3$$constant & 0x3f);
11704 %}
11705
11706 ins_pipe(ialu_reg_reg_shift);
11707 %}
11708
11709 // This pattern is automatically generated from aarch64_ad.m4
11710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11711 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11712 iRegIorL2I src1, iRegIorL2I src2,
11713 immI src3) %{
11714 match(Set dst (XorI src1 (URShiftI src2 src3)));
11715
11716 ins_cost(1.9 * INSN_COST);
11717 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11718
11719 ins_encode %{
11720 __ eorw(as_Register($dst$$reg),
11721 as_Register($src1$$reg),
11722 as_Register($src2$$reg),
11723 Assembler::LSR,
11724 $src3$$constant & 0x1f);
11725 %}
11726
11727 ins_pipe(ialu_reg_reg_shift);
11728 %}
11729
11730 // This pattern is automatically generated from aarch64_ad.m4
11731 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11732 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11733 iRegL src1, iRegL src2,
11734 immI src3) %{
11735 match(Set dst (XorL src1 (URShiftL src2 src3)));
11736
11737 ins_cost(1.9 * INSN_COST);
11738 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11739
11740 ins_encode %{
11741 __ eor(as_Register($dst$$reg),
11742 as_Register($src1$$reg),
11743 as_Register($src2$$reg),
11744 Assembler::LSR,
11745 $src3$$constant & 0x3f);
11746 %}
11747
11748 ins_pipe(ialu_reg_reg_shift);
11749 %}
11750
11751 // This pattern is automatically generated from aarch64_ad.m4
11752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11753 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11754 iRegIorL2I src1, iRegIorL2I src2,
11755 immI src3) %{
11756 match(Set dst (XorI src1 (RShiftI src2 src3)));
11757
11758 ins_cost(1.9 * INSN_COST);
11759 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11760
11761 ins_encode %{
11762 __ eorw(as_Register($dst$$reg),
11763 as_Register($src1$$reg),
11764 as_Register($src2$$reg),
11765 Assembler::ASR,
11766 $src3$$constant & 0x1f);
11767 %}
11768
11769 ins_pipe(ialu_reg_reg_shift);
11770 %}
11771
11772 // This pattern is automatically generated from aarch64_ad.m4
11773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11774 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11775 iRegL src1, iRegL src2,
11776 immI src3) %{
11777 match(Set dst (XorL src1 (RShiftL src2 src3)));
11778
11779 ins_cost(1.9 * INSN_COST);
11780 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11781
11782 ins_encode %{
11783 __ eor(as_Register($dst$$reg),
11784 as_Register($src1$$reg),
11785 as_Register($src2$$reg),
11786 Assembler::ASR,
11787 $src3$$constant & 0x3f);
11788 %}
11789
11790 ins_pipe(ialu_reg_reg_shift);
11791 %}
11792
11793 // This pattern is automatically generated from aarch64_ad.m4
11794 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11795 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11796 iRegIorL2I src1, iRegIorL2I src2,
11797 immI src3) %{
11798 match(Set dst (XorI src1 (LShiftI src2 src3)));
11799
11800 ins_cost(1.9 * INSN_COST);
11801 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11802
11803 ins_encode %{
11804 __ eorw(as_Register($dst$$reg),
11805 as_Register($src1$$reg),
11806 as_Register($src2$$reg),
11807 Assembler::LSL,
11808 $src3$$constant & 0x1f);
11809 %}
11810
11811 ins_pipe(ialu_reg_reg_shift);
11812 %}
11813
11814 // This pattern is automatically generated from aarch64_ad.m4
11815 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11816 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11817 iRegL src1, iRegL src2,
11818 immI src3) %{
11819 match(Set dst (XorL src1 (LShiftL src2 src3)));
11820
11821 ins_cost(1.9 * INSN_COST);
11822 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11823
11824 ins_encode %{
11825 __ eor(as_Register($dst$$reg),
11826 as_Register($src1$$reg),
11827 as_Register($src2$$reg),
11828 Assembler::LSL,
11829 $src3$$constant & 0x3f);
11830 %}
11831
11832 ins_pipe(ialu_reg_reg_shift);
11833 %}
11834
11835 // This pattern is automatically generated from aarch64_ad.m4
11836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11837 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11838 iRegIorL2I src1, iRegIorL2I src2,
11839 immI src3) %{
11840 match(Set dst (XorI src1 (RotateRight src2 src3)));
11841
11842 ins_cost(1.9 * INSN_COST);
11843 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11844
11845 ins_encode %{
11846 __ eorw(as_Register($dst$$reg),
11847 as_Register($src1$$reg),
11848 as_Register($src2$$reg),
11849 Assembler::ROR,
11850 $src3$$constant & 0x1f);
11851 %}
11852
11853 ins_pipe(ialu_reg_reg_shift);
11854 %}
11855
11856 // This pattern is automatically generated from aarch64_ad.m4
11857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11858 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11859 iRegL src1, iRegL src2,
11860 immI src3) %{
11861 match(Set dst (XorL src1 (RotateRight src2 src3)));
11862
11863 ins_cost(1.9 * INSN_COST);
11864 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11865
11866 ins_encode %{
11867 __ eor(as_Register($dst$$reg),
11868 as_Register($src1$$reg),
11869 as_Register($src2$$reg),
11870 Assembler::ROR,
11871 $src3$$constant & 0x3f);
11872 %}
11873
11874 ins_pipe(ialu_reg_reg_shift);
11875 %}
11876
11877 // This pattern is automatically generated from aarch64_ad.m4
11878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11879 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11880 iRegIorL2I src1, iRegIorL2I src2,
11881 immI src3) %{
11882 match(Set dst (OrI src1 (URShiftI src2 src3)));
11883
11884 ins_cost(1.9 * INSN_COST);
11885 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11886
11887 ins_encode %{
11888 __ orrw(as_Register($dst$$reg),
11889 as_Register($src1$$reg),
11890 as_Register($src2$$reg),
11891 Assembler::LSR,
11892 $src3$$constant & 0x1f);
11893 %}
11894
11895 ins_pipe(ialu_reg_reg_shift);
11896 %}
11897
11898 // This pattern is automatically generated from aarch64_ad.m4
11899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11900 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11901 iRegL src1, iRegL src2,
11902 immI src3) %{
11903 match(Set dst (OrL src1 (URShiftL src2 src3)));
11904
11905 ins_cost(1.9 * INSN_COST);
11906 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11907
11908 ins_encode %{
11909 __ orr(as_Register($dst$$reg),
11910 as_Register($src1$$reg),
11911 as_Register($src2$$reg),
11912 Assembler::LSR,
11913 $src3$$constant & 0x3f);
11914 %}
11915
11916 ins_pipe(ialu_reg_reg_shift);
11917 %}
11918
11919 // This pattern is automatically generated from aarch64_ad.m4
11920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11921 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11922 iRegIorL2I src1, iRegIorL2I src2,
11923 immI src3) %{
11924 match(Set dst (OrI src1 (RShiftI src2 src3)));
11925
11926 ins_cost(1.9 * INSN_COST);
11927 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11928
11929 ins_encode %{
11930 __ orrw(as_Register($dst$$reg),
11931 as_Register($src1$$reg),
11932 as_Register($src2$$reg),
11933 Assembler::ASR,
11934 $src3$$constant & 0x1f);
11935 %}
11936
11937 ins_pipe(ialu_reg_reg_shift);
11938 %}
11939
11940 // This pattern is automatically generated from aarch64_ad.m4
11941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11942 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11943 iRegL src1, iRegL src2,
11944 immI src3) %{
11945 match(Set dst (OrL src1 (RShiftL src2 src3)));
11946
11947 ins_cost(1.9 * INSN_COST);
11948 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11949
11950 ins_encode %{
11951 __ orr(as_Register($dst$$reg),
11952 as_Register($src1$$reg),
11953 as_Register($src2$$reg),
11954 Assembler::ASR,
11955 $src3$$constant & 0x3f);
11956 %}
11957
11958 ins_pipe(ialu_reg_reg_shift);
11959 %}
11960
11961 // This pattern is automatically generated from aarch64_ad.m4
11962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11963 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11964 iRegIorL2I src1, iRegIorL2I src2,
11965 immI src3) %{
11966 match(Set dst (OrI src1 (LShiftI src2 src3)));
11967
11968 ins_cost(1.9 * INSN_COST);
11969 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11970
11971 ins_encode %{
11972 __ orrw(as_Register($dst$$reg),
11973 as_Register($src1$$reg),
11974 as_Register($src2$$reg),
11975 Assembler::LSL,
11976 $src3$$constant & 0x1f);
11977 %}
11978
11979 ins_pipe(ialu_reg_reg_shift);
11980 %}
11981
11982 // This pattern is automatically generated from aarch64_ad.m4
11983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11984 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11985 iRegL src1, iRegL src2,
11986 immI src3) %{
11987 match(Set dst (OrL src1 (LShiftL src2 src3)));
11988
11989 ins_cost(1.9 * INSN_COST);
11990 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11991
11992 ins_encode %{
11993 __ orr(as_Register($dst$$reg),
11994 as_Register($src1$$reg),
11995 as_Register($src2$$reg),
11996 Assembler::LSL,
11997 $src3$$constant & 0x3f);
11998 %}
11999
12000 ins_pipe(ialu_reg_reg_shift);
12001 %}
12002
12003 // This pattern is automatically generated from aarch64_ad.m4
12004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12005 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12006 iRegIorL2I src1, iRegIorL2I src2,
12007 immI src3) %{
12008 match(Set dst (OrI src1 (RotateRight src2 src3)));
12009
12010 ins_cost(1.9 * INSN_COST);
12011 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12012
12013 ins_encode %{
12014 __ orrw(as_Register($dst$$reg),
12015 as_Register($src1$$reg),
12016 as_Register($src2$$reg),
12017 Assembler::ROR,
12018 $src3$$constant & 0x1f);
12019 %}
12020
12021 ins_pipe(ialu_reg_reg_shift);
12022 %}
12023
12024 // This pattern is automatically generated from aarch64_ad.m4
12025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12026 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12027 iRegL src1, iRegL src2,
12028 immI src3) %{
12029 match(Set dst (OrL src1 (RotateRight src2 src3)));
12030
12031 ins_cost(1.9 * INSN_COST);
12032 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12033
12034 ins_encode %{
12035 __ orr(as_Register($dst$$reg),
12036 as_Register($src1$$reg),
12037 as_Register($src2$$reg),
12038 Assembler::ROR,
12039 $src3$$constant & 0x3f);
12040 %}
12041
12042 ins_pipe(ialu_reg_reg_shift);
12043 %}
12044
12045 // This pattern is automatically generated from aarch64_ad.m4
12046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12047 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12048 iRegIorL2I src1, iRegIorL2I src2,
12049 immI src3) %{
12050 match(Set dst (AddI src1 (URShiftI src2 src3)));
12051
12052 ins_cost(1.9 * INSN_COST);
12053 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12054
12055 ins_encode %{
12056 __ addw(as_Register($dst$$reg),
12057 as_Register($src1$$reg),
12058 as_Register($src2$$reg),
12059 Assembler::LSR,
12060 $src3$$constant & 0x1f);
12061 %}
12062
12063 ins_pipe(ialu_reg_reg_shift);
12064 %}
12065
12066 // This pattern is automatically generated from aarch64_ad.m4
12067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12068 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12069 iRegL src1, iRegL src2,
12070 immI src3) %{
12071 match(Set dst (AddL src1 (URShiftL src2 src3)));
12072
12073 ins_cost(1.9 * INSN_COST);
12074 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12075
12076 ins_encode %{
12077 __ add(as_Register($dst$$reg),
12078 as_Register($src1$$reg),
12079 as_Register($src2$$reg),
12080 Assembler::LSR,
12081 $src3$$constant & 0x3f);
12082 %}
12083
12084 ins_pipe(ialu_reg_reg_shift);
12085 %}
12086
12087 // This pattern is automatically generated from aarch64_ad.m4
12088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12089 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12090 iRegIorL2I src1, iRegIorL2I src2,
12091 immI src3) %{
12092 match(Set dst (AddI src1 (RShiftI src2 src3)));
12093
12094 ins_cost(1.9 * INSN_COST);
12095 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12096
12097 ins_encode %{
12098 __ addw(as_Register($dst$$reg),
12099 as_Register($src1$$reg),
12100 as_Register($src2$$reg),
12101 Assembler::ASR,
12102 $src3$$constant & 0x1f);
12103 %}
12104
12105 ins_pipe(ialu_reg_reg_shift);
12106 %}
12107
12108 // This pattern is automatically generated from aarch64_ad.m4
12109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12110 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12111 iRegL src1, iRegL src2,
12112 immI src3) %{
12113 match(Set dst (AddL src1 (RShiftL src2 src3)));
12114
12115 ins_cost(1.9 * INSN_COST);
12116 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12117
12118 ins_encode %{
12119 __ add(as_Register($dst$$reg),
12120 as_Register($src1$$reg),
12121 as_Register($src2$$reg),
12122 Assembler::ASR,
12123 $src3$$constant & 0x3f);
12124 %}
12125
12126 ins_pipe(ialu_reg_reg_shift);
12127 %}
12128
12129 // This pattern is automatically generated from aarch64_ad.m4
12130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12131 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12132 iRegIorL2I src1, iRegIorL2I src2,
12133 immI src3) %{
12134 match(Set dst (AddI src1 (LShiftI src2 src3)));
12135
12136 ins_cost(1.9 * INSN_COST);
12137 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12138
12139 ins_encode %{
12140 __ addw(as_Register($dst$$reg),
12141 as_Register($src1$$reg),
12142 as_Register($src2$$reg),
12143 Assembler::LSL,
12144 $src3$$constant & 0x1f);
12145 %}
12146
12147 ins_pipe(ialu_reg_reg_shift);
12148 %}
12149
12150 // This pattern is automatically generated from aarch64_ad.m4
12151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12152 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12153 iRegL src1, iRegL src2,
12154 immI src3) %{
12155 match(Set dst (AddL src1 (LShiftL src2 src3)));
12156
12157 ins_cost(1.9 * INSN_COST);
12158 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12159
12160 ins_encode %{
12161 __ add(as_Register($dst$$reg),
12162 as_Register($src1$$reg),
12163 as_Register($src2$$reg),
12164 Assembler::LSL,
12165 $src3$$constant & 0x3f);
12166 %}
12167
12168 ins_pipe(ialu_reg_reg_shift);
12169 %}
12170
12171 // This pattern is automatically generated from aarch64_ad.m4
12172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12173 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12174 iRegIorL2I src1, iRegIorL2I src2,
12175 immI src3) %{
12176 match(Set dst (SubI src1 (URShiftI src2 src3)));
12177
12178 ins_cost(1.9 * INSN_COST);
12179 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12180
12181 ins_encode %{
12182 __ subw(as_Register($dst$$reg),
12183 as_Register($src1$$reg),
12184 as_Register($src2$$reg),
12185 Assembler::LSR,
12186 $src3$$constant & 0x1f);
12187 %}
12188
12189 ins_pipe(ialu_reg_reg_shift);
12190 %}
12191
12192 // This pattern is automatically generated from aarch64_ad.m4
12193 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12194 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12195 iRegL src1, iRegL src2,
12196 immI src3) %{
12197 match(Set dst (SubL src1 (URShiftL src2 src3)));
12198
12199 ins_cost(1.9 * INSN_COST);
12200 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12201
12202 ins_encode %{
12203 __ sub(as_Register($dst$$reg),
12204 as_Register($src1$$reg),
12205 as_Register($src2$$reg),
12206 Assembler::LSR,
12207 $src3$$constant & 0x3f);
12208 %}
12209
12210 ins_pipe(ialu_reg_reg_shift);
12211 %}
12212
12213 // This pattern is automatically generated from aarch64_ad.m4
12214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12215 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12216 iRegIorL2I src1, iRegIorL2I src2,
12217 immI src3) %{
12218 match(Set dst (SubI src1 (RShiftI src2 src3)));
12219
12220 ins_cost(1.9 * INSN_COST);
12221 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12222
12223 ins_encode %{
12224 __ subw(as_Register($dst$$reg),
12225 as_Register($src1$$reg),
12226 as_Register($src2$$reg),
12227 Assembler::ASR,
12228 $src3$$constant & 0x1f);
12229 %}
12230
12231 ins_pipe(ialu_reg_reg_shift);
12232 %}
12233
12234 // This pattern is automatically generated from aarch64_ad.m4
12235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12236 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12237 iRegL src1, iRegL src2,
12238 immI src3) %{
12239 match(Set dst (SubL src1 (RShiftL src2 src3)));
12240
12241 ins_cost(1.9 * INSN_COST);
12242 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12243
12244 ins_encode %{
12245 __ sub(as_Register($dst$$reg),
12246 as_Register($src1$$reg),
12247 as_Register($src2$$reg),
12248 Assembler::ASR,
12249 $src3$$constant & 0x3f);
12250 %}
12251
12252 ins_pipe(ialu_reg_reg_shift);
12253 %}
12254
12255 // This pattern is automatically generated from aarch64_ad.m4
12256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12257 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12258 iRegIorL2I src1, iRegIorL2I src2,
12259 immI src3) %{
12260 match(Set dst (SubI src1 (LShiftI src2 src3)));
12261
12262 ins_cost(1.9 * INSN_COST);
12263 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12264
12265 ins_encode %{
12266 __ subw(as_Register($dst$$reg),
12267 as_Register($src1$$reg),
12268 as_Register($src2$$reg),
12269 Assembler::LSL,
12270 $src3$$constant & 0x1f);
12271 %}
12272
12273 ins_pipe(ialu_reg_reg_shift);
12274 %}
12275
12276 // This pattern is automatically generated from aarch64_ad.m4
12277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12278 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12279 iRegL src1, iRegL src2,
12280 immI src3) %{
12281 match(Set dst (SubL src1 (LShiftL src2 src3)));
12282
12283 ins_cost(1.9 * INSN_COST);
12284 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12285
12286 ins_encode %{
12287 __ sub(as_Register($dst$$reg),
12288 as_Register($src1$$reg),
12289 as_Register($src2$$reg),
12290 Assembler::LSL,
12291 $src3$$constant & 0x3f);
12292 %}
12293
12294 ins_pipe(ialu_reg_reg_shift);
12295 %}
12296
12297 // This pattern is automatically generated from aarch64_ad.m4
12298 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12299
12300 // Shift Left followed by Shift Right.
12301 // This idiom is used by the compiler for the i2b bytecode etc.
12302 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12303 %{
12304 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12305 ins_cost(INSN_COST * 2);
12306 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12307 ins_encode %{
12308 int lshift = $lshift_count$$constant & 63;
12309 int rshift = $rshift_count$$constant & 63;
12310 int s = 63 - lshift;
12311 int r = (rshift - lshift) & 63;
12312 __ sbfm(as_Register($dst$$reg),
12313 as_Register($src$$reg),
12314 r, s);
12315 %}
12316
12317 ins_pipe(ialu_reg_shift);
12318 %}
12319
12320 // This pattern is automatically generated from aarch64_ad.m4
12321 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12322
12323 // Shift Left followed by Shift Right.
12324 // This idiom is used by the compiler for the i2b bytecode etc.
12325 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12326 %{
12327 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12328 ins_cost(INSN_COST * 2);
12329 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12330 ins_encode %{
12331 int lshift = $lshift_count$$constant & 31;
12332 int rshift = $rshift_count$$constant & 31;
12333 int s = 31 - lshift;
12334 int r = (rshift - lshift) & 31;
12335 __ sbfmw(as_Register($dst$$reg),
12336 as_Register($src$$reg),
12337 r, s);
12338 %}
12339
12340 ins_pipe(ialu_reg_shift);
12341 %}
12342
12343 // This pattern is automatically generated from aarch64_ad.m4
12344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12345
12346 // Shift Left followed by Shift Right.
12347 // This idiom is used by the compiler for the i2b bytecode etc.
12348 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12349 %{
12350 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12351 ins_cost(INSN_COST * 2);
12352 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12353 ins_encode %{
12354 int lshift = $lshift_count$$constant & 63;
12355 int rshift = $rshift_count$$constant & 63;
12356 int s = 63 - lshift;
12357 int r = (rshift - lshift) & 63;
12358 __ ubfm(as_Register($dst$$reg),
12359 as_Register($src$$reg),
12360 r, s);
12361 %}
12362
12363 ins_pipe(ialu_reg_shift);
12364 %}
12365
12366 // This pattern is automatically generated from aarch64_ad.m4
12367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12368
12369 // Shift Left followed by Shift Right.
12370 // This idiom is used by the compiler for the i2b bytecode etc.
12371 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12372 %{
12373 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12374 ins_cost(INSN_COST * 2);
12375 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12376 ins_encode %{
12377 int lshift = $lshift_count$$constant & 31;
12378 int rshift = $rshift_count$$constant & 31;
12379 int s = 31 - lshift;
12380 int r = (rshift - lshift) & 31;
12381 __ ubfmw(as_Register($dst$$reg),
12382 as_Register($src$$reg),
12383 r, s);
12384 %}
12385
12386 ins_pipe(ialu_reg_shift);
12387 %}
12388
12389 // Bitfield extract with shift & mask
12390
12391 // This pattern is automatically generated from aarch64_ad.m4
12392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12393 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12394 %{
12395 match(Set dst (AndI (URShiftI src rshift) mask));
12396 // Make sure we are not going to exceed what ubfxw can do.
12397 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12398
12399 ins_cost(INSN_COST);
12400 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12401 ins_encode %{
12402 int rshift = $rshift$$constant & 31;
12403 intptr_t mask = $mask$$constant;
12404 int width = exact_log2(mask+1);
12405 __ ubfxw(as_Register($dst$$reg),
12406 as_Register($src$$reg), rshift, width);
12407 %}
12408 ins_pipe(ialu_reg_shift);
12409 %}
12410
12411 // This pattern is automatically generated from aarch64_ad.m4
12412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12413 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12414 %{
12415 match(Set dst (AndL (URShiftL src rshift) mask));
12416 // Make sure we are not going to exceed what ubfx can do.
12417 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12418
12419 ins_cost(INSN_COST);
12420 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12421 ins_encode %{
12422 int rshift = $rshift$$constant & 63;
12423 intptr_t mask = $mask$$constant;
12424 int width = exact_log2_long(mask+1);
12425 __ ubfx(as_Register($dst$$reg),
12426 as_Register($src$$reg), rshift, width);
12427 %}
12428 ins_pipe(ialu_reg_shift);
12429 %}
12430
12431
12432 // This pattern is automatically generated from aarch64_ad.m4
12433 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12434
12435 // We can use ubfx when extending an And with a mask when we know mask
12436 // is positive. We know that because immI_bitmask guarantees it.
12437 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12438 %{
12439 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12440 // Make sure we are not going to exceed what ubfxw can do.
12441 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12442
12443 ins_cost(INSN_COST * 2);
12444 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12445 ins_encode %{
12446 int rshift = $rshift$$constant & 31;
12447 intptr_t mask = $mask$$constant;
12448 int width = exact_log2(mask+1);
12449 __ ubfx(as_Register($dst$$reg),
12450 as_Register($src$$reg), rshift, width);
12451 %}
12452 ins_pipe(ialu_reg_shift);
12453 %}
12454
12455
12456 // This pattern is automatically generated from aarch64_ad.m4
12457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12458
12459 // We can use ubfiz when masking by a positive number and then left shifting the result.
12460 // We know that the mask is positive because immI_bitmask guarantees it.
12461 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12462 %{
12463 match(Set dst (LShiftI (AndI src mask) lshift));
12464 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12465
12466 ins_cost(INSN_COST);
12467 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12468 ins_encode %{
12469 int lshift = $lshift$$constant & 31;
12470 intptr_t mask = $mask$$constant;
12471 int width = exact_log2(mask+1);
12472 __ ubfizw(as_Register($dst$$reg),
12473 as_Register($src$$reg), lshift, width);
12474 %}
12475 ins_pipe(ialu_reg_shift);
12476 %}
12477
12478 // This pattern is automatically generated from aarch64_ad.m4
12479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12480
12481 // We can use ubfiz when masking by a positive number and then left shifting the result.
12482 // We know that the mask is positive because immL_bitmask guarantees it.
12483 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12484 %{
12485 match(Set dst (LShiftL (AndL src mask) lshift));
12486 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12487
12488 ins_cost(INSN_COST);
12489 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12490 ins_encode %{
12491 int lshift = $lshift$$constant & 63;
12492 intptr_t mask = $mask$$constant;
12493 int width = exact_log2_long(mask+1);
12494 __ ubfiz(as_Register($dst$$reg),
12495 as_Register($src$$reg), lshift, width);
12496 %}
12497 ins_pipe(ialu_reg_shift);
12498 %}
12499
12500 // This pattern is automatically generated from aarch64_ad.m4
12501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12502
12503 // We can use ubfiz when masking by a positive number and then left shifting the result.
12504 // We know that the mask is positive because immI_bitmask guarantees it.
12505 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12506 %{
12507 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12508 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12509
12510 ins_cost(INSN_COST);
12511 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12512 ins_encode %{
12513 int lshift = $lshift$$constant & 31;
12514 intptr_t mask = $mask$$constant;
12515 int width = exact_log2(mask+1);
12516 __ ubfizw(as_Register($dst$$reg),
12517 as_Register($src$$reg), lshift, width);
12518 %}
12519 ins_pipe(ialu_reg_shift);
12520 %}
12521
12522 // This pattern is automatically generated from aarch64_ad.m4
12523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12524
12525 // We can use ubfiz when masking by a positive number and then left shifting the result.
12526 // We know that the mask is positive because immL_bitmask guarantees it.
12527 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12528 %{
12529 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12530 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12531
12532 ins_cost(INSN_COST);
12533 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12534 ins_encode %{
12535 int lshift = $lshift$$constant & 63;
12536 intptr_t mask = $mask$$constant;
12537 int width = exact_log2_long(mask+1);
12538 __ ubfiz(as_Register($dst$$reg),
12539 as_Register($src$$reg), lshift, width);
12540 %}
12541 ins_pipe(ialu_reg_shift);
12542 %}
12543
12544
12545 // This pattern is automatically generated from aarch64_ad.m4
12546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12547
12548 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12549 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12550 %{
12551 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12552 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12553
12554 ins_cost(INSN_COST);
12555 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12556 ins_encode %{
12557 int lshift = $lshift$$constant & 63;
12558 intptr_t mask = $mask$$constant;
12559 int width = exact_log2(mask+1);
12560 __ ubfiz(as_Register($dst$$reg),
12561 as_Register($src$$reg), lshift, width);
12562 %}
12563 ins_pipe(ialu_reg_shift);
12564 %}
12565
12566 // This pattern is automatically generated from aarch64_ad.m4
12567 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12568
12569 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12570 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12571 %{
12572 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12573 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12574
12575 ins_cost(INSN_COST);
12576 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12577 ins_encode %{
12578 int lshift = $lshift$$constant & 31;
12579 intptr_t mask = $mask$$constant;
12580 int width = exact_log2(mask+1);
12581 __ ubfiz(as_Register($dst$$reg),
12582 as_Register($src$$reg), lshift, width);
12583 %}
12584 ins_pipe(ialu_reg_shift);
12585 %}
12586
12587 // This pattern is automatically generated from aarch64_ad.m4
12588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12589
12590 // Can skip int2long conversions after AND with small bitmask
12591 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12592 %{
12593 match(Set dst (ConvI2L (AndI src msk)));
12594 ins_cost(INSN_COST);
12595 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12596 ins_encode %{
12597 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12598 %}
12599 ins_pipe(ialu_reg_shift);
12600 %}
12601
12602
12603 // Rotations
12604
12605 // This pattern is automatically generated from aarch64_ad.m4
12606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12607 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12608 %{
12609 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12610 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12611
12612 ins_cost(INSN_COST);
12613 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12614
12615 ins_encode %{
12616 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12617 $rshift$$constant & 63);
12618 %}
12619 ins_pipe(ialu_reg_reg_extr);
12620 %}
12621
12622
12623 // This pattern is automatically generated from aarch64_ad.m4
12624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12625 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12626 %{
12627 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12628 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12629
12630 ins_cost(INSN_COST);
12631 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12632
12633 ins_encode %{
12634 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12635 $rshift$$constant & 31);
12636 %}
12637 ins_pipe(ialu_reg_reg_extr);
12638 %}
12639
12640
12641 // This pattern is automatically generated from aarch64_ad.m4
12642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12643 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12644 %{
12645 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12646 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12647
12648 ins_cost(INSN_COST);
12649 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12650
12651 ins_encode %{
12652 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12653 $rshift$$constant & 63);
12654 %}
12655 ins_pipe(ialu_reg_reg_extr);
12656 %}
12657
12658
12659 // This pattern is automatically generated from aarch64_ad.m4
12660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12661 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12662 %{
12663 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12664 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12665
12666 ins_cost(INSN_COST);
12667 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12668
12669 ins_encode %{
12670 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12671 $rshift$$constant & 31);
12672 %}
12673 ins_pipe(ialu_reg_reg_extr);
12674 %}
12675
12676 // This pattern is automatically generated from aarch64_ad.m4
12677 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12678 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12679 %{
12680 match(Set dst (RotateRight src shift));
12681
12682 ins_cost(INSN_COST);
12683 format %{ "ror $dst, $src, $shift" %}
12684
12685 ins_encode %{
12686 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12687 $shift$$constant & 0x1f);
12688 %}
12689 ins_pipe(ialu_reg_reg_vshift);
12690 %}
12691
12692 // This pattern is automatically generated from aarch64_ad.m4
12693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12694 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12695 %{
12696 match(Set dst (RotateRight src shift));
12697
12698 ins_cost(INSN_COST);
12699 format %{ "ror $dst, $src, $shift" %}
12700
12701 ins_encode %{
12702 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12703 $shift$$constant & 0x3f);
12704 %}
12705 ins_pipe(ialu_reg_reg_vshift);
12706 %}
12707
12708 // This pattern is automatically generated from aarch64_ad.m4
12709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12710 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12711 %{
12712 match(Set dst (RotateRight src shift));
12713
12714 ins_cost(INSN_COST);
12715 format %{ "ror $dst, $src, $shift" %}
12716
12717 ins_encode %{
12718 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12719 %}
12720 ins_pipe(ialu_reg_reg_vshift);
12721 %}
12722
12723 // This pattern is automatically generated from aarch64_ad.m4
12724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12725 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12726 %{
12727 match(Set dst (RotateRight src shift));
12728
12729 ins_cost(INSN_COST);
12730 format %{ "ror $dst, $src, $shift" %}
12731
12732 ins_encode %{
12733 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12734 %}
12735 ins_pipe(ialu_reg_reg_vshift);
12736 %}
12737
12738 // This pattern is automatically generated from aarch64_ad.m4
12739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12740 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12741 %{
12742 match(Set dst (RotateLeft src shift));
12743
12744 ins_cost(INSN_COST);
12745 format %{ "rol $dst, $src, $shift" %}
12746
12747 ins_encode %{
12748 __ subw(rscratch1, zr, as_Register($shift$$reg));
12749 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12750 %}
12751 ins_pipe(ialu_reg_reg_vshift);
12752 %}
12753
12754 // This pattern is automatically generated from aarch64_ad.m4
12755 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12756 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12757 %{
12758 match(Set dst (RotateLeft src shift));
12759
12760 ins_cost(INSN_COST);
12761 format %{ "rol $dst, $src, $shift" %}
12762
12763 ins_encode %{
12764 __ subw(rscratch1, zr, as_Register($shift$$reg));
12765 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12766 %}
12767 ins_pipe(ialu_reg_reg_vshift);
12768 %}
12769
12770
12771 // Add/subtract (extended)
12772
12773 // This pattern is automatically generated from aarch64_ad.m4
12774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12775 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12776 %{
12777 match(Set dst (AddL src1 (ConvI2L src2)));
12778 ins_cost(INSN_COST);
12779 format %{ "add $dst, $src1, $src2, sxtw" %}
12780
12781 ins_encode %{
12782 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12783 as_Register($src2$$reg), ext::sxtw);
12784 %}
12785 ins_pipe(ialu_reg_reg);
12786 %}
12787
12788 // This pattern is automatically generated from aarch64_ad.m4
12789 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12790 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12791 %{
12792 match(Set dst (SubL src1 (ConvI2L src2)));
12793 ins_cost(INSN_COST);
12794 format %{ "sub $dst, $src1, $src2, sxtw" %}
12795
12796 ins_encode %{
12797 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12798 as_Register($src2$$reg), ext::sxtw);
12799 %}
12800 ins_pipe(ialu_reg_reg);
12801 %}
12802
12803 // This pattern is automatically generated from aarch64_ad.m4
12804 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12805 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12806 %{
12807 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12808 ins_cost(INSN_COST);
12809 format %{ "add $dst, $src1, $src2, sxth" %}
12810
12811 ins_encode %{
12812 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12813 as_Register($src2$$reg), ext::sxth);
12814 %}
12815 ins_pipe(ialu_reg_reg);
12816 %}
12817
12818 // This pattern is automatically generated from aarch64_ad.m4
12819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12820 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12821 %{
12822 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12823 ins_cost(INSN_COST);
12824 format %{ "add $dst, $src1, $src2, sxtb" %}
12825
12826 ins_encode %{
12827 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12828 as_Register($src2$$reg), ext::sxtb);
12829 %}
12830 ins_pipe(ialu_reg_reg);
12831 %}
12832
12833 // This pattern is automatically generated from aarch64_ad.m4
12834 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12835 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12836 %{
12837 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12838 ins_cost(INSN_COST);
12839 format %{ "add $dst, $src1, $src2, uxtb" %}
12840
12841 ins_encode %{
12842 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12843 as_Register($src2$$reg), ext::uxtb);
12844 %}
12845 ins_pipe(ialu_reg_reg);
12846 %}
12847
12848 // This pattern is automatically generated from aarch64_ad.m4
12849 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12850 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12851 %{
12852 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12853 ins_cost(INSN_COST);
12854 format %{ "add $dst, $src1, $src2, sxth" %}
12855
12856 ins_encode %{
12857 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12858 as_Register($src2$$reg), ext::sxth);
12859 %}
12860 ins_pipe(ialu_reg_reg);
12861 %}
12862
12863 // This pattern is automatically generated from aarch64_ad.m4
12864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12865 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12866 %{
12867 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12868 ins_cost(INSN_COST);
12869 format %{ "add $dst, $src1, $src2, sxtw" %}
12870
12871 ins_encode %{
12872 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12873 as_Register($src2$$reg), ext::sxtw);
12874 %}
12875 ins_pipe(ialu_reg_reg);
12876 %}
12877
12878 // This pattern is automatically generated from aarch64_ad.m4
12879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12880 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12881 %{
12882 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12883 ins_cost(INSN_COST);
12884 format %{ "add $dst, $src1, $src2, sxtb" %}
12885
12886 ins_encode %{
12887 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12888 as_Register($src2$$reg), ext::sxtb);
12889 %}
12890 ins_pipe(ialu_reg_reg);
12891 %}
12892
12893 // This pattern is automatically generated from aarch64_ad.m4
12894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12895 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12896 %{
12897 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12898 ins_cost(INSN_COST);
12899 format %{ "add $dst, $src1, $src2, uxtb" %}
12900
12901 ins_encode %{
12902 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12903 as_Register($src2$$reg), ext::uxtb);
12904 %}
12905 ins_pipe(ialu_reg_reg);
12906 %}
12907
12908 // This pattern is automatically generated from aarch64_ad.m4
12909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12910 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12911 %{
12912 match(Set dst (AddI src1 (AndI src2 mask)));
12913 ins_cost(INSN_COST);
12914 format %{ "addw $dst, $src1, $src2, uxtb" %}
12915
12916 ins_encode %{
12917 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12918 as_Register($src2$$reg), ext::uxtb);
12919 %}
12920 ins_pipe(ialu_reg_reg);
12921 %}
12922
12923 // This pattern is automatically generated from aarch64_ad.m4
12924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12925 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12926 %{
12927 match(Set dst (AddI src1 (AndI src2 mask)));
12928 ins_cost(INSN_COST);
12929 format %{ "addw $dst, $src1, $src2, uxth" %}
12930
12931 ins_encode %{
12932 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12933 as_Register($src2$$reg), ext::uxth);
12934 %}
12935 ins_pipe(ialu_reg_reg);
12936 %}
12937
12938 // This pattern is automatically generated from aarch64_ad.m4
12939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12940 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12941 %{
12942 match(Set dst (AddL src1 (AndL src2 mask)));
12943 ins_cost(INSN_COST);
12944 format %{ "add $dst, $src1, $src2, uxtb" %}
12945
12946 ins_encode %{
12947 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12948 as_Register($src2$$reg), ext::uxtb);
12949 %}
12950 ins_pipe(ialu_reg_reg);
12951 %}
12952
12953 // This pattern is automatically generated from aarch64_ad.m4
12954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12955 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12956 %{
12957 match(Set dst (AddL src1 (AndL src2 mask)));
12958 ins_cost(INSN_COST);
12959 format %{ "add $dst, $src1, $src2, uxth" %}
12960
12961 ins_encode %{
12962 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12963 as_Register($src2$$reg), ext::uxth);
12964 %}
12965 ins_pipe(ialu_reg_reg);
12966 %}
12967
12968 // This pattern is automatically generated from aarch64_ad.m4
12969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12970 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12971 %{
12972 match(Set dst (AddL src1 (AndL src2 mask)));
12973 ins_cost(INSN_COST);
12974 format %{ "add $dst, $src1, $src2, uxtw" %}
12975
12976 ins_encode %{
12977 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12978 as_Register($src2$$reg), ext::uxtw);
12979 %}
12980 ins_pipe(ialu_reg_reg);
12981 %}
12982
12983 // This pattern is automatically generated from aarch64_ad.m4
12984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12985 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12986 %{
12987 match(Set dst (SubI src1 (AndI src2 mask)));
12988 ins_cost(INSN_COST);
12989 format %{ "subw $dst, $src1, $src2, uxtb" %}
12990
12991 ins_encode %{
12992 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12993 as_Register($src2$$reg), ext::uxtb);
12994 %}
12995 ins_pipe(ialu_reg_reg);
12996 %}
12997
12998 // This pattern is automatically generated from aarch64_ad.m4
12999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13000 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13001 %{
13002 match(Set dst (SubI src1 (AndI src2 mask)));
13003 ins_cost(INSN_COST);
13004 format %{ "subw $dst, $src1, $src2, uxth" %}
13005
13006 ins_encode %{
13007 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13008 as_Register($src2$$reg), ext::uxth);
13009 %}
13010 ins_pipe(ialu_reg_reg);
13011 %}
13012
13013 // This pattern is automatically generated from aarch64_ad.m4
13014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13015 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13016 %{
13017 match(Set dst (SubL src1 (AndL src2 mask)));
13018 ins_cost(INSN_COST);
13019 format %{ "sub $dst, $src1, $src2, uxtb" %}
13020
13021 ins_encode %{
13022 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13023 as_Register($src2$$reg), ext::uxtb);
13024 %}
13025 ins_pipe(ialu_reg_reg);
13026 %}
13027
13028 // This pattern is automatically generated from aarch64_ad.m4
13029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13030 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13031 %{
13032 match(Set dst (SubL src1 (AndL src2 mask)));
13033 ins_cost(INSN_COST);
13034 format %{ "sub $dst, $src1, $src2, uxth" %}
13035
13036 ins_encode %{
13037 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13038 as_Register($src2$$reg), ext::uxth);
13039 %}
13040 ins_pipe(ialu_reg_reg);
13041 %}
13042
13043 // This pattern is automatically generated from aarch64_ad.m4
13044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13045 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13046 %{
13047 match(Set dst (SubL src1 (AndL src2 mask)));
13048 ins_cost(INSN_COST);
13049 format %{ "sub $dst, $src1, $src2, uxtw" %}
13050
13051 ins_encode %{
13052 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13053 as_Register($src2$$reg), ext::uxtw);
13054 %}
13055 ins_pipe(ialu_reg_reg);
13056 %}
13057
13058
13059 // This pattern is automatically generated from aarch64_ad.m4
13060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13061 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13062 %{
13063 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13064 ins_cost(1.9 * INSN_COST);
13065 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13066
13067 ins_encode %{
13068 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13069 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13070 %}
13071 ins_pipe(ialu_reg_reg_shift);
13072 %}
13073
13074 // This pattern is automatically generated from aarch64_ad.m4
13075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13076 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13077 %{
13078 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13079 ins_cost(1.9 * INSN_COST);
13080 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13081
13082 ins_encode %{
13083 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13084 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13085 %}
13086 ins_pipe(ialu_reg_reg_shift);
13087 %}
13088
13089 // This pattern is automatically generated from aarch64_ad.m4
13090 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13091 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13092 %{
13093 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13094 ins_cost(1.9 * INSN_COST);
13095 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13096
13097 ins_encode %{
13098 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13099 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13100 %}
13101 ins_pipe(ialu_reg_reg_shift);
13102 %}
13103
13104 // This pattern is automatically generated from aarch64_ad.m4
13105 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13106 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13107 %{
13108 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13109 ins_cost(1.9 * INSN_COST);
13110 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13111
13112 ins_encode %{
13113 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13114 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13115 %}
13116 ins_pipe(ialu_reg_reg_shift);
13117 %}
13118
13119 // This pattern is automatically generated from aarch64_ad.m4
13120 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13121 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13122 %{
13123 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13124 ins_cost(1.9 * INSN_COST);
13125 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13126
13127 ins_encode %{
13128 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13129 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13130 %}
13131 ins_pipe(ialu_reg_reg_shift);
13132 %}
13133
13134 // This pattern is automatically generated from aarch64_ad.m4
13135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13136 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13137 %{
13138 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13139 ins_cost(1.9 * INSN_COST);
13140 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13141
13142 ins_encode %{
13143 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13144 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13145 %}
13146 ins_pipe(ialu_reg_reg_shift);
13147 %}
13148
13149 // This pattern is automatically generated from aarch64_ad.m4
13150 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13151 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13152 %{
13153 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13154 ins_cost(1.9 * INSN_COST);
13155 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13156
13157 ins_encode %{
13158 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13159 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13160 %}
13161 ins_pipe(ialu_reg_reg_shift);
13162 %}
13163
13164 // This pattern is automatically generated from aarch64_ad.m4
13165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13166 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13167 %{
13168 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13169 ins_cost(1.9 * INSN_COST);
13170 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13171
13172 ins_encode %{
13173 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13174 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13175 %}
13176 ins_pipe(ialu_reg_reg_shift);
13177 %}
13178
13179 // This pattern is automatically generated from aarch64_ad.m4
13180 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13181 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13182 %{
13183 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13184 ins_cost(1.9 * INSN_COST);
13185 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13186
13187 ins_encode %{
13188 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13189 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13190 %}
13191 ins_pipe(ialu_reg_reg_shift);
13192 %}
13193
13194 // This pattern is automatically generated from aarch64_ad.m4
13195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13196 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13197 %{
13198 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13199 ins_cost(1.9 * INSN_COST);
13200 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13201
13202 ins_encode %{
13203 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13204 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13205 %}
13206 ins_pipe(ialu_reg_reg_shift);
13207 %}
13208
13209 // This pattern is automatically generated from aarch64_ad.m4
13210 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13211 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13212 %{
13213 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13214 ins_cost(1.9 * INSN_COST);
13215 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13216
13217 ins_encode %{
13218 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13219 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13220 %}
13221 ins_pipe(ialu_reg_reg_shift);
13222 %}
13223
13224 // This pattern is automatically generated from aarch64_ad.m4
13225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13226 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13227 %{
13228 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13229 ins_cost(1.9 * INSN_COST);
13230 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13231
13232 ins_encode %{
13233 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13234 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13235 %}
13236 ins_pipe(ialu_reg_reg_shift);
13237 %}
13238
13239 // This pattern is automatically generated from aarch64_ad.m4
13240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13241 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13242 %{
13243 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13244 ins_cost(1.9 * INSN_COST);
13245 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13246
13247 ins_encode %{
13248 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13249 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13250 %}
13251 ins_pipe(ialu_reg_reg_shift);
13252 %}
13253
13254 // This pattern is automatically generated from aarch64_ad.m4
13255 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13256 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13257 %{
13258 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13259 ins_cost(1.9 * INSN_COST);
13260 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13261
13262 ins_encode %{
13263 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13264 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13265 %}
13266 ins_pipe(ialu_reg_reg_shift);
13267 %}
13268
13269 // This pattern is automatically generated from aarch64_ad.m4
13270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13271 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13272 %{
13273 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13274 ins_cost(1.9 * INSN_COST);
13275 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13276
13277 ins_encode %{
13278 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13279 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13280 %}
13281 ins_pipe(ialu_reg_reg_shift);
13282 %}
13283
13284 // This pattern is automatically generated from aarch64_ad.m4
13285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13286 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13287 %{
13288 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13289 ins_cost(1.9 * INSN_COST);
13290 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13291
13292 ins_encode %{
13293 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13294 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13295 %}
13296 ins_pipe(ialu_reg_reg_shift);
13297 %}
13298
13299 // This pattern is automatically generated from aarch64_ad.m4
13300 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13301 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13302 %{
13303 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13304 ins_cost(1.9 * INSN_COST);
13305 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13306
13307 ins_encode %{
13308 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13309 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13310 %}
13311 ins_pipe(ialu_reg_reg_shift);
13312 %}
13313
13314 // This pattern is automatically generated from aarch64_ad.m4
13315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13316 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13317 %{
13318 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13319 ins_cost(1.9 * INSN_COST);
13320 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13321
13322 ins_encode %{
13323 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13324 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13325 %}
13326 ins_pipe(ialu_reg_reg_shift);
13327 %}
13328
13329 // This pattern is automatically generated from aarch64_ad.m4
13330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13331 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13332 %{
13333 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13334 ins_cost(1.9 * INSN_COST);
13335 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13336
13337 ins_encode %{
13338 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13339 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13340 %}
13341 ins_pipe(ialu_reg_reg_shift);
13342 %}
13343
13344 // This pattern is automatically generated from aarch64_ad.m4
13345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13346 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13347 %{
13348 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13349 ins_cost(1.9 * INSN_COST);
13350 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13351
13352 ins_encode %{
13353 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13354 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13355 %}
13356 ins_pipe(ialu_reg_reg_shift);
13357 %}
13358
13359 // This pattern is automatically generated from aarch64_ad.m4
13360 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13361 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13362 %{
13363 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13364 ins_cost(1.9 * INSN_COST);
13365 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13366
13367 ins_encode %{
13368 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13369 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13370 %}
13371 ins_pipe(ialu_reg_reg_shift);
13372 %}
13373
13374 // This pattern is automatically generated from aarch64_ad.m4
13375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13376 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13377 %{
13378 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13379 ins_cost(1.9 * INSN_COST);
13380 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13381
13382 ins_encode %{
13383 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13384 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13385 %}
13386 ins_pipe(ialu_reg_reg_shift);
13387 %}
13388
13389 // This pattern is automatically generated from aarch64_ad.m4
13390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13391 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13392 %{
13393 effect(DEF dst, USE src1, USE src2, USE cr);
13394 ins_cost(INSN_COST * 2);
13395 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13396
13397 ins_encode %{
13398 __ cselw($dst$$Register,
13399 $src1$$Register,
13400 $src2$$Register,
13401 Assembler::LT);
13402 %}
13403 ins_pipe(icond_reg_reg);
13404 %}
13405
13406 // This pattern is automatically generated from aarch64_ad.m4
13407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13408 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13409 %{
13410 effect(DEF dst, USE src1, USE src2, USE cr);
13411 ins_cost(INSN_COST * 2);
13412 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13413
13414 ins_encode %{
13415 __ cselw($dst$$Register,
13416 $src1$$Register,
13417 $src2$$Register,
13418 Assembler::GT);
13419 %}
13420 ins_pipe(icond_reg_reg);
13421 %}
13422
13423 // This pattern is automatically generated from aarch64_ad.m4
13424 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13425 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13426 %{
13427 effect(DEF dst, USE src1, USE cr);
13428 ins_cost(INSN_COST * 2);
13429 format %{ "cselw $dst, $src1, zr lt\t" %}
13430
13431 ins_encode %{
13432 __ cselw($dst$$Register,
13433 $src1$$Register,
13434 zr,
13435 Assembler::LT);
13436 %}
13437 ins_pipe(icond_reg);
13438 %}
13439
13440 // This pattern is automatically generated from aarch64_ad.m4
13441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13442 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13443 %{
13444 effect(DEF dst, USE src1, USE cr);
13445 ins_cost(INSN_COST * 2);
13446 format %{ "cselw $dst, $src1, zr gt\t" %}
13447
13448 ins_encode %{
13449 __ cselw($dst$$Register,
13450 $src1$$Register,
13451 zr,
13452 Assembler::GT);
13453 %}
13454 ins_pipe(icond_reg);
13455 %}
13456
13457 // This pattern is automatically generated from aarch64_ad.m4
13458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13459 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13460 %{
13461 effect(DEF dst, USE src1, USE cr);
13462 ins_cost(INSN_COST * 2);
13463 format %{ "csincw $dst, $src1, zr le\t" %}
13464
13465 ins_encode %{
13466 __ csincw($dst$$Register,
13467 $src1$$Register,
13468 zr,
13469 Assembler::LE);
13470 %}
13471 ins_pipe(icond_reg);
13472 %}
13473
13474 // This pattern is automatically generated from aarch64_ad.m4
13475 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13476 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13477 %{
13478 effect(DEF dst, USE src1, USE cr);
13479 ins_cost(INSN_COST * 2);
13480 format %{ "csincw $dst, $src1, zr gt\t" %}
13481
13482 ins_encode %{
13483 __ csincw($dst$$Register,
13484 $src1$$Register,
13485 zr,
13486 Assembler::GT);
13487 %}
13488 ins_pipe(icond_reg);
13489 %}
13490
13491 // This pattern is automatically generated from aarch64_ad.m4
13492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13493 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13494 %{
13495 effect(DEF dst, USE src1, USE cr);
13496 ins_cost(INSN_COST * 2);
13497 format %{ "csinvw $dst, $src1, zr lt\t" %}
13498
13499 ins_encode %{
13500 __ csinvw($dst$$Register,
13501 $src1$$Register,
13502 zr,
13503 Assembler::LT);
13504 %}
13505 ins_pipe(icond_reg);
13506 %}
13507
13508 // This pattern is automatically generated from aarch64_ad.m4
13509 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13510 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13511 %{
13512 effect(DEF dst, USE src1, USE cr);
13513 ins_cost(INSN_COST * 2);
13514 format %{ "csinvw $dst, $src1, zr ge\t" %}
13515
13516 ins_encode %{
13517 __ csinvw($dst$$Register,
13518 $src1$$Register,
13519 zr,
13520 Assembler::GE);
13521 %}
13522 ins_pipe(icond_reg);
13523 %}
13524
13525 // This pattern is automatically generated from aarch64_ad.m4
13526 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13527 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13528 %{
13529 match(Set dst (MinI src imm));
13530 ins_cost(INSN_COST * 3);
13531 expand %{
13532 rFlagsReg cr;
13533 compI_reg_imm0(cr, src);
13534 cmovI_reg_imm0_lt(dst, src, cr);
13535 %}
13536 %}
13537
13538 // This pattern is automatically generated from aarch64_ad.m4
13539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13540 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13541 %{
13542 match(Set dst (MinI imm src));
13543 ins_cost(INSN_COST * 3);
13544 expand %{
13545 rFlagsReg cr;
13546 compI_reg_imm0(cr, src);
13547 cmovI_reg_imm0_lt(dst, src, cr);
13548 %}
13549 %}
13550
13551 // This pattern is automatically generated from aarch64_ad.m4
13552 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13553 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13554 %{
13555 match(Set dst (MinI src imm));
13556 ins_cost(INSN_COST * 3);
13557 expand %{
13558 rFlagsReg cr;
13559 compI_reg_imm0(cr, src);
13560 cmovI_reg_imm1_le(dst, src, cr);
13561 %}
13562 %}
13563
13564 // This pattern is automatically generated from aarch64_ad.m4
13565 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13566 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13567 %{
13568 match(Set dst (MinI imm src));
13569 ins_cost(INSN_COST * 3);
13570 expand %{
13571 rFlagsReg cr;
13572 compI_reg_imm0(cr, src);
13573 cmovI_reg_imm1_le(dst, src, cr);
13574 %}
13575 %}
13576
13577 // This pattern is automatically generated from aarch64_ad.m4
13578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13579 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13580 %{
13581 match(Set dst (MinI src imm));
13582 ins_cost(INSN_COST * 3);
13583 expand %{
13584 rFlagsReg cr;
13585 compI_reg_imm0(cr, src);
13586 cmovI_reg_immM1_lt(dst, src, cr);
13587 %}
13588 %}
13589
13590 // This pattern is automatically generated from aarch64_ad.m4
13591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13592 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13593 %{
13594 match(Set dst (MinI imm src));
13595 ins_cost(INSN_COST * 3);
13596 expand %{
13597 rFlagsReg cr;
13598 compI_reg_imm0(cr, src);
13599 cmovI_reg_immM1_lt(dst, src, cr);
13600 %}
13601 %}
13602
13603 // This pattern is automatically generated from aarch64_ad.m4
13604 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13605 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13606 %{
13607 match(Set dst (MaxI src imm));
13608 ins_cost(INSN_COST * 3);
13609 expand %{
13610 rFlagsReg cr;
13611 compI_reg_imm0(cr, src);
13612 cmovI_reg_imm0_gt(dst, src, cr);
13613 %}
13614 %}
13615
13616 // This pattern is automatically generated from aarch64_ad.m4
13617 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13618 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13619 %{
13620 match(Set dst (MaxI imm src));
13621 ins_cost(INSN_COST * 3);
13622 expand %{
13623 rFlagsReg cr;
13624 compI_reg_imm0(cr, src);
13625 cmovI_reg_imm0_gt(dst, src, cr);
13626 %}
13627 %}
13628
13629 // This pattern is automatically generated from aarch64_ad.m4
13630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13631 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13632 %{
13633 match(Set dst (MaxI src imm));
13634 ins_cost(INSN_COST * 3);
13635 expand %{
13636 rFlagsReg cr;
13637 compI_reg_imm0(cr, src);
13638 cmovI_reg_imm1_gt(dst, src, cr);
13639 %}
13640 %}
13641
13642 // This pattern is automatically generated from aarch64_ad.m4
13643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13644 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13645 %{
13646 match(Set dst (MaxI imm src));
13647 ins_cost(INSN_COST * 3);
13648 expand %{
13649 rFlagsReg cr;
13650 compI_reg_imm0(cr, src);
13651 cmovI_reg_imm1_gt(dst, src, cr);
13652 %}
13653 %}
13654
13655 // This pattern is automatically generated from aarch64_ad.m4
13656 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13657 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13658 %{
13659 match(Set dst (MaxI src imm));
13660 ins_cost(INSN_COST * 3);
13661 expand %{
13662 rFlagsReg cr;
13663 compI_reg_imm0(cr, src);
13664 cmovI_reg_immM1_ge(dst, src, cr);
13665 %}
13666 %}
13667
13668 // This pattern is automatically generated from aarch64_ad.m4
13669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13670 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13671 %{
13672 match(Set dst (MaxI imm src));
13673 ins_cost(INSN_COST * 3);
13674 expand %{
13675 rFlagsReg cr;
13676 compI_reg_imm0(cr, src);
13677 cmovI_reg_immM1_ge(dst, src, cr);
13678 %}
13679 %}
13680
13681 // This pattern is automatically generated from aarch64_ad.m4
13682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13683 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13684 %{
13685 match(Set dst (ReverseI src));
13686 ins_cost(INSN_COST);
13687 format %{ "rbitw $dst, $src" %}
13688 ins_encode %{
13689 __ rbitw($dst$$Register, $src$$Register);
13690 %}
13691 ins_pipe(ialu_reg);
13692 %}
13693
13694 // This pattern is automatically generated from aarch64_ad.m4
13695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13696 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13697 %{
13698 match(Set dst (ReverseL src));
13699 ins_cost(INSN_COST);
13700 format %{ "rbit $dst, $src" %}
13701 ins_encode %{
13702 __ rbit($dst$$Register, $src$$Register);
13703 %}
13704 ins_pipe(ialu_reg);
13705 %}
13706
13707
13708 // END This section of the file is automatically generated. Do not edit --------------
13709
13710
13711 // ============================================================================
13712 // Floating Point Arithmetic Instructions
13713
13714 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13715 match(Set dst (AddHF src1 src2));
13716 format %{ "faddh $dst, $src1, $src2" %}
13717 ins_encode %{
13718 __ faddh($dst$$FloatRegister,
13719 $src1$$FloatRegister,
13720 $src2$$FloatRegister);
13721 %}
13722 ins_pipe(fp_dop_reg_reg_s);
13723 %}
13724
13725 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13726 match(Set dst (AddF src1 src2));
13727
13728 ins_cost(INSN_COST * 5);
13729 format %{ "fadds $dst, $src1, $src2" %}
13730
13731 ins_encode %{
13732 __ fadds(as_FloatRegister($dst$$reg),
13733 as_FloatRegister($src1$$reg),
13734 as_FloatRegister($src2$$reg));
13735 %}
13736
13737 ins_pipe(fp_dop_reg_reg_s);
13738 %}
13739
13740 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13741 match(Set dst (AddD src1 src2));
13742
13743 ins_cost(INSN_COST * 5);
13744 format %{ "faddd $dst, $src1, $src2" %}
13745
13746 ins_encode %{
13747 __ faddd(as_FloatRegister($dst$$reg),
13748 as_FloatRegister($src1$$reg),
13749 as_FloatRegister($src2$$reg));
13750 %}
13751
13752 ins_pipe(fp_dop_reg_reg_d);
13753 %}
13754
13755 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13756 match(Set dst (SubHF src1 src2));
13757 format %{ "fsubh $dst, $src1, $src2" %}
13758 ins_encode %{
13759 __ fsubh($dst$$FloatRegister,
13760 $src1$$FloatRegister,
13761 $src2$$FloatRegister);
13762 %}
13763 ins_pipe(fp_dop_reg_reg_s);
13764 %}
13765
13766 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13767 match(Set dst (SubF src1 src2));
13768
13769 ins_cost(INSN_COST * 5);
13770 format %{ "fsubs $dst, $src1, $src2" %}
13771
13772 ins_encode %{
13773 __ fsubs(as_FloatRegister($dst$$reg),
13774 as_FloatRegister($src1$$reg),
13775 as_FloatRegister($src2$$reg));
13776 %}
13777
13778 ins_pipe(fp_dop_reg_reg_s);
13779 %}
13780
13781 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13782 match(Set dst (SubD src1 src2));
13783
13784 ins_cost(INSN_COST * 5);
13785 format %{ "fsubd $dst, $src1, $src2" %}
13786
13787 ins_encode %{
13788 __ fsubd(as_FloatRegister($dst$$reg),
13789 as_FloatRegister($src1$$reg),
13790 as_FloatRegister($src2$$reg));
13791 %}
13792
13793 ins_pipe(fp_dop_reg_reg_d);
13794 %}
13795
13796 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13797 match(Set dst (MulHF src1 src2));
13798 format %{ "fmulh $dst, $src1, $src2" %}
13799 ins_encode %{
13800 __ fmulh($dst$$FloatRegister,
13801 $src1$$FloatRegister,
13802 $src2$$FloatRegister);
13803 %}
13804 ins_pipe(fp_dop_reg_reg_s);
13805 %}
13806
13807 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13808 match(Set dst (MulF src1 src2));
13809
13810 ins_cost(INSN_COST * 6);
13811 format %{ "fmuls $dst, $src1, $src2" %}
13812
13813 ins_encode %{
13814 __ fmuls(as_FloatRegister($dst$$reg),
13815 as_FloatRegister($src1$$reg),
13816 as_FloatRegister($src2$$reg));
13817 %}
13818
13819 ins_pipe(fp_dop_reg_reg_s);
13820 %}
13821
13822 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13823 match(Set dst (MulD src1 src2));
13824
13825 ins_cost(INSN_COST * 6);
13826 format %{ "fmuld $dst, $src1, $src2" %}
13827
13828 ins_encode %{
13829 __ fmuld(as_FloatRegister($dst$$reg),
13830 as_FloatRegister($src1$$reg),
13831 as_FloatRegister($src2$$reg));
13832 %}
13833
13834 ins_pipe(fp_dop_reg_reg_d);
13835 %}
13836
13837 // src1 * src2 + src3 (half-precision float)
13838 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13839 match(Set dst (FmaHF src3 (Binary src1 src2)));
13840 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13841 ins_encode %{
13842 assert(UseFMA, "Needs FMA instructions support.");
13843 __ fmaddh($dst$$FloatRegister,
13844 $src1$$FloatRegister,
13845 $src2$$FloatRegister,
13846 $src3$$FloatRegister);
13847 %}
13848 ins_pipe(pipe_class_default);
13849 %}
13850
13851 // src1 * src2 + src3
13852 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13853 match(Set dst (FmaF src3 (Binary src1 src2)));
13854
13855 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13856
13857 ins_encode %{
13858 assert(UseFMA, "Needs FMA instructions support.");
13859 __ fmadds(as_FloatRegister($dst$$reg),
13860 as_FloatRegister($src1$$reg),
13861 as_FloatRegister($src2$$reg),
13862 as_FloatRegister($src3$$reg));
13863 %}
13864
13865 ins_pipe(pipe_class_default);
13866 %}
13867
13868 // src1 * src2 + src3
13869 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13870 match(Set dst (FmaD src3 (Binary src1 src2)));
13871
13872 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13873
13874 ins_encode %{
13875 assert(UseFMA, "Needs FMA instructions support.");
13876 __ fmaddd(as_FloatRegister($dst$$reg),
13877 as_FloatRegister($src1$$reg),
13878 as_FloatRegister($src2$$reg),
13879 as_FloatRegister($src3$$reg));
13880 %}
13881
13882 ins_pipe(pipe_class_default);
13883 %}
13884
13885 // src1 * (-src2) + src3
13886 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13887 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13888 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13889
13890 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13891
13892 ins_encode %{
13893 assert(UseFMA, "Needs FMA instructions support.");
13894 __ fmsubs(as_FloatRegister($dst$$reg),
13895 as_FloatRegister($src1$$reg),
13896 as_FloatRegister($src2$$reg),
13897 as_FloatRegister($src3$$reg));
13898 %}
13899
13900 ins_pipe(pipe_class_default);
13901 %}
13902
13903 // src1 * (-src2) + src3
13904 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13905 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13906 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13907
13908 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13909
13910 ins_encode %{
13911 assert(UseFMA, "Needs FMA instructions support.");
13912 __ fmsubd(as_FloatRegister($dst$$reg),
13913 as_FloatRegister($src1$$reg),
13914 as_FloatRegister($src2$$reg),
13915 as_FloatRegister($src3$$reg));
13916 %}
13917
13918 ins_pipe(pipe_class_default);
13919 %}
13920
13921 // src1 * (-src2) - src3
13922 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13923 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13924 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13925
13926 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13927
13928 ins_encode %{
13929 assert(UseFMA, "Needs FMA instructions support.");
13930 __ fnmadds(as_FloatRegister($dst$$reg),
13931 as_FloatRegister($src1$$reg),
13932 as_FloatRegister($src2$$reg),
13933 as_FloatRegister($src3$$reg));
13934 %}
13935
13936 ins_pipe(pipe_class_default);
13937 %}
13938
13939 // src1 * (-src2) - src3
13940 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13941 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13942 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13943
13944 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13945
13946 ins_encode %{
13947 assert(UseFMA, "Needs FMA instructions support.");
13948 __ fnmaddd(as_FloatRegister($dst$$reg),
13949 as_FloatRegister($src1$$reg),
13950 as_FloatRegister($src2$$reg),
13951 as_FloatRegister($src3$$reg));
13952 %}
13953
13954 ins_pipe(pipe_class_default);
13955 %}
13956
13957 // src1 * src2 - src3
13958 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13959 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13960
13961 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13962
13963 ins_encode %{
13964 assert(UseFMA, "Needs FMA instructions support.");
13965 __ fnmsubs(as_FloatRegister($dst$$reg),
13966 as_FloatRegister($src1$$reg),
13967 as_FloatRegister($src2$$reg),
13968 as_FloatRegister($src3$$reg));
13969 %}
13970
13971 ins_pipe(pipe_class_default);
13972 %}
13973
13974 // src1 * src2 - src3
13975 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13976 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13977
13978 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13979
13980 ins_encode %{
13981 assert(UseFMA, "Needs FMA instructions support.");
13982 // n.b. insn name should be fnmsubd
13983 __ fnmsub(as_FloatRegister($dst$$reg),
13984 as_FloatRegister($src1$$reg),
13985 as_FloatRegister($src2$$reg),
13986 as_FloatRegister($src3$$reg));
13987 %}
13988
13989 ins_pipe(pipe_class_default);
13990 %}
13991
13992 // Math.max(HH)H (half-precision float)
13993 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13994 match(Set dst (MaxHF src1 src2));
13995 format %{ "fmaxh $dst, $src1, $src2" %}
13996 ins_encode %{
13997 __ fmaxh($dst$$FloatRegister,
13998 $src1$$FloatRegister,
13999 $src2$$FloatRegister);
14000 %}
14001 ins_pipe(fp_dop_reg_reg_s);
14002 %}
14003
14004 // Math.min(HH)H (half-precision float)
14005 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14006 match(Set dst (MinHF src1 src2));
14007 format %{ "fminh $dst, $src1, $src2" %}
14008 ins_encode %{
14009 __ fminh($dst$$FloatRegister,
14010 $src1$$FloatRegister,
14011 $src2$$FloatRegister);
14012 %}
14013 ins_pipe(fp_dop_reg_reg_s);
14014 %}
14015
14016 // Math.max(FF)F
14017 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14018 match(Set dst (MaxF src1 src2));
14019
14020 format %{ "fmaxs $dst, $src1, $src2" %}
14021 ins_encode %{
14022 __ fmaxs(as_FloatRegister($dst$$reg),
14023 as_FloatRegister($src1$$reg),
14024 as_FloatRegister($src2$$reg));
14025 %}
14026
14027 ins_pipe(fp_dop_reg_reg_s);
14028 %}
14029
14030 // Math.min(FF)F
14031 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14032 match(Set dst (MinF src1 src2));
14033
14034 format %{ "fmins $dst, $src1, $src2" %}
14035 ins_encode %{
14036 __ fmins(as_FloatRegister($dst$$reg),
14037 as_FloatRegister($src1$$reg),
14038 as_FloatRegister($src2$$reg));
14039 %}
14040
14041 ins_pipe(fp_dop_reg_reg_s);
14042 %}
14043
14044 // Math.max(DD)D
14045 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14046 match(Set dst (MaxD src1 src2));
14047
14048 format %{ "fmaxd $dst, $src1, $src2" %}
14049 ins_encode %{
14050 __ fmaxd(as_FloatRegister($dst$$reg),
14051 as_FloatRegister($src1$$reg),
14052 as_FloatRegister($src2$$reg));
14053 %}
14054
14055 ins_pipe(fp_dop_reg_reg_d);
14056 %}
14057
14058 // Math.min(DD)D
14059 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14060 match(Set dst (MinD src1 src2));
14061
14062 format %{ "fmind $dst, $src1, $src2" %}
14063 ins_encode %{
14064 __ fmind(as_FloatRegister($dst$$reg),
14065 as_FloatRegister($src1$$reg),
14066 as_FloatRegister($src2$$reg));
14067 %}
14068
14069 ins_pipe(fp_dop_reg_reg_d);
14070 %}
14071
14072 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14073 match(Set dst (DivHF src1 src2));
14074 format %{ "fdivh $dst, $src1, $src2" %}
14075 ins_encode %{
14076 __ fdivh($dst$$FloatRegister,
14077 $src1$$FloatRegister,
14078 $src2$$FloatRegister);
14079 %}
14080 ins_pipe(fp_div_s);
14081 %}
14082
14083 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14084 match(Set dst (DivF src1 src2));
14085
14086 ins_cost(INSN_COST * 18);
14087 format %{ "fdivs $dst, $src1, $src2" %}
14088
14089 ins_encode %{
14090 __ fdivs(as_FloatRegister($dst$$reg),
14091 as_FloatRegister($src1$$reg),
14092 as_FloatRegister($src2$$reg));
14093 %}
14094
14095 ins_pipe(fp_div_s);
14096 %}
14097
14098 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14099 match(Set dst (DivD src1 src2));
14100
14101 ins_cost(INSN_COST * 32);
14102 format %{ "fdivd $dst, $src1, $src2" %}
14103
14104 ins_encode %{
14105 __ fdivd(as_FloatRegister($dst$$reg),
14106 as_FloatRegister($src1$$reg),
14107 as_FloatRegister($src2$$reg));
14108 %}
14109
14110 ins_pipe(fp_div_d);
14111 %}
14112
14113 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14114 match(Set dst (NegF src));
14115
14116 ins_cost(INSN_COST * 3);
14117 format %{ "fneg $dst, $src" %}
14118
14119 ins_encode %{
14120 __ fnegs(as_FloatRegister($dst$$reg),
14121 as_FloatRegister($src$$reg));
14122 %}
14123
14124 ins_pipe(fp_uop_s);
14125 %}
14126
14127 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14128 match(Set dst (NegD src));
14129
14130 ins_cost(INSN_COST * 3);
14131 format %{ "fnegd $dst, $src" %}
14132
14133 ins_encode %{
14134 __ fnegd(as_FloatRegister($dst$$reg),
14135 as_FloatRegister($src$$reg));
14136 %}
14137
14138 ins_pipe(fp_uop_d);
14139 %}
14140
14141 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14142 %{
14143 match(Set dst (AbsI src));
14144
14145 effect(KILL cr);
14146 ins_cost(INSN_COST * 2);
14147 format %{ "cmpw $src, zr\n\t"
14148 "cnegw $dst, $src, Assembler::LT\t# int abs"
14149 %}
14150
14151 ins_encode %{
14152 __ cmpw(as_Register($src$$reg), zr);
14153 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14154 %}
14155 ins_pipe(pipe_class_default);
14156 %}
14157
14158 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14159 %{
14160 match(Set dst (AbsL src));
14161
14162 effect(KILL cr);
14163 ins_cost(INSN_COST * 2);
14164 format %{ "cmp $src, zr\n\t"
14165 "cneg $dst, $src, Assembler::LT\t# long abs"
14166 %}
14167
14168 ins_encode %{
14169 __ cmp(as_Register($src$$reg), zr);
14170 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14171 %}
14172 ins_pipe(pipe_class_default);
14173 %}
14174
14175 instruct absF_reg(vRegF dst, vRegF src) %{
14176 match(Set dst (AbsF src));
14177
14178 ins_cost(INSN_COST * 3);
14179 format %{ "fabss $dst, $src" %}
14180 ins_encode %{
14181 __ fabss(as_FloatRegister($dst$$reg),
14182 as_FloatRegister($src$$reg));
14183 %}
14184
14185 ins_pipe(fp_uop_s);
14186 %}
14187
14188 instruct absD_reg(vRegD dst, vRegD src) %{
14189 match(Set dst (AbsD src));
14190
14191 ins_cost(INSN_COST * 3);
14192 format %{ "fabsd $dst, $src" %}
14193 ins_encode %{
14194 __ fabsd(as_FloatRegister($dst$$reg),
14195 as_FloatRegister($src$$reg));
14196 %}
14197
14198 ins_pipe(fp_uop_d);
14199 %}
14200
14201 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14202 match(Set dst (AbsF (SubF src1 src2)));
14203
14204 ins_cost(INSN_COST * 3);
14205 format %{ "fabds $dst, $src1, $src2" %}
14206 ins_encode %{
14207 __ fabds(as_FloatRegister($dst$$reg),
14208 as_FloatRegister($src1$$reg),
14209 as_FloatRegister($src2$$reg));
14210 %}
14211
14212 ins_pipe(fp_uop_s);
14213 %}
14214
14215 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14216 match(Set dst (AbsD (SubD src1 src2)));
14217
14218 ins_cost(INSN_COST * 3);
14219 format %{ "fabdd $dst, $src1, $src2" %}
14220 ins_encode %{
14221 __ fabdd(as_FloatRegister($dst$$reg),
14222 as_FloatRegister($src1$$reg),
14223 as_FloatRegister($src2$$reg));
14224 %}
14225
14226 ins_pipe(fp_uop_d);
14227 %}
14228
14229 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14230 match(Set dst (SqrtD src));
14231
14232 ins_cost(INSN_COST * 50);
14233 format %{ "fsqrtd $dst, $src" %}
14234 ins_encode %{
14235 __ fsqrtd(as_FloatRegister($dst$$reg),
14236 as_FloatRegister($src$$reg));
14237 %}
14238
14239 ins_pipe(fp_div_s);
14240 %}
14241
14242 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14243 match(Set dst (SqrtF src));
14244
14245 ins_cost(INSN_COST * 50);
14246 format %{ "fsqrts $dst, $src" %}
14247 ins_encode %{
14248 __ fsqrts(as_FloatRegister($dst$$reg),
14249 as_FloatRegister($src$$reg));
14250 %}
14251
14252 ins_pipe(fp_div_d);
14253 %}
14254
14255 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14256 match(Set dst (SqrtHF src));
14257 format %{ "fsqrth $dst, $src" %}
14258 ins_encode %{
14259 __ fsqrth($dst$$FloatRegister,
14260 $src$$FloatRegister);
14261 %}
14262 ins_pipe(fp_div_s);
14263 %}
14264
14265 // Math.rint, floor, ceil
14266 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14267 match(Set dst (RoundDoubleMode src rmode));
14268 format %{ "frint $dst, $src, $rmode" %}
14269 ins_encode %{
14270 switch ($rmode$$constant) {
14271 case RoundDoubleModeNode::rmode_rint:
14272 __ frintnd(as_FloatRegister($dst$$reg),
14273 as_FloatRegister($src$$reg));
14274 break;
14275 case RoundDoubleModeNode::rmode_floor:
14276 __ frintmd(as_FloatRegister($dst$$reg),
14277 as_FloatRegister($src$$reg));
14278 break;
14279 case RoundDoubleModeNode::rmode_ceil:
14280 __ frintpd(as_FloatRegister($dst$$reg),
14281 as_FloatRegister($src$$reg));
14282 break;
14283 }
14284 %}
14285 ins_pipe(fp_uop_d);
14286 %}
14287
14288 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14289 match(Set dst (CopySignD src1 (Binary src2 zero)));
14290 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14291 format %{ "CopySignD $dst $src1 $src2" %}
14292 ins_encode %{
14293 FloatRegister dst = as_FloatRegister($dst$$reg),
14294 src1 = as_FloatRegister($src1$$reg),
14295 src2 = as_FloatRegister($src2$$reg),
14296 zero = as_FloatRegister($zero$$reg);
14297 __ fnegd(dst, zero);
14298 __ bsl(dst, __ T8B, src2, src1);
14299 %}
14300 ins_pipe(fp_uop_d);
14301 %}
14302
14303 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14304 match(Set dst (CopySignF src1 src2));
14305 effect(TEMP_DEF dst, USE src1, USE src2);
14306 format %{ "CopySignF $dst $src1 $src2" %}
14307 ins_encode %{
14308 FloatRegister dst = as_FloatRegister($dst$$reg),
14309 src1 = as_FloatRegister($src1$$reg),
14310 src2 = as_FloatRegister($src2$$reg);
14311 __ movi(dst, __ T2S, 0x80, 24);
14312 __ bsl(dst, __ T8B, src2, src1);
14313 %}
14314 ins_pipe(fp_uop_d);
14315 %}
14316
14317 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14318 match(Set dst (SignumD src (Binary zero one)));
14319 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14320 format %{ "signumD $dst, $src" %}
14321 ins_encode %{
14322 FloatRegister src = as_FloatRegister($src$$reg),
14323 dst = as_FloatRegister($dst$$reg),
14324 zero = as_FloatRegister($zero$$reg),
14325 one = as_FloatRegister($one$$reg);
14326 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14327 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14328 // Bit selection instruction gets bit from "one" for each enabled bit in
14329 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14330 // NaN the whole "src" will be copied because "dst" is zero. For all other
14331 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14332 // from "src", and all other bits are copied from 1.0.
14333 __ bsl(dst, __ T8B, one, src);
14334 %}
14335 ins_pipe(fp_uop_d);
14336 %}
14337
14338 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14339 match(Set dst (SignumF src (Binary zero one)));
14340 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14341 format %{ "signumF $dst, $src" %}
14342 ins_encode %{
14343 FloatRegister src = as_FloatRegister($src$$reg),
14344 dst = as_FloatRegister($dst$$reg),
14345 zero = as_FloatRegister($zero$$reg),
14346 one = as_FloatRegister($one$$reg);
14347 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14348 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14349 // Bit selection instruction gets bit from "one" for each enabled bit in
14350 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14351 // NaN the whole "src" will be copied because "dst" is zero. For all other
14352 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14353 // from "src", and all other bits are copied from 1.0.
14354 __ bsl(dst, __ T8B, one, src);
14355 %}
14356 ins_pipe(fp_uop_d);
14357 %}
14358
14359 instruct onspinwait() %{
14360 match(OnSpinWait);
14361 ins_cost(INSN_COST);
14362
14363 format %{ "onspinwait" %}
14364
14365 ins_encode %{
14366 __ spin_wait();
14367 %}
14368 ins_pipe(pipe_class_empty);
14369 %}
14370
14371 // ============================================================================
14372 // Logical Instructions
14373
14374 // Integer Logical Instructions
14375
14376 // And Instructions
14377
14378
14379 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14380 match(Set dst (AndI src1 src2));
14381
14382 format %{ "andw $dst, $src1, $src2\t# int" %}
14383
14384 ins_cost(INSN_COST);
14385 ins_encode %{
14386 __ andw(as_Register($dst$$reg),
14387 as_Register($src1$$reg),
14388 as_Register($src2$$reg));
14389 %}
14390
14391 ins_pipe(ialu_reg_reg);
14392 %}
14393
14394 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14395 match(Set dst (AndI src1 src2));
14396
14397 format %{ "andsw $dst, $src1, $src2\t# int" %}
14398
14399 ins_cost(INSN_COST);
14400 ins_encode %{
14401 __ andw(as_Register($dst$$reg),
14402 as_Register($src1$$reg),
14403 (uint64_t)($src2$$constant));
14404 %}
14405
14406 ins_pipe(ialu_reg_imm);
14407 %}
14408
14409 // Or Instructions
14410
14411 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14412 match(Set dst (OrI src1 src2));
14413
14414 format %{ "orrw $dst, $src1, $src2\t# int" %}
14415
14416 ins_cost(INSN_COST);
14417 ins_encode %{
14418 __ orrw(as_Register($dst$$reg),
14419 as_Register($src1$$reg),
14420 as_Register($src2$$reg));
14421 %}
14422
14423 ins_pipe(ialu_reg_reg);
14424 %}
14425
14426 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14427 match(Set dst (OrI src1 src2));
14428
14429 format %{ "orrw $dst, $src1, $src2\t# int" %}
14430
14431 ins_cost(INSN_COST);
14432 ins_encode %{
14433 __ orrw(as_Register($dst$$reg),
14434 as_Register($src1$$reg),
14435 (uint64_t)($src2$$constant));
14436 %}
14437
14438 ins_pipe(ialu_reg_imm);
14439 %}
14440
14441 // Xor Instructions
14442
14443 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14444 match(Set dst (XorI src1 src2));
14445
14446 format %{ "eorw $dst, $src1, $src2\t# int" %}
14447
14448 ins_cost(INSN_COST);
14449 ins_encode %{
14450 __ eorw(as_Register($dst$$reg),
14451 as_Register($src1$$reg),
14452 as_Register($src2$$reg));
14453 %}
14454
14455 ins_pipe(ialu_reg_reg);
14456 %}
14457
14458 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14459 match(Set dst (XorI src1 src2));
14460
14461 format %{ "eorw $dst, $src1, $src2\t# int" %}
14462
14463 ins_cost(INSN_COST);
14464 ins_encode %{
14465 __ eorw(as_Register($dst$$reg),
14466 as_Register($src1$$reg),
14467 (uint64_t)($src2$$constant));
14468 %}
14469
14470 ins_pipe(ialu_reg_imm);
14471 %}
14472
14473 // Long Logical Instructions
14474 // TODO
14475
14476 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14477 match(Set dst (AndL src1 src2));
14478
14479 format %{ "and $dst, $src1, $src2\t# int" %}
14480
14481 ins_cost(INSN_COST);
14482 ins_encode %{
14483 __ andr(as_Register($dst$$reg),
14484 as_Register($src1$$reg),
14485 as_Register($src2$$reg));
14486 %}
14487
14488 ins_pipe(ialu_reg_reg);
14489 %}
14490
14491 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14492 match(Set dst (AndL src1 src2));
14493
14494 format %{ "and $dst, $src1, $src2\t# int" %}
14495
14496 ins_cost(INSN_COST);
14497 ins_encode %{
14498 __ andr(as_Register($dst$$reg),
14499 as_Register($src1$$reg),
14500 (uint64_t)($src2$$constant));
14501 %}
14502
14503 ins_pipe(ialu_reg_imm);
14504 %}
14505
14506 // Or Instructions
14507
14508 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14509 match(Set dst (OrL src1 src2));
14510
14511 format %{ "orr $dst, $src1, $src2\t# int" %}
14512
14513 ins_cost(INSN_COST);
14514 ins_encode %{
14515 __ orr(as_Register($dst$$reg),
14516 as_Register($src1$$reg),
14517 as_Register($src2$$reg));
14518 %}
14519
14520 ins_pipe(ialu_reg_reg);
14521 %}
14522
14523 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14524 match(Set dst (OrL src1 src2));
14525
14526 format %{ "orr $dst, $src1, $src2\t# int" %}
14527
14528 ins_cost(INSN_COST);
14529 ins_encode %{
14530 __ orr(as_Register($dst$$reg),
14531 as_Register($src1$$reg),
14532 (uint64_t)($src2$$constant));
14533 %}
14534
14535 ins_pipe(ialu_reg_imm);
14536 %}
14537
14538 // Xor Instructions
14539
14540 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14541 match(Set dst (XorL src1 src2));
14542
14543 format %{ "eor $dst, $src1, $src2\t# int" %}
14544
14545 ins_cost(INSN_COST);
14546 ins_encode %{
14547 __ eor(as_Register($dst$$reg),
14548 as_Register($src1$$reg),
14549 as_Register($src2$$reg));
14550 %}
14551
14552 ins_pipe(ialu_reg_reg);
14553 %}
14554
14555 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14556 match(Set dst (XorL src1 src2));
14557
14558 ins_cost(INSN_COST);
14559 format %{ "eor $dst, $src1, $src2\t# int" %}
14560
14561 ins_encode %{
14562 __ eor(as_Register($dst$$reg),
14563 as_Register($src1$$reg),
14564 (uint64_t)($src2$$constant));
14565 %}
14566
14567 ins_pipe(ialu_reg_imm);
14568 %}
14569
14570 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14571 %{
14572 match(Set dst (ConvI2L src));
14573
14574 ins_cost(INSN_COST);
14575 format %{ "sxtw $dst, $src\t# i2l" %}
14576 ins_encode %{
14577 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14578 %}
14579 ins_pipe(ialu_reg_shift);
14580 %}
14581
14582 // this pattern occurs in bigmath arithmetic
14583 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14584 %{
14585 match(Set dst (AndL (ConvI2L src) mask));
14586
14587 ins_cost(INSN_COST);
14588 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14589 ins_encode %{
14590 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14591 %}
14592
14593 ins_pipe(ialu_reg_shift);
14594 %}
14595
14596 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14597 match(Set dst (ConvL2I src));
14598
14599 ins_cost(INSN_COST);
14600 format %{ "movw $dst, $src \t// l2i" %}
14601
14602 ins_encode %{
14603 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14604 %}
14605
14606 ins_pipe(ialu_reg);
14607 %}
14608
14609 instruct convD2F_reg(vRegF dst, vRegD src) %{
14610 match(Set dst (ConvD2F src));
14611
14612 ins_cost(INSN_COST * 5);
14613 format %{ "fcvtd $dst, $src \t// d2f" %}
14614
14615 ins_encode %{
14616 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14617 %}
14618
14619 ins_pipe(fp_d2f);
14620 %}
14621
14622 instruct convF2D_reg(vRegD dst, vRegF src) %{
14623 match(Set dst (ConvF2D src));
14624
14625 ins_cost(INSN_COST * 5);
14626 format %{ "fcvts $dst, $src \t// f2d" %}
14627
14628 ins_encode %{
14629 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14630 %}
14631
14632 ins_pipe(fp_f2d);
14633 %}
14634
14635 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14636 match(Set dst (ConvF2I src));
14637
14638 ins_cost(INSN_COST * 5);
14639 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14640
14641 ins_encode %{
14642 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14643 %}
14644
14645 ins_pipe(fp_f2i);
14646 %}
14647
14648 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14649 match(Set dst (ConvF2L src));
14650
14651 ins_cost(INSN_COST * 5);
14652 format %{ "fcvtzs $dst, $src \t// f2l" %}
14653
14654 ins_encode %{
14655 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14656 %}
14657
14658 ins_pipe(fp_f2l);
14659 %}
14660
14661 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14662 match(Set dst (ConvF2HF src));
14663 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14664 "smov $dst, $tmp\t# move result from $tmp to $dst"
14665 %}
14666 effect(TEMP tmp);
14667 ins_encode %{
14668 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14669 %}
14670 ins_pipe(pipe_slow);
14671 %}
14672
14673 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14674 match(Set dst (ConvHF2F src));
14675 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14676 "fcvt $dst, $tmp\t# convert half to single precision"
14677 %}
14678 effect(TEMP tmp);
14679 ins_encode %{
14680 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14681 %}
14682 ins_pipe(pipe_slow);
14683 %}
14684
14685 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14686 match(Set dst (ConvI2F src));
14687
14688 ins_cost(INSN_COST * 5);
14689 format %{ "scvtfws $dst, $src \t// i2f" %}
14690
14691 ins_encode %{
14692 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14693 %}
14694
14695 ins_pipe(fp_i2f);
14696 %}
14697
14698 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14699 match(Set dst (ConvL2F src));
14700
14701 ins_cost(INSN_COST * 5);
14702 format %{ "scvtfs $dst, $src \t// l2f" %}
14703
14704 ins_encode %{
14705 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14706 %}
14707
14708 ins_pipe(fp_l2f);
14709 %}
14710
14711 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14712 match(Set dst (ConvD2I src));
14713
14714 ins_cost(INSN_COST * 5);
14715 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14716
14717 ins_encode %{
14718 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14719 %}
14720
14721 ins_pipe(fp_d2i);
14722 %}
14723
14724 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14725 match(Set dst (ConvD2L src));
14726
14727 ins_cost(INSN_COST * 5);
14728 format %{ "fcvtzd $dst, $src \t// d2l" %}
14729
14730 ins_encode %{
14731 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14732 %}
14733
14734 ins_pipe(fp_d2l);
14735 %}
14736
14737 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14738 match(Set dst (ConvI2D src));
14739
14740 ins_cost(INSN_COST * 5);
14741 format %{ "scvtfwd $dst, $src \t// i2d" %}
14742
14743 ins_encode %{
14744 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14745 %}
14746
14747 ins_pipe(fp_i2d);
14748 %}
14749
14750 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14751 match(Set dst (ConvL2D src));
14752
14753 ins_cost(INSN_COST * 5);
14754 format %{ "scvtfd $dst, $src \t// l2d" %}
14755
14756 ins_encode %{
14757 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14758 %}
14759
14760 ins_pipe(fp_l2d);
14761 %}
14762
14763 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14764 %{
14765 match(Set dst (RoundD src));
14766 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14767 format %{ "java_round_double $dst,$src"%}
14768 ins_encode %{
14769 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14770 as_FloatRegister($ftmp$$reg));
14771 %}
14772 ins_pipe(pipe_slow);
14773 %}
14774
14775 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14776 %{
14777 match(Set dst (RoundF src));
14778 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14779 format %{ "java_round_float $dst,$src"%}
14780 ins_encode %{
14781 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14782 as_FloatRegister($ftmp$$reg));
14783 %}
14784 ins_pipe(pipe_slow);
14785 %}
14786
14787 // stack <-> reg and reg <-> reg shuffles with no conversion
14788
14789 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14790
14791 match(Set dst (MoveF2I src));
14792
14793 effect(DEF dst, USE src);
14794
14795 ins_cost(4 * INSN_COST);
14796
14797 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14798
14799 ins_encode %{
14800 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14801 %}
14802
14803 ins_pipe(iload_reg_reg);
14804
14805 %}
14806
14807 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14808
14809 match(Set dst (MoveI2F src));
14810
14811 effect(DEF dst, USE src);
14812
14813 ins_cost(4 * INSN_COST);
14814
14815 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14816
14817 ins_encode %{
14818 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14819 %}
14820
14821 ins_pipe(pipe_class_memory);
14822
14823 %}
14824
14825 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14826
14827 match(Set dst (MoveD2L src));
14828
14829 effect(DEF dst, USE src);
14830
14831 ins_cost(4 * INSN_COST);
14832
14833 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14834
14835 ins_encode %{
14836 __ ldr($dst$$Register, Address(sp, $src$$disp));
14837 %}
14838
14839 ins_pipe(iload_reg_reg);
14840
14841 %}
14842
14843 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14844
14845 match(Set dst (MoveL2D src));
14846
14847 effect(DEF dst, USE src);
14848
14849 ins_cost(4 * INSN_COST);
14850
14851 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14852
14853 ins_encode %{
14854 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14855 %}
14856
14857 ins_pipe(pipe_class_memory);
14858
14859 %}
14860
14861 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14862
14863 match(Set dst (MoveF2I src));
14864
14865 effect(DEF dst, USE src);
14866
14867 ins_cost(INSN_COST);
14868
14869 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14870
14871 ins_encode %{
14872 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14873 %}
14874
14875 ins_pipe(pipe_class_memory);
14876
14877 %}
14878
14879 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14880
14881 match(Set dst (MoveI2F src));
14882
14883 effect(DEF dst, USE src);
14884
14885 ins_cost(INSN_COST);
14886
14887 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14888
14889 ins_encode %{
14890 __ strw($src$$Register, Address(sp, $dst$$disp));
14891 %}
14892
14893 ins_pipe(istore_reg_reg);
14894
14895 %}
14896
14897 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14898
14899 match(Set dst (MoveD2L src));
14900
14901 effect(DEF dst, USE src);
14902
14903 ins_cost(INSN_COST);
14904
14905 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14906
14907 ins_encode %{
14908 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14909 %}
14910
14911 ins_pipe(pipe_class_memory);
14912
14913 %}
14914
14915 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14916
14917 match(Set dst (MoveL2D src));
14918
14919 effect(DEF dst, USE src);
14920
14921 ins_cost(INSN_COST);
14922
14923 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14924
14925 ins_encode %{
14926 __ str($src$$Register, Address(sp, $dst$$disp));
14927 %}
14928
14929 ins_pipe(istore_reg_reg);
14930
14931 %}
14932
14933 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14934
14935 match(Set dst (MoveF2I src));
14936
14937 effect(DEF dst, USE src);
14938
14939 ins_cost(INSN_COST);
14940
14941 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14942
14943 ins_encode %{
14944 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14945 %}
14946
14947 ins_pipe(fp_f2i);
14948
14949 %}
14950
14951 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14952
14953 match(Set dst (MoveI2F src));
14954
14955 effect(DEF dst, USE src);
14956
14957 ins_cost(INSN_COST);
14958
14959 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14960
14961 ins_encode %{
14962 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14963 %}
14964
14965 ins_pipe(fp_i2f);
14966
14967 %}
14968
14969 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14970
14971 match(Set dst (MoveD2L src));
14972
14973 effect(DEF dst, USE src);
14974
14975 ins_cost(INSN_COST);
14976
14977 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14978
14979 ins_encode %{
14980 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14981 %}
14982
14983 ins_pipe(fp_d2l);
14984
14985 %}
14986
14987 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14988
14989 match(Set dst (MoveL2D src));
14990
14991 effect(DEF dst, USE src);
14992
14993 ins_cost(INSN_COST);
14994
14995 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14996
14997 ins_encode %{
14998 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14999 %}
15000
15001 ins_pipe(fp_l2d);
15002
15003 %}
15004
15005 // ============================================================================
15006 // clearing of an array
15007
15008 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15009 %{
15010 match(Set dummy (ClearArray cnt base));
15011 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15012
15013 ins_cost(4 * INSN_COST);
15014 format %{ "ClearArray $cnt, $base" %}
15015
15016 ins_encode %{
15017 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15018 if (tpc == nullptr) {
15019 ciEnv::current()->record_failure("CodeCache is full");
15020 return;
15021 }
15022 %}
15023
15024 ins_pipe(pipe_class_memory);
15025 %}
15026
15027 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15028 %{
15029 predicate((uint64_t)n->in(2)->get_long()
15030 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15031 match(Set dummy (ClearArray cnt base));
15032 effect(TEMP temp, USE_KILL base, KILL cr);
15033
15034 ins_cost(4 * INSN_COST);
15035 format %{ "ClearArray $cnt, $base" %}
15036
15037 ins_encode %{
15038 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15039 if (tpc == nullptr) {
15040 ciEnv::current()->record_failure("CodeCache is full");
15041 return;
15042 }
15043 %}
15044
15045 ins_pipe(pipe_class_memory);
15046 %}
15047
15048 // ============================================================================
15049 // Overflow Math Instructions
15050
15051 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15052 %{
15053 match(Set cr (OverflowAddI op1 op2));
15054
15055 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15056 ins_cost(INSN_COST);
15057 ins_encode %{
15058 __ cmnw($op1$$Register, $op2$$Register);
15059 %}
15060
15061 ins_pipe(icmp_reg_reg);
15062 %}
15063
15064 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15065 %{
15066 match(Set cr (OverflowAddI op1 op2));
15067
15068 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15069 ins_cost(INSN_COST);
15070 ins_encode %{
15071 __ cmnw($op1$$Register, $op2$$constant);
15072 %}
15073
15074 ins_pipe(icmp_reg_imm);
15075 %}
15076
15077 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15078 %{
15079 match(Set cr (OverflowAddL op1 op2));
15080
15081 format %{ "cmn $op1, $op2\t# overflow check long" %}
15082 ins_cost(INSN_COST);
15083 ins_encode %{
15084 __ cmn($op1$$Register, $op2$$Register);
15085 %}
15086
15087 ins_pipe(icmp_reg_reg);
15088 %}
15089
15090 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15091 %{
15092 match(Set cr (OverflowAddL op1 op2));
15093
15094 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15095 ins_cost(INSN_COST);
15096 ins_encode %{
15097 __ adds(zr, $op1$$Register, $op2$$constant);
15098 %}
15099
15100 ins_pipe(icmp_reg_imm);
15101 %}
15102
15103 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15104 %{
15105 match(Set cr (OverflowSubI op1 op2));
15106
15107 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15108 ins_cost(INSN_COST);
15109 ins_encode %{
15110 __ cmpw($op1$$Register, $op2$$Register);
15111 %}
15112
15113 ins_pipe(icmp_reg_reg);
15114 %}
15115
15116 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15117 %{
15118 match(Set cr (OverflowSubI op1 op2));
15119
15120 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15121 ins_cost(INSN_COST);
15122 ins_encode %{
15123 __ cmpw($op1$$Register, $op2$$constant);
15124 %}
15125
15126 ins_pipe(icmp_reg_imm);
15127 %}
15128
15129 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15130 %{
15131 match(Set cr (OverflowSubL op1 op2));
15132
15133 format %{ "cmp $op1, $op2\t# overflow check long" %}
15134 ins_cost(INSN_COST);
15135 ins_encode %{
15136 __ cmp($op1$$Register, $op2$$Register);
15137 %}
15138
15139 ins_pipe(icmp_reg_reg);
15140 %}
15141
15142 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15143 %{
15144 match(Set cr (OverflowSubL op1 op2));
15145
15146 format %{ "cmp $op1, $op2\t# overflow check long" %}
15147 ins_cost(INSN_COST);
15148 ins_encode %{
15149 __ subs(zr, $op1$$Register, $op2$$constant);
15150 %}
15151
15152 ins_pipe(icmp_reg_imm);
15153 %}
15154
15155 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15156 %{
15157 match(Set cr (OverflowSubI zero op1));
15158
15159 format %{ "cmpw zr, $op1\t# overflow check int" %}
15160 ins_cost(INSN_COST);
15161 ins_encode %{
15162 __ cmpw(zr, $op1$$Register);
15163 %}
15164
15165 ins_pipe(icmp_reg_imm);
15166 %}
15167
15168 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15169 %{
15170 match(Set cr (OverflowSubL zero op1));
15171
15172 format %{ "cmp zr, $op1\t# overflow check long" %}
15173 ins_cost(INSN_COST);
15174 ins_encode %{
15175 __ cmp(zr, $op1$$Register);
15176 %}
15177
15178 ins_pipe(icmp_reg_imm);
15179 %}
15180
15181 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15182 %{
15183 match(Set cr (OverflowMulI op1 op2));
15184
15185 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15186 "cmp rscratch1, rscratch1, sxtw\n\t"
15187 "movw rscratch1, #0x80000000\n\t"
15188 "cselw rscratch1, rscratch1, zr, NE\n\t"
15189 "cmpw rscratch1, #1" %}
15190 ins_cost(5 * INSN_COST);
15191 ins_encode %{
15192 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15193 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15194 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15195 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15196 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15197 %}
15198
15199 ins_pipe(pipe_slow);
15200 %}
15201
15202 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15203 %{
15204 match(If cmp (OverflowMulI op1 op2));
15205 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15206 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15207 effect(USE labl, KILL cr);
15208
15209 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15210 "cmp rscratch1, rscratch1, sxtw\n\t"
15211 "b$cmp $labl" %}
15212 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15213 ins_encode %{
15214 Label* L = $labl$$label;
15215 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15216 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15217 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15218 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15219 %}
15220
15221 ins_pipe(pipe_serial);
15222 %}
15223
15224 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15225 %{
15226 match(Set cr (OverflowMulL op1 op2));
15227
15228 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15229 "smulh rscratch2, $op1, $op2\n\t"
15230 "cmp rscratch2, rscratch1, ASR #63\n\t"
15231 "movw rscratch1, #0x80000000\n\t"
15232 "cselw rscratch1, rscratch1, zr, NE\n\t"
15233 "cmpw rscratch1, #1" %}
15234 ins_cost(6 * INSN_COST);
15235 ins_encode %{
15236 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15237 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15238 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15239 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15240 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15241 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15242 %}
15243
15244 ins_pipe(pipe_slow);
15245 %}
15246
15247 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15248 %{
15249 match(If cmp (OverflowMulL op1 op2));
15250 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15251 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15252 effect(USE labl, KILL cr);
15253
15254 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15255 "smulh rscratch2, $op1, $op2\n\t"
15256 "cmp rscratch2, rscratch1, ASR #63\n\t"
15257 "b$cmp $labl" %}
15258 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15259 ins_encode %{
15260 Label* L = $labl$$label;
15261 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15262 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15263 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15264 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15265 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15266 %}
15267
15268 ins_pipe(pipe_serial);
15269 %}
15270
15271 // ============================================================================
15272 // Compare Instructions
15273
15274 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15275 %{
15276 match(Set cr (CmpI op1 op2));
15277
15278 effect(DEF cr, USE op1, USE op2);
15279
15280 ins_cost(INSN_COST);
15281 format %{ "cmpw $op1, $op2" %}
15282
15283 ins_encode(aarch64_enc_cmpw(op1, op2));
15284
15285 ins_pipe(icmp_reg_reg);
15286 %}
15287
15288 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15289 %{
15290 match(Set cr (CmpI op1 zero));
15291
15292 effect(DEF cr, USE op1);
15293
15294 ins_cost(INSN_COST);
15295 format %{ "cmpw $op1, 0" %}
15296
15297 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15298
15299 ins_pipe(icmp_reg_imm);
15300 %}
15301
15302 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15303 %{
15304 match(Set cr (CmpI op1 op2));
15305
15306 effect(DEF cr, USE op1);
15307
15308 ins_cost(INSN_COST);
15309 format %{ "cmpw $op1, $op2" %}
15310
15311 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15312
15313 ins_pipe(icmp_reg_imm);
15314 %}
15315
15316 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15317 %{
15318 match(Set cr (CmpI op1 op2));
15319
15320 effect(DEF cr, USE op1);
15321
15322 ins_cost(INSN_COST * 2);
15323 format %{ "cmpw $op1, $op2" %}
15324
15325 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15326
15327 ins_pipe(icmp_reg_imm);
15328 %}
15329
15330 // Unsigned compare Instructions; really, same as signed compare
15331 // except it should only be used to feed an If or a CMovI which takes a
15332 // cmpOpU.
15333
15334 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15335 %{
15336 match(Set cr (CmpU op1 op2));
15337
15338 effect(DEF cr, USE op1, USE op2);
15339
15340 ins_cost(INSN_COST);
15341 format %{ "cmpw $op1, $op2\t# unsigned" %}
15342
15343 ins_encode(aarch64_enc_cmpw(op1, op2));
15344
15345 ins_pipe(icmp_reg_reg);
15346 %}
15347
15348 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15349 %{
15350 match(Set cr (CmpU op1 zero));
15351
15352 effect(DEF cr, USE op1);
15353
15354 ins_cost(INSN_COST);
15355 format %{ "cmpw $op1, #0\t# unsigned" %}
15356
15357 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15358
15359 ins_pipe(icmp_reg_imm);
15360 %}
15361
15362 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15363 %{
15364 match(Set cr (CmpU op1 op2));
15365
15366 effect(DEF cr, USE op1);
15367
15368 ins_cost(INSN_COST);
15369 format %{ "cmpw $op1, $op2\t# unsigned" %}
15370
15371 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15372
15373 ins_pipe(icmp_reg_imm);
15374 %}
15375
15376 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15377 %{
15378 match(Set cr (CmpU op1 op2));
15379
15380 effect(DEF cr, USE op1);
15381
15382 ins_cost(INSN_COST * 2);
15383 format %{ "cmpw $op1, $op2\t# unsigned" %}
15384
15385 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15386
15387 ins_pipe(icmp_reg_imm);
15388 %}
15389
15390 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15391 %{
15392 match(Set cr (CmpL op1 op2));
15393
15394 effect(DEF cr, USE op1, USE op2);
15395
15396 ins_cost(INSN_COST);
15397 format %{ "cmp $op1, $op2" %}
15398
15399 ins_encode(aarch64_enc_cmp(op1, op2));
15400
15401 ins_pipe(icmp_reg_reg);
15402 %}
15403
15404 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15405 %{
15406 match(Set cr (CmpL op1 zero));
15407
15408 effect(DEF cr, USE op1);
15409
15410 ins_cost(INSN_COST);
15411 format %{ "tst $op1" %}
15412
15413 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15414
15415 ins_pipe(icmp_reg_imm);
15416 %}
15417
15418 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15419 %{
15420 match(Set cr (CmpL op1 op2));
15421
15422 effect(DEF cr, USE op1);
15423
15424 ins_cost(INSN_COST);
15425 format %{ "cmp $op1, $op2" %}
15426
15427 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15428
15429 ins_pipe(icmp_reg_imm);
15430 %}
15431
15432 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15433 %{
15434 match(Set cr (CmpL op1 op2));
15435
15436 effect(DEF cr, USE op1);
15437
15438 ins_cost(INSN_COST * 2);
15439 format %{ "cmp $op1, $op2" %}
15440
15441 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15442
15443 ins_pipe(icmp_reg_imm);
15444 %}
15445
15446 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15447 %{
15448 match(Set cr (CmpUL op1 op2));
15449
15450 effect(DEF cr, USE op1, USE op2);
15451
15452 ins_cost(INSN_COST);
15453 format %{ "cmp $op1, $op2" %}
15454
15455 ins_encode(aarch64_enc_cmp(op1, op2));
15456
15457 ins_pipe(icmp_reg_reg);
15458 %}
15459
15460 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15461 %{
15462 match(Set cr (CmpUL op1 zero));
15463
15464 effect(DEF cr, USE op1);
15465
15466 ins_cost(INSN_COST);
15467 format %{ "tst $op1" %}
15468
15469 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15470
15471 ins_pipe(icmp_reg_imm);
15472 %}
15473
15474 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15475 %{
15476 match(Set cr (CmpUL op1 op2));
15477
15478 effect(DEF cr, USE op1);
15479
15480 ins_cost(INSN_COST);
15481 format %{ "cmp $op1, $op2" %}
15482
15483 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15484
15485 ins_pipe(icmp_reg_imm);
15486 %}
15487
15488 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15489 %{
15490 match(Set cr (CmpUL op1 op2));
15491
15492 effect(DEF cr, USE op1);
15493
15494 ins_cost(INSN_COST * 2);
15495 format %{ "cmp $op1, $op2" %}
15496
15497 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15498
15499 ins_pipe(icmp_reg_imm);
15500 %}
15501
15502 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15503 %{
15504 match(Set cr (CmpP op1 op2));
15505
15506 effect(DEF cr, USE op1, USE op2);
15507
15508 ins_cost(INSN_COST);
15509 format %{ "cmp $op1, $op2\t // ptr" %}
15510
15511 ins_encode(aarch64_enc_cmpp(op1, op2));
15512
15513 ins_pipe(icmp_reg_reg);
15514 %}
15515
15516 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15517 %{
15518 match(Set cr (CmpN op1 op2));
15519
15520 effect(DEF cr, USE op1, USE op2);
15521
15522 ins_cost(INSN_COST);
15523 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15524
15525 ins_encode(aarch64_enc_cmpn(op1, op2));
15526
15527 ins_pipe(icmp_reg_reg);
15528 %}
15529
15530 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15531 %{
15532 match(Set cr (CmpP op1 zero));
15533
15534 effect(DEF cr, USE op1, USE zero);
15535
15536 ins_cost(INSN_COST);
15537 format %{ "cmp $op1, 0\t // ptr" %}
15538
15539 ins_encode(aarch64_enc_testp(op1));
15540
15541 ins_pipe(icmp_reg_imm);
15542 %}
15543
15544 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15545 %{
15546 match(Set cr (CmpN op1 zero));
15547
15548 effect(DEF cr, USE op1, USE zero);
15549
15550 ins_cost(INSN_COST);
15551 format %{ "cmp $op1, 0\t // compressed ptr" %}
15552
15553 ins_encode(aarch64_enc_testn(op1));
15554
15555 ins_pipe(icmp_reg_imm);
15556 %}
15557
15558 // FP comparisons
15559 //
15560 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15561 // using normal cmpOp. See declaration of rFlagsReg for details.
15562
15563 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15564 %{
15565 match(Set cr (CmpF src1 src2));
15566
15567 ins_cost(3 * INSN_COST);
15568 format %{ "fcmps $src1, $src2" %}
15569
15570 ins_encode %{
15571 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15572 %}
15573
15574 ins_pipe(pipe_class_compare);
15575 %}
15576
15577 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15578 %{
15579 match(Set cr (CmpF src1 src2));
15580
15581 ins_cost(3 * INSN_COST);
15582 format %{ "fcmps $src1, 0.0" %}
15583
15584 ins_encode %{
15585 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15586 %}
15587
15588 ins_pipe(pipe_class_compare);
15589 %}
15590 // FROM HERE
15591
15592 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15593 %{
15594 match(Set cr (CmpD src1 src2));
15595
15596 ins_cost(3 * INSN_COST);
15597 format %{ "fcmpd $src1, $src2" %}
15598
15599 ins_encode %{
15600 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15601 %}
15602
15603 ins_pipe(pipe_class_compare);
15604 %}
15605
15606 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15607 %{
15608 match(Set cr (CmpD src1 src2));
15609
15610 ins_cost(3 * INSN_COST);
15611 format %{ "fcmpd $src1, 0.0" %}
15612
15613 ins_encode %{
15614 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15615 %}
15616
15617 ins_pipe(pipe_class_compare);
15618 %}
15619
15620 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15621 %{
15622 match(Set dst (CmpF3 src1 src2));
15623 effect(KILL cr);
15624
15625 ins_cost(5 * INSN_COST);
15626 format %{ "fcmps $src1, $src2\n\t"
15627 "csinvw($dst, zr, zr, eq\n\t"
15628 "csnegw($dst, $dst, $dst, lt)"
15629 %}
15630
15631 ins_encode %{
15632 Label done;
15633 FloatRegister s1 = as_FloatRegister($src1$$reg);
15634 FloatRegister s2 = as_FloatRegister($src2$$reg);
15635 Register d = as_Register($dst$$reg);
15636 __ fcmps(s1, s2);
15637 // installs 0 if EQ else -1
15638 __ csinvw(d, zr, zr, Assembler::EQ);
15639 // keeps -1 if less or unordered else installs 1
15640 __ csnegw(d, d, d, Assembler::LT);
15641 __ bind(done);
15642 %}
15643
15644 ins_pipe(pipe_class_default);
15645
15646 %}
15647
15648 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15649 %{
15650 match(Set dst (CmpD3 src1 src2));
15651 effect(KILL cr);
15652
15653 ins_cost(5 * INSN_COST);
15654 format %{ "fcmpd $src1, $src2\n\t"
15655 "csinvw($dst, zr, zr, eq\n\t"
15656 "csnegw($dst, $dst, $dst, lt)"
15657 %}
15658
15659 ins_encode %{
15660 Label done;
15661 FloatRegister s1 = as_FloatRegister($src1$$reg);
15662 FloatRegister s2 = as_FloatRegister($src2$$reg);
15663 Register d = as_Register($dst$$reg);
15664 __ fcmpd(s1, s2);
15665 // installs 0 if EQ else -1
15666 __ csinvw(d, zr, zr, Assembler::EQ);
15667 // keeps -1 if less or unordered else installs 1
15668 __ csnegw(d, d, d, Assembler::LT);
15669 __ bind(done);
15670 %}
15671 ins_pipe(pipe_class_default);
15672
15673 %}
15674
15675 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15676 %{
15677 match(Set dst (CmpF3 src1 zero));
15678 effect(KILL cr);
15679
15680 ins_cost(5 * INSN_COST);
15681 format %{ "fcmps $src1, 0.0\n\t"
15682 "csinvw($dst, zr, zr, eq\n\t"
15683 "csnegw($dst, $dst, $dst, lt)"
15684 %}
15685
15686 ins_encode %{
15687 Label done;
15688 FloatRegister s1 = as_FloatRegister($src1$$reg);
15689 Register d = as_Register($dst$$reg);
15690 __ fcmps(s1, 0.0);
15691 // installs 0 if EQ else -1
15692 __ csinvw(d, zr, zr, Assembler::EQ);
15693 // keeps -1 if less or unordered else installs 1
15694 __ csnegw(d, d, d, Assembler::LT);
15695 __ bind(done);
15696 %}
15697
15698 ins_pipe(pipe_class_default);
15699
15700 %}
15701
15702 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15703 %{
15704 match(Set dst (CmpD3 src1 zero));
15705 effect(KILL cr);
15706
15707 ins_cost(5 * INSN_COST);
15708 format %{ "fcmpd $src1, 0.0\n\t"
15709 "csinvw($dst, zr, zr, eq\n\t"
15710 "csnegw($dst, $dst, $dst, lt)"
15711 %}
15712
15713 ins_encode %{
15714 Label done;
15715 FloatRegister s1 = as_FloatRegister($src1$$reg);
15716 Register d = as_Register($dst$$reg);
15717 __ fcmpd(s1, 0.0);
15718 // installs 0 if EQ else -1
15719 __ csinvw(d, zr, zr, Assembler::EQ);
15720 // keeps -1 if less or unordered else installs 1
15721 __ csnegw(d, d, d, Assembler::LT);
15722 __ bind(done);
15723 %}
15724 ins_pipe(pipe_class_default);
15725
15726 %}
15727
15728 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15729 %{
15730 match(Set dst (CmpLTMask p q));
15731 effect(KILL cr);
15732
15733 ins_cost(3 * INSN_COST);
15734
15735 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15736 "csetw $dst, lt\n\t"
15737 "subw $dst, zr, $dst"
15738 %}
15739
15740 ins_encode %{
15741 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15742 __ csetw(as_Register($dst$$reg), Assembler::LT);
15743 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15744 %}
15745
15746 ins_pipe(ialu_reg_reg);
15747 %}
15748
15749 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15750 %{
15751 match(Set dst (CmpLTMask src zero));
15752 effect(KILL cr);
15753
15754 ins_cost(INSN_COST);
15755
15756 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15757
15758 ins_encode %{
15759 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15760 %}
15761
15762 ins_pipe(ialu_reg_shift);
15763 %}
15764
15765 // ============================================================================
15766 // Max and Min
15767
15768 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15769
15770 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15771 %{
15772 effect(DEF cr, USE src);
15773 ins_cost(INSN_COST);
15774 format %{ "cmpw $src, 0" %}
15775
15776 ins_encode %{
15777 __ cmpw($src$$Register, 0);
15778 %}
15779 ins_pipe(icmp_reg_imm);
15780 %}
15781
15782 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15783 %{
15784 match(Set dst (MinI src1 src2));
15785 ins_cost(INSN_COST * 3);
15786
15787 expand %{
15788 rFlagsReg cr;
15789 compI_reg_reg(cr, src1, src2);
15790 cmovI_reg_reg_lt(dst, src1, src2, cr);
15791 %}
15792 %}
15793
15794 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15795 %{
15796 match(Set dst (MaxI src1 src2));
15797 ins_cost(INSN_COST * 3);
15798
15799 expand %{
15800 rFlagsReg cr;
15801 compI_reg_reg(cr, src1, src2);
15802 cmovI_reg_reg_gt(dst, src1, src2, cr);
15803 %}
15804 %}
15805
15806
15807 // ============================================================================
15808 // Branch Instructions
15809
15810 // Direct Branch.
15811 instruct branch(label lbl)
15812 %{
15813 match(Goto);
15814
15815 effect(USE lbl);
15816
15817 ins_cost(BRANCH_COST);
15818 format %{ "b $lbl" %}
15819
15820 ins_encode(aarch64_enc_b(lbl));
15821
15822 ins_pipe(pipe_branch);
15823 %}
15824
15825 // Conditional Near Branch
15826 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15827 %{
15828 // Same match rule as `branchConFar'.
15829 match(If cmp cr);
15830
15831 effect(USE lbl);
15832
15833 ins_cost(BRANCH_COST);
15834 // If set to 1 this indicates that the current instruction is a
15835 // short variant of a long branch. This avoids using this
15836 // instruction in first-pass matching. It will then only be used in
15837 // the `Shorten_branches' pass.
15838 // ins_short_branch(1);
15839 format %{ "b$cmp $lbl" %}
15840
15841 ins_encode(aarch64_enc_br_con(cmp, lbl));
15842
15843 ins_pipe(pipe_branch_cond);
15844 %}
15845
15846 // Conditional Near Branch Unsigned
15847 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15848 %{
15849 // Same match rule as `branchConFar'.
15850 match(If cmp cr);
15851
15852 effect(USE lbl);
15853
15854 ins_cost(BRANCH_COST);
15855 // If set to 1 this indicates that the current instruction is a
15856 // short variant of a long branch. This avoids using this
15857 // instruction in first-pass matching. It will then only be used in
15858 // the `Shorten_branches' pass.
15859 // ins_short_branch(1);
15860 format %{ "b$cmp $lbl\t# unsigned" %}
15861
15862 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15863
15864 ins_pipe(pipe_branch_cond);
15865 %}
15866
15867 // Make use of CBZ and CBNZ. These instructions, as well as being
15868 // shorter than (cmp; branch), have the additional benefit of not
15869 // killing the flags.
15870
15871 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15872 match(If cmp (CmpI op1 op2));
15873 effect(USE labl);
15874
15875 ins_cost(BRANCH_COST);
15876 format %{ "cbw$cmp $op1, $labl" %}
15877 ins_encode %{
15878 Label* L = $labl$$label;
15879 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15880 if (cond == Assembler::EQ)
15881 __ cbzw($op1$$Register, *L);
15882 else
15883 __ cbnzw($op1$$Register, *L);
15884 %}
15885 ins_pipe(pipe_cmp_branch);
15886 %}
15887
15888 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15889 match(If cmp (CmpL op1 op2));
15890 effect(USE labl);
15891
15892 ins_cost(BRANCH_COST);
15893 format %{ "cb$cmp $op1, $labl" %}
15894 ins_encode %{
15895 Label* L = $labl$$label;
15896 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15897 if (cond == Assembler::EQ)
15898 __ cbz($op1$$Register, *L);
15899 else
15900 __ cbnz($op1$$Register, *L);
15901 %}
15902 ins_pipe(pipe_cmp_branch);
15903 %}
15904
15905 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15906 match(If cmp (CmpP op1 op2));
15907 effect(USE labl);
15908
15909 ins_cost(BRANCH_COST);
15910 format %{ "cb$cmp $op1, $labl" %}
15911 ins_encode %{
15912 Label* L = $labl$$label;
15913 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15914 if (cond == Assembler::EQ)
15915 __ cbz($op1$$Register, *L);
15916 else
15917 __ cbnz($op1$$Register, *L);
15918 %}
15919 ins_pipe(pipe_cmp_branch);
15920 %}
15921
15922 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15923 match(If cmp (CmpN op1 op2));
15924 effect(USE labl);
15925
15926 ins_cost(BRANCH_COST);
15927 format %{ "cbw$cmp $op1, $labl" %}
15928 ins_encode %{
15929 Label* L = $labl$$label;
15930 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15931 if (cond == Assembler::EQ)
15932 __ cbzw($op1$$Register, *L);
15933 else
15934 __ cbnzw($op1$$Register, *L);
15935 %}
15936 ins_pipe(pipe_cmp_branch);
15937 %}
15938
15939 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15940 match(If cmp (CmpP (DecodeN oop) zero));
15941 effect(USE labl);
15942
15943 ins_cost(BRANCH_COST);
15944 format %{ "cb$cmp $oop, $labl" %}
15945 ins_encode %{
15946 Label* L = $labl$$label;
15947 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15948 if (cond == Assembler::EQ)
15949 __ cbzw($oop$$Register, *L);
15950 else
15951 __ cbnzw($oop$$Register, *L);
15952 %}
15953 ins_pipe(pipe_cmp_branch);
15954 %}
15955
15956 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15957 match(If cmp (CmpU op1 op2));
15958 effect(USE labl);
15959
15960 ins_cost(BRANCH_COST);
15961 format %{ "cbw$cmp $op1, $labl" %}
15962 ins_encode %{
15963 Label* L = $labl$$label;
15964 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15965 if (cond == Assembler::EQ || cond == Assembler::LS) {
15966 __ cbzw($op1$$Register, *L);
15967 } else {
15968 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15969 __ cbnzw($op1$$Register, *L);
15970 }
15971 %}
15972 ins_pipe(pipe_cmp_branch);
15973 %}
15974
15975 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15976 match(If cmp (CmpUL op1 op2));
15977 effect(USE labl);
15978
15979 ins_cost(BRANCH_COST);
15980 format %{ "cb$cmp $op1, $labl" %}
15981 ins_encode %{
15982 Label* L = $labl$$label;
15983 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15984 if (cond == Assembler::EQ || cond == Assembler::LS) {
15985 __ cbz($op1$$Register, *L);
15986 } else {
15987 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15988 __ cbnz($op1$$Register, *L);
15989 }
15990 %}
15991 ins_pipe(pipe_cmp_branch);
15992 %}
15993
15994 // Test bit and Branch
15995
15996 // Patterns for short (< 32KiB) variants
15997 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15998 match(If cmp (CmpL op1 op2));
15999 effect(USE labl);
16000
16001 ins_cost(BRANCH_COST);
16002 format %{ "cb$cmp $op1, $labl # long" %}
16003 ins_encode %{
16004 Label* L = $labl$$label;
16005 Assembler::Condition cond =
16006 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16007 __ tbr(cond, $op1$$Register, 63, *L);
16008 %}
16009 ins_pipe(pipe_cmp_branch);
16010 ins_short_branch(1);
16011 %}
16012
16013 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16014 match(If cmp (CmpI op1 op2));
16015 effect(USE labl);
16016
16017 ins_cost(BRANCH_COST);
16018 format %{ "cb$cmp $op1, $labl # int" %}
16019 ins_encode %{
16020 Label* L = $labl$$label;
16021 Assembler::Condition cond =
16022 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16023 __ tbr(cond, $op1$$Register, 31, *L);
16024 %}
16025 ins_pipe(pipe_cmp_branch);
16026 ins_short_branch(1);
16027 %}
16028
16029 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16030 match(If cmp (CmpL (AndL op1 op2) op3));
16031 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16032 effect(USE labl);
16033
16034 ins_cost(BRANCH_COST);
16035 format %{ "tb$cmp $op1, $op2, $labl" %}
16036 ins_encode %{
16037 Label* L = $labl$$label;
16038 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16039 int bit = exact_log2_long($op2$$constant);
16040 __ tbr(cond, $op1$$Register, bit, *L);
16041 %}
16042 ins_pipe(pipe_cmp_branch);
16043 ins_short_branch(1);
16044 %}
16045
16046 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16047 match(If cmp (CmpI (AndI op1 op2) op3));
16048 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16049 effect(USE labl);
16050
16051 ins_cost(BRANCH_COST);
16052 format %{ "tb$cmp $op1, $op2, $labl" %}
16053 ins_encode %{
16054 Label* L = $labl$$label;
16055 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16056 int bit = exact_log2((juint)$op2$$constant);
16057 __ tbr(cond, $op1$$Register, bit, *L);
16058 %}
16059 ins_pipe(pipe_cmp_branch);
16060 ins_short_branch(1);
16061 %}
16062
16063 // And far variants
16064 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16065 match(If cmp (CmpL op1 op2));
16066 effect(USE labl);
16067
16068 ins_cost(BRANCH_COST);
16069 format %{ "cb$cmp $op1, $labl # long" %}
16070 ins_encode %{
16071 Label* L = $labl$$label;
16072 Assembler::Condition cond =
16073 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16074 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16075 %}
16076 ins_pipe(pipe_cmp_branch);
16077 %}
16078
16079 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16080 match(If cmp (CmpI op1 op2));
16081 effect(USE labl);
16082
16083 ins_cost(BRANCH_COST);
16084 format %{ "cb$cmp $op1, $labl # int" %}
16085 ins_encode %{
16086 Label* L = $labl$$label;
16087 Assembler::Condition cond =
16088 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16089 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16090 %}
16091 ins_pipe(pipe_cmp_branch);
16092 %}
16093
16094 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16095 match(If cmp (CmpL (AndL op1 op2) op3));
16096 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16097 effect(USE labl);
16098
16099 ins_cost(BRANCH_COST);
16100 format %{ "tb$cmp $op1, $op2, $labl" %}
16101 ins_encode %{
16102 Label* L = $labl$$label;
16103 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16104 int bit = exact_log2_long($op2$$constant);
16105 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16106 %}
16107 ins_pipe(pipe_cmp_branch);
16108 %}
16109
16110 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16111 match(If cmp (CmpI (AndI op1 op2) op3));
16112 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16113 effect(USE labl);
16114
16115 ins_cost(BRANCH_COST);
16116 format %{ "tb$cmp $op1, $op2, $labl" %}
16117 ins_encode %{
16118 Label* L = $labl$$label;
16119 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16120 int bit = exact_log2((juint)$op2$$constant);
16121 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16122 %}
16123 ins_pipe(pipe_cmp_branch);
16124 %}
16125
16126 // Test bits
16127
16128 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16129 match(Set cr (CmpL (AndL op1 op2) op3));
16130 predicate(Assembler::operand_valid_for_logical_immediate
16131 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16132
16133 ins_cost(INSN_COST);
16134 format %{ "tst $op1, $op2 # long" %}
16135 ins_encode %{
16136 __ tst($op1$$Register, $op2$$constant);
16137 %}
16138 ins_pipe(ialu_reg_reg);
16139 %}
16140
16141 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16142 match(Set cr (CmpI (AndI op1 op2) op3));
16143 predicate(Assembler::operand_valid_for_logical_immediate
16144 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16145
16146 ins_cost(INSN_COST);
16147 format %{ "tst $op1, $op2 # int" %}
16148 ins_encode %{
16149 __ tstw($op1$$Register, $op2$$constant);
16150 %}
16151 ins_pipe(ialu_reg_reg);
16152 %}
16153
16154 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16155 match(Set cr (CmpL (AndL op1 op2) op3));
16156
16157 ins_cost(INSN_COST);
16158 format %{ "tst $op1, $op2 # long" %}
16159 ins_encode %{
16160 __ tst($op1$$Register, $op2$$Register);
16161 %}
16162 ins_pipe(ialu_reg_reg);
16163 %}
16164
16165 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16166 match(Set cr (CmpI (AndI op1 op2) op3));
16167
16168 ins_cost(INSN_COST);
16169 format %{ "tstw $op1, $op2 # int" %}
16170 ins_encode %{
16171 __ tstw($op1$$Register, $op2$$Register);
16172 %}
16173 ins_pipe(ialu_reg_reg);
16174 %}
16175
16176
16177 // Conditional Far Branch
16178 // Conditional Far Branch Unsigned
16179 // TODO: fixme
16180
16181 // counted loop end branch near
16182 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16183 %{
16184 match(CountedLoopEnd cmp cr);
16185
16186 effect(USE lbl);
16187
16188 ins_cost(BRANCH_COST);
16189 // short variant.
16190 // ins_short_branch(1);
16191 format %{ "b$cmp $lbl \t// counted loop end" %}
16192
16193 ins_encode(aarch64_enc_br_con(cmp, lbl));
16194
16195 ins_pipe(pipe_branch);
16196 %}
16197
16198 // counted loop end branch far
16199 // TODO: fixme
16200
16201 // ============================================================================
16202 // inlined locking and unlocking
16203
16204 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16205 %{
16206 predicate(LockingMode != LM_LIGHTWEIGHT);
16207 match(Set cr (FastLock object box));
16208 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16209
16210 ins_cost(5 * INSN_COST);
16211 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16212
16213 ins_encode %{
16214 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16215 %}
16216
16217 ins_pipe(pipe_serial);
16218 %}
16219
16220 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16221 %{
16222 predicate(LockingMode != LM_LIGHTWEIGHT);
16223 match(Set cr (FastUnlock object box));
16224 effect(TEMP tmp, TEMP tmp2);
16225
16226 ins_cost(5 * INSN_COST);
16227 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16228
16229 ins_encode %{
16230 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16231 %}
16232
16233 ins_pipe(pipe_serial);
16234 %}
16235
16236 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16237 %{
16238 predicate(LockingMode == LM_LIGHTWEIGHT);
16239 match(Set cr (FastLock object box));
16240 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16241
16242 ins_cost(5 * INSN_COST);
16243 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16244
16245 ins_encode %{
16246 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16247 %}
16248
16249 ins_pipe(pipe_serial);
16250 %}
16251
16252 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16253 %{
16254 predicate(LockingMode == LM_LIGHTWEIGHT);
16255 match(Set cr (FastUnlock object box));
16256 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16257
16258 ins_cost(5 * INSN_COST);
16259 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16260
16261 ins_encode %{
16262 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16263 %}
16264
16265 ins_pipe(pipe_serial);
16266 %}
16267
16268 // ============================================================================
16269 // Safepoint Instructions
16270
16271 // TODO
16272 // provide a near and far version of this code
16273
16274 instruct safePoint(rFlagsReg cr, iRegP poll)
16275 %{
16276 match(SafePoint poll);
16277 effect(KILL cr);
16278
16279 format %{
16280 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16281 %}
16282 ins_encode %{
16283 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16284 %}
16285 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16286 %}
16287
16288
16289 // ============================================================================
16290 // Procedure Call/Return Instructions
16291
16292 // Call Java Static Instruction
16293
16294 instruct CallStaticJavaDirect(method meth)
16295 %{
16296 match(CallStaticJava);
16297
16298 effect(USE meth);
16299
16300 ins_cost(CALL_COST);
16301
16302 format %{ "call,static $meth \t// ==> " %}
16303
16304 ins_encode(aarch64_enc_java_static_call(meth),
16305 aarch64_enc_call_epilog);
16306
16307 ins_pipe(pipe_class_call);
16308 %}
16309
16310 // TO HERE
16311
16312 // Call Java Dynamic Instruction
16313 instruct CallDynamicJavaDirect(method meth)
16314 %{
16315 match(CallDynamicJava);
16316
16317 effect(USE meth);
16318
16319 ins_cost(CALL_COST);
16320
16321 format %{ "CALL,dynamic $meth \t// ==> " %}
16322
16323 ins_encode(aarch64_enc_java_dynamic_call(meth),
16324 aarch64_enc_call_epilog);
16325
16326 ins_pipe(pipe_class_call);
16327 %}
16328
16329 // Call Runtime Instruction
16330
16331 instruct CallRuntimeDirect(method meth)
16332 %{
16333 match(CallRuntime);
16334
16335 effect(USE meth);
16336
16337 ins_cost(CALL_COST);
16338
16339 format %{ "CALL, runtime $meth" %}
16340
16341 ins_encode( aarch64_enc_java_to_runtime(meth) );
16342
16343 ins_pipe(pipe_class_call);
16344 %}
16345
16346 // Call Runtime Instruction
16347
16348 instruct CallLeafDirect(method meth)
16349 %{
16350 match(CallLeaf);
16351
16352 effect(USE meth);
16353
16354 ins_cost(CALL_COST);
16355
16356 format %{ "CALL, runtime leaf $meth" %}
16357
16358 ins_encode( aarch64_enc_java_to_runtime(meth) );
16359
16360 ins_pipe(pipe_class_call);
16361 %}
16362
16363 // Call Runtime Instruction without safepoint and with vector arguments
16364 instruct CallLeafDirectVector(method meth)
16365 %{
16366 match(CallLeafVector);
16367
16368 effect(USE meth);
16369
16370 ins_cost(CALL_COST);
16371
16372 format %{ "CALL, runtime leaf vector $meth" %}
16373
16374 ins_encode(aarch64_enc_java_to_runtime(meth));
16375
16376 ins_pipe(pipe_class_call);
16377 %}
16378
16379 // Call Runtime Instruction
16380
16381 instruct CallLeafNoFPDirect(method meth)
16382 %{
16383 match(CallLeafNoFP);
16384
16385 effect(USE meth);
16386
16387 ins_cost(CALL_COST);
16388
16389 format %{ "CALL, runtime leaf nofp $meth" %}
16390
16391 ins_encode( aarch64_enc_java_to_runtime(meth) );
16392
16393 ins_pipe(pipe_class_call);
16394 %}
16395
16396 // Tail Call; Jump from runtime stub to Java code.
16397 // Also known as an 'interprocedural jump'.
16398 // Target of jump will eventually return to caller.
16399 // TailJump below removes the return address.
16400 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16401 // emitted just above the TailCall which has reset rfp to the caller state.
16402 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16403 %{
16404 match(TailCall jump_target method_ptr);
16405
16406 ins_cost(CALL_COST);
16407
16408 format %{ "br $jump_target\t# $method_ptr holds method" %}
16409
16410 ins_encode(aarch64_enc_tail_call(jump_target));
16411
16412 ins_pipe(pipe_class_call);
16413 %}
16414
16415 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16416 %{
16417 match(TailJump jump_target ex_oop);
16418
16419 ins_cost(CALL_COST);
16420
16421 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16422
16423 ins_encode(aarch64_enc_tail_jmp(jump_target));
16424
16425 ins_pipe(pipe_class_call);
16426 %}
16427
16428 // Forward exception.
16429 instruct ForwardExceptionjmp()
16430 %{
16431 match(ForwardException);
16432 ins_cost(CALL_COST);
16433
16434 format %{ "b forward_exception_stub" %}
16435 ins_encode %{
16436 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16437 %}
16438 ins_pipe(pipe_class_call);
16439 %}
16440
16441 // Create exception oop: created by stack-crawling runtime code.
16442 // Created exception is now available to this handler, and is setup
16443 // just prior to jumping to this handler. No code emitted.
16444 // TODO check
16445 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16446 instruct CreateException(iRegP_R0 ex_oop)
16447 %{
16448 match(Set ex_oop (CreateEx));
16449
16450 format %{ " -- \t// exception oop; no code emitted" %}
16451
16452 size(0);
16453
16454 ins_encode( /*empty*/ );
16455
16456 ins_pipe(pipe_class_empty);
16457 %}
16458
16459 // Rethrow exception: The exception oop will come in the first
16460 // argument position. Then JUMP (not call) to the rethrow stub code.
16461 instruct RethrowException() %{
16462 match(Rethrow);
16463 ins_cost(CALL_COST);
16464
16465 format %{ "b rethrow_stub" %}
16466
16467 ins_encode( aarch64_enc_rethrow() );
16468
16469 ins_pipe(pipe_class_call);
16470 %}
16471
16472
16473 // Return Instruction
16474 // epilog node loads ret address into lr as part of frame pop
16475 instruct Ret()
16476 %{
16477 match(Return);
16478
16479 format %{ "ret\t// return register" %}
16480
16481 ins_encode( aarch64_enc_ret() );
16482
16483 ins_pipe(pipe_branch);
16484 %}
16485
16486 // Die now.
16487 instruct ShouldNotReachHere() %{
16488 match(Halt);
16489
16490 ins_cost(CALL_COST);
16491 format %{ "ShouldNotReachHere" %}
16492
16493 ins_encode %{
16494 if (is_reachable()) {
16495 const char* str = __ code_string(_halt_reason);
16496 __ stop(str);
16497 }
16498 %}
16499
16500 ins_pipe(pipe_class_default);
16501 %}
16502
16503 // ============================================================================
16504 // Partial Subtype Check
16505 //
16506 // superklass array for an instance of the superklass. Set a hidden
16507 // internal cache on a hit (cache is checked with exposed code in
16508 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16509 // encoding ALSO sets flags.
16510
16511 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16512 %{
16513 match(Set result (PartialSubtypeCheck sub super));
16514 predicate(!UseSecondarySupersTable);
16515 effect(KILL cr, KILL temp);
16516
16517 ins_cost(20 * INSN_COST); // slightly larger than the next version
16518 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16519
16520 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16521
16522 opcode(0x1); // Force zero of result reg on hit
16523
16524 ins_pipe(pipe_class_memory);
16525 %}
16526
16527 // Two versions of partialSubtypeCheck, both used when we need to
16528 // search for a super class in the secondary supers array. The first
16529 // is used when we don't know _a priori_ the class being searched
16530 // for. The second, far more common, is used when we do know: this is
16531 // used for instanceof, checkcast, and any case where C2 can determine
16532 // it by constant propagation.
16533
16534 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16535 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16536 rFlagsReg cr)
16537 %{
16538 match(Set result (PartialSubtypeCheck sub super));
16539 predicate(UseSecondarySupersTable);
16540 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16541
16542 ins_cost(10 * INSN_COST); // slightly larger than the next version
16543 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16544
16545 ins_encode %{
16546 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16547 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16548 $vtemp$$FloatRegister,
16549 $result$$Register, /*L_success*/nullptr);
16550 %}
16551
16552 ins_pipe(pipe_class_memory);
16553 %}
16554
16555 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16556 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16557 rFlagsReg cr)
16558 %{
16559 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16560 predicate(UseSecondarySupersTable);
16561 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16562
16563 ins_cost(5 * INSN_COST); // smaller than the next version
16564 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16565
16566 ins_encode %{
16567 bool success = false;
16568 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16569 if (InlineSecondarySupersTest) {
16570 success =
16571 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16572 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16573 $vtemp$$FloatRegister,
16574 $result$$Register,
16575 super_klass_slot);
16576 } else {
16577 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16578 success = (call != nullptr);
16579 }
16580 if (!success) {
16581 ciEnv::current()->record_failure("CodeCache is full");
16582 return;
16583 }
16584 %}
16585
16586 ins_pipe(pipe_class_memory);
16587 %}
16588
16589 // Intrisics for String.compareTo()
16590
16591 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16592 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16593 %{
16594 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16595 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16596 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16597
16598 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16599 ins_encode %{
16600 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16601 __ string_compare($str1$$Register, $str2$$Register,
16602 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16603 $tmp1$$Register, $tmp2$$Register,
16604 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16605 %}
16606 ins_pipe(pipe_class_memory);
16607 %}
16608
16609 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16610 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16611 %{
16612 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16613 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16614 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16615
16616 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16617 ins_encode %{
16618 __ string_compare($str1$$Register, $str2$$Register,
16619 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16620 $tmp1$$Register, $tmp2$$Register,
16621 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16622 %}
16623 ins_pipe(pipe_class_memory);
16624 %}
16625
16626 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16627 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16628 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16629 %{
16630 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16631 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16632 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16633 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16634
16635 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16636 ins_encode %{
16637 __ string_compare($str1$$Register, $str2$$Register,
16638 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16639 $tmp1$$Register, $tmp2$$Register,
16640 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16641 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16642 %}
16643 ins_pipe(pipe_class_memory);
16644 %}
16645
16646 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16647 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16648 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16649 %{
16650 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16651 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16652 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16653 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16654
16655 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16656 ins_encode %{
16657 __ string_compare($str1$$Register, $str2$$Register,
16658 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16659 $tmp1$$Register, $tmp2$$Register,
16660 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16661 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16662 %}
16663 ins_pipe(pipe_class_memory);
16664 %}
16665
16666 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16667 // these string_compare variants as NEON register type for convenience so that the prototype of
16668 // string_compare can be shared with all variants.
16669
16670 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16671 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16672 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16673 pRegGov_P1 pgtmp2, rFlagsReg cr)
16674 %{
16675 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16676 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16677 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16678 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16679
16680 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16681 ins_encode %{
16682 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16683 __ string_compare($str1$$Register, $str2$$Register,
16684 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16685 $tmp1$$Register, $tmp2$$Register,
16686 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16687 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16688 StrIntrinsicNode::LL);
16689 %}
16690 ins_pipe(pipe_class_memory);
16691 %}
16692
16693 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16694 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16695 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16696 pRegGov_P1 pgtmp2, rFlagsReg cr)
16697 %{
16698 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16699 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16700 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16701 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16702
16703 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16704 ins_encode %{
16705 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16706 __ string_compare($str1$$Register, $str2$$Register,
16707 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16708 $tmp1$$Register, $tmp2$$Register,
16709 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16710 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16711 StrIntrinsicNode::LU);
16712 %}
16713 ins_pipe(pipe_class_memory);
16714 %}
16715
16716 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16717 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16718 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16719 pRegGov_P1 pgtmp2, rFlagsReg cr)
16720 %{
16721 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16722 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16723 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16724 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16725
16726 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16727 ins_encode %{
16728 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16729 __ string_compare($str1$$Register, $str2$$Register,
16730 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16731 $tmp1$$Register, $tmp2$$Register,
16732 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16733 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16734 StrIntrinsicNode::UL);
16735 %}
16736 ins_pipe(pipe_class_memory);
16737 %}
16738
16739 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16740 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16741 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16742 pRegGov_P1 pgtmp2, rFlagsReg cr)
16743 %{
16744 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16745 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16746 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16747 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16748
16749 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16750 ins_encode %{
16751 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16752 __ string_compare($str1$$Register, $str2$$Register,
16753 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16754 $tmp1$$Register, $tmp2$$Register,
16755 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16756 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16757 StrIntrinsicNode::UU);
16758 %}
16759 ins_pipe(pipe_class_memory);
16760 %}
16761
16762 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16763 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16764 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16765 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16766 %{
16767 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16768 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16769 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16770 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16771 TEMP vtmp0, TEMP vtmp1, KILL cr);
16772 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16773 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16774
16775 ins_encode %{
16776 __ string_indexof($str1$$Register, $str2$$Register,
16777 $cnt1$$Register, $cnt2$$Register,
16778 $tmp1$$Register, $tmp2$$Register,
16779 $tmp3$$Register, $tmp4$$Register,
16780 $tmp5$$Register, $tmp6$$Register,
16781 -1, $result$$Register, StrIntrinsicNode::UU);
16782 %}
16783 ins_pipe(pipe_class_memory);
16784 %}
16785
16786 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16787 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16788 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16789 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16790 %{
16791 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16792 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16793 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16794 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16795 TEMP vtmp0, TEMP vtmp1, KILL cr);
16796 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16797 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16798
16799 ins_encode %{
16800 __ string_indexof($str1$$Register, $str2$$Register,
16801 $cnt1$$Register, $cnt2$$Register,
16802 $tmp1$$Register, $tmp2$$Register,
16803 $tmp3$$Register, $tmp4$$Register,
16804 $tmp5$$Register, $tmp6$$Register,
16805 -1, $result$$Register, StrIntrinsicNode::LL);
16806 %}
16807 ins_pipe(pipe_class_memory);
16808 %}
16809
16810 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16811 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16812 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16813 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16814 %{
16815 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16816 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16817 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16818 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16819 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16820 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16821 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16822
16823 ins_encode %{
16824 __ string_indexof($str1$$Register, $str2$$Register,
16825 $cnt1$$Register, $cnt2$$Register,
16826 $tmp1$$Register, $tmp2$$Register,
16827 $tmp3$$Register, $tmp4$$Register,
16828 $tmp5$$Register, $tmp6$$Register,
16829 -1, $result$$Register, StrIntrinsicNode::UL);
16830 %}
16831 ins_pipe(pipe_class_memory);
16832 %}
16833
16834 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16835 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16836 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16837 %{
16838 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16839 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16840 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16841 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16842 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16843 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16844
16845 ins_encode %{
16846 int icnt2 = (int)$int_cnt2$$constant;
16847 __ string_indexof($str1$$Register, $str2$$Register,
16848 $cnt1$$Register, zr,
16849 $tmp1$$Register, $tmp2$$Register,
16850 $tmp3$$Register, $tmp4$$Register, zr, zr,
16851 icnt2, $result$$Register, StrIntrinsicNode::UU);
16852 %}
16853 ins_pipe(pipe_class_memory);
16854 %}
16855
16856 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16857 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16858 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16859 %{
16860 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16861 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16862 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16863 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16864 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16865 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16866
16867 ins_encode %{
16868 int icnt2 = (int)$int_cnt2$$constant;
16869 __ string_indexof($str1$$Register, $str2$$Register,
16870 $cnt1$$Register, zr,
16871 $tmp1$$Register, $tmp2$$Register,
16872 $tmp3$$Register, $tmp4$$Register, zr, zr,
16873 icnt2, $result$$Register, StrIntrinsicNode::LL);
16874 %}
16875 ins_pipe(pipe_class_memory);
16876 %}
16877
16878 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16879 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16880 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16881 %{
16882 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16883 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16884 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16885 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16886 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16887 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16888
16889 ins_encode %{
16890 int icnt2 = (int)$int_cnt2$$constant;
16891 __ string_indexof($str1$$Register, $str2$$Register,
16892 $cnt1$$Register, zr,
16893 $tmp1$$Register, $tmp2$$Register,
16894 $tmp3$$Register, $tmp4$$Register, zr, zr,
16895 icnt2, $result$$Register, StrIntrinsicNode::UL);
16896 %}
16897 ins_pipe(pipe_class_memory);
16898 %}
16899
16900 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16901 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16902 iRegINoSp tmp3, rFlagsReg cr)
16903 %{
16904 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16905 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16906 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16907 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16908
16909 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16910
16911 ins_encode %{
16912 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16913 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16914 $tmp3$$Register);
16915 %}
16916 ins_pipe(pipe_class_memory);
16917 %}
16918
16919 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16920 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16921 iRegINoSp tmp3, rFlagsReg cr)
16922 %{
16923 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16924 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16925 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16926 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16927
16928 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16929
16930 ins_encode %{
16931 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16932 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16933 $tmp3$$Register);
16934 %}
16935 ins_pipe(pipe_class_memory);
16936 %}
16937
16938 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16939 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16940 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16941 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16942 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16943 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16944 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16945 ins_encode %{
16946 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16947 $result$$Register, $ztmp1$$FloatRegister,
16948 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16949 $ptmp$$PRegister, true /* isL */);
16950 %}
16951 ins_pipe(pipe_class_memory);
16952 %}
16953
16954 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16955 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16956 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16957 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16958 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16959 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16960 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16961 ins_encode %{
16962 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16963 $result$$Register, $ztmp1$$FloatRegister,
16964 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16965 $ptmp$$PRegister, false /* isL */);
16966 %}
16967 ins_pipe(pipe_class_memory);
16968 %}
16969
16970 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16971 iRegI_R0 result, rFlagsReg cr)
16972 %{
16973 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16974 match(Set result (StrEquals (Binary str1 str2) cnt));
16975 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16976
16977 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16978 ins_encode %{
16979 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16980 __ string_equals($str1$$Register, $str2$$Register,
16981 $result$$Register, $cnt$$Register);
16982 %}
16983 ins_pipe(pipe_class_memory);
16984 %}
16985
16986 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16987 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16988 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16989 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16990 iRegP_R10 tmp, rFlagsReg cr)
16991 %{
16992 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16993 match(Set result (AryEq ary1 ary2));
16994 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16995 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16996 TEMP vtmp6, TEMP vtmp7, KILL cr);
16997
16998 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16999 ins_encode %{
17000 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17001 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17002 $result$$Register, $tmp$$Register, 1);
17003 if (tpc == nullptr) {
17004 ciEnv::current()->record_failure("CodeCache is full");
17005 return;
17006 }
17007 %}
17008 ins_pipe(pipe_class_memory);
17009 %}
17010
17011 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17012 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17013 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17014 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17015 iRegP_R10 tmp, rFlagsReg cr)
17016 %{
17017 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17018 match(Set result (AryEq ary1 ary2));
17019 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17020 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17021 TEMP vtmp6, TEMP vtmp7, KILL cr);
17022
17023 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17024 ins_encode %{
17025 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17026 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17027 $result$$Register, $tmp$$Register, 2);
17028 if (tpc == nullptr) {
17029 ciEnv::current()->record_failure("CodeCache is full");
17030 return;
17031 }
17032 %}
17033 ins_pipe(pipe_class_memory);
17034 %}
17035
17036 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17037 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17038 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17039 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17040 %{
17041 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17042 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17043 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17044
17045 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
17046 ins_encode %{
17047 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17048 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17049 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17050 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17051 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17052 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17053 (BasicType)$basic_type$$constant);
17054 if (tpc == nullptr) {
17055 ciEnv::current()->record_failure("CodeCache is full");
17056 return;
17057 }
17058 %}
17059 ins_pipe(pipe_class_memory);
17060 %}
17061
17062 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17063 %{
17064 match(Set result (CountPositives ary1 len));
17065 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17066 format %{ "count positives byte[] $ary1,$len -> $result" %}
17067 ins_encode %{
17068 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17069 if (tpc == nullptr) {
17070 ciEnv::current()->record_failure("CodeCache is full");
17071 return;
17072 }
17073 %}
17074 ins_pipe( pipe_slow );
17075 %}
17076
17077 // fast char[] to byte[] compression
17078 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17079 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17080 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17081 iRegI_R0 result, rFlagsReg cr)
17082 %{
17083 match(Set result (StrCompressedCopy src (Binary dst len)));
17084 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17085 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17086
17087 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17088 ins_encode %{
17089 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17090 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17091 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17092 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17093 %}
17094 ins_pipe(pipe_slow);
17095 %}
17096
17097 // fast byte[] to char[] inflation
17098 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17099 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17100 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17101 %{
17102 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17103 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17104 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17105 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17106
17107 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17108 ins_encode %{
17109 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17110 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17111 $vtmp2$$FloatRegister, $tmp$$Register);
17112 if (tpc == nullptr) {
17113 ciEnv::current()->record_failure("CodeCache is full");
17114 return;
17115 }
17116 %}
17117 ins_pipe(pipe_class_memory);
17118 %}
17119
17120 // encode char[] to byte[] in ISO_8859_1
17121 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17122 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17123 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17124 iRegI_R0 result, rFlagsReg cr)
17125 %{
17126 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17127 match(Set result (EncodeISOArray src (Binary dst len)));
17128 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17129 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17130
17131 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17132 ins_encode %{
17133 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17134 $result$$Register, false,
17135 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17136 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17137 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17138 %}
17139 ins_pipe(pipe_class_memory);
17140 %}
17141
17142 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17143 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17144 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17145 iRegI_R0 result, rFlagsReg cr)
17146 %{
17147 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17148 match(Set result (EncodeISOArray src (Binary dst len)));
17149 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17150 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17151
17152 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17153 ins_encode %{
17154 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17155 $result$$Register, true,
17156 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17157 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17158 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17159 %}
17160 ins_pipe(pipe_class_memory);
17161 %}
17162
17163 //----------------------------- CompressBits/ExpandBits ------------------------
17164
17165 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17166 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17167 match(Set dst (CompressBits src mask));
17168 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17169 format %{ "mov $tsrc, $src\n\t"
17170 "mov $tmask, $mask\n\t"
17171 "bext $tdst, $tsrc, $tmask\n\t"
17172 "mov $dst, $tdst"
17173 %}
17174 ins_encode %{
17175 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17176 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17177 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17178 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17179 %}
17180 ins_pipe(pipe_slow);
17181 %}
17182
17183 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17184 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17185 match(Set dst (CompressBits (LoadI mem) mask));
17186 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17187 format %{ "ldrs $tsrc, $mem\n\t"
17188 "ldrs $tmask, $mask\n\t"
17189 "bext $tdst, $tsrc, $tmask\n\t"
17190 "mov $dst, $tdst"
17191 %}
17192 ins_encode %{
17193 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17194 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17195 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17196 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17197 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17198 %}
17199 ins_pipe(pipe_slow);
17200 %}
17201
17202 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17203 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17204 match(Set dst (CompressBits src mask));
17205 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17206 format %{ "mov $tsrc, $src\n\t"
17207 "mov $tmask, $mask\n\t"
17208 "bext $tdst, $tsrc, $tmask\n\t"
17209 "mov $dst, $tdst"
17210 %}
17211 ins_encode %{
17212 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17213 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17214 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17215 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17216 %}
17217 ins_pipe(pipe_slow);
17218 %}
17219
17220 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17221 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17222 match(Set dst (CompressBits (LoadL mem) mask));
17223 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17224 format %{ "ldrd $tsrc, $mem\n\t"
17225 "ldrd $tmask, $mask\n\t"
17226 "bext $tdst, $tsrc, $tmask\n\t"
17227 "mov $dst, $tdst"
17228 %}
17229 ins_encode %{
17230 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17231 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17232 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17233 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17234 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17235 %}
17236 ins_pipe(pipe_slow);
17237 %}
17238
17239 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17240 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17241 match(Set dst (ExpandBits src mask));
17242 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17243 format %{ "mov $tsrc, $src\n\t"
17244 "mov $tmask, $mask\n\t"
17245 "bdep $tdst, $tsrc, $tmask\n\t"
17246 "mov $dst, $tdst"
17247 %}
17248 ins_encode %{
17249 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17250 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17251 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17252 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17253 %}
17254 ins_pipe(pipe_slow);
17255 %}
17256
17257 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17258 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17259 match(Set dst (ExpandBits (LoadI mem) mask));
17260 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17261 format %{ "ldrs $tsrc, $mem\n\t"
17262 "ldrs $tmask, $mask\n\t"
17263 "bdep $tdst, $tsrc, $tmask\n\t"
17264 "mov $dst, $tdst"
17265 %}
17266 ins_encode %{
17267 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17268 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17269 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17270 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17271 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17272 %}
17273 ins_pipe(pipe_slow);
17274 %}
17275
17276 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17277 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17278 match(Set dst (ExpandBits src mask));
17279 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17280 format %{ "mov $tsrc, $src\n\t"
17281 "mov $tmask, $mask\n\t"
17282 "bdep $tdst, $tsrc, $tmask\n\t"
17283 "mov $dst, $tdst"
17284 %}
17285 ins_encode %{
17286 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17287 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17288 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17289 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17290 %}
17291 ins_pipe(pipe_slow);
17292 %}
17293
17294
17295 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17296 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17297 match(Set dst (ExpandBits (LoadL mem) mask));
17298 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17299 format %{ "ldrd $tsrc, $mem\n\t"
17300 "ldrd $tmask, $mask\n\t"
17301 "bdep $tdst, $tsrc, $tmask\n\t"
17302 "mov $dst, $tdst"
17303 %}
17304 ins_encode %{
17305 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17306 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17307 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17308 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17309 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17310 %}
17311 ins_pipe(pipe_slow);
17312 %}
17313
17314 //----------------------------- Reinterpret ----------------------------------
17315 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17316 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17317 match(Set dst (ReinterpretHF2S src));
17318 format %{ "reinterpretHF2S $dst, $src" %}
17319 ins_encode %{
17320 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17321 %}
17322 ins_pipe(pipe_slow);
17323 %}
17324
17325 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17326 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17327 match(Set dst (ReinterpretS2HF src));
17328 format %{ "reinterpretS2HF $dst, $src" %}
17329 ins_encode %{
17330 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17331 %}
17332 ins_pipe(pipe_slow);
17333 %}
17334
17335 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17336 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17337 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
17338 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
17339 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
17340 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17341 // can be omitted in this pattern, resulting in -
17342 // fcvt $dst, $src // Convert float to half-precision float
17343 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17344 %{
17345 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17346 format %{ "convF2HFAndS2HF $dst, $src" %}
17347 ins_encode %{
17348 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17349 %}
17350 ins_pipe(pipe_slow);
17351 %}
17352
17353 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17354 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17355 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
17356 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17357 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
17358 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17359 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17360 // resulting in -
17361 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
17362 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17363 %{
17364 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17365 format %{ "convHF2SAndHF2F $dst, $src" %}
17366 ins_encode %{
17367 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17368 %}
17369 ins_pipe(pipe_slow);
17370 %}
17371
17372 // ============================================================================
17373 // This name is KNOWN by the ADLC and cannot be changed.
17374 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17375 // for this guy.
17376 instruct tlsLoadP(thread_RegP dst)
17377 %{
17378 match(Set dst (ThreadLocal));
17379
17380 ins_cost(0);
17381
17382 format %{ " -- \t// $dst=Thread::current(), empty" %}
17383
17384 size(0);
17385
17386 ins_encode( /*empty*/ );
17387
17388 ins_pipe(pipe_class_empty);
17389 %}
17390
17391 //----------PEEPHOLE RULES-----------------------------------------------------
17392 // These must follow all instruction definitions as they use the names
17393 // defined in the instructions definitions.
17394 //
17395 // peepmatch ( root_instr_name [preceding_instruction]* );
17396 //
17397 // peepconstraint %{
17398 // (instruction_number.operand_name relational_op instruction_number.operand_name
17399 // [, ...] );
17400 // // instruction numbers are zero-based using left to right order in peepmatch
17401 //
17402 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17403 // // provide an instruction_number.operand_name for each operand that appears
17404 // // in the replacement instruction's match rule
17405 //
17406 // ---------VM FLAGS---------------------------------------------------------
17407 //
17408 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17409 //
17410 // Each peephole rule is given an identifying number starting with zero and
17411 // increasing by one in the order seen by the parser. An individual peephole
17412 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17413 // on the command-line.
17414 //
17415 // ---------CURRENT LIMITATIONS----------------------------------------------
17416 //
17417 // Only match adjacent instructions in same basic block
17418 // Only equality constraints
17419 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17420 // Only one replacement instruction
17421 //
17422 // ---------EXAMPLE----------------------------------------------------------
17423 //
17424 // // pertinent parts of existing instructions in architecture description
17425 // instruct movI(iRegINoSp dst, iRegI src)
17426 // %{
17427 // match(Set dst (CopyI src));
17428 // %}
17429 //
17430 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17431 // %{
17432 // match(Set dst (AddI dst src));
17433 // effect(KILL cr);
17434 // %}
17435 //
17436 // // Change (inc mov) to lea
17437 // peephole %{
17438 // // increment preceded by register-register move
17439 // peepmatch ( incI_iReg movI );
17440 // // require that the destination register of the increment
17441 // // match the destination register of the move
17442 // peepconstraint ( 0.dst == 1.dst );
17443 // // construct a replacement instruction that sets
17444 // // the destination to ( move's source register + one )
17445 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17446 // %}
17447 //
17448
17449 // Implementation no longer uses movX instructions since
17450 // machine-independent system no longer uses CopyX nodes.
17451 //
17452 // peephole
17453 // %{
17454 // peepmatch (incI_iReg movI);
17455 // peepconstraint (0.dst == 1.dst);
17456 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17457 // %}
17458
17459 // peephole
17460 // %{
17461 // peepmatch (decI_iReg movI);
17462 // peepconstraint (0.dst == 1.dst);
17463 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17464 // %}
17465
17466 // peephole
17467 // %{
17468 // peepmatch (addI_iReg_imm movI);
17469 // peepconstraint (0.dst == 1.dst);
17470 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17471 // %}
17472
17473 // peephole
17474 // %{
17475 // peepmatch (incL_iReg movL);
17476 // peepconstraint (0.dst == 1.dst);
17477 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17478 // %}
17479
17480 // peephole
17481 // %{
17482 // peepmatch (decL_iReg movL);
17483 // peepconstraint (0.dst == 1.dst);
17484 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17485 // %}
17486
17487 // peephole
17488 // %{
17489 // peepmatch (addL_iReg_imm movL);
17490 // peepconstraint (0.dst == 1.dst);
17491 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17492 // %}
17493
17494 // peephole
17495 // %{
17496 // peepmatch (addP_iReg_imm movP);
17497 // peepconstraint (0.dst == 1.dst);
17498 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17499 // %}
17500
17501 // // Change load of spilled value to only a spill
17502 // instruct storeI(memory mem, iRegI src)
17503 // %{
17504 // match(Set mem (StoreI mem src));
17505 // %}
17506 //
17507 // instruct loadI(iRegINoSp dst, memory mem)
17508 // %{
17509 // match(Set dst (LoadI mem));
17510 // %}
17511 //
17512
17513 //----------SMARTSPILL RULES---------------------------------------------------
17514 // These must follow all instruction definitions as they use the names
17515 // defined in the instructions definitions.
17516
17517 // Local Variables:
17518 // mode: c++
17519 // End: